def save_main(starttime, endtime, jobid, program_name, corehour, title,
title_IOmode, host_t):
global jobid_print
jobid_print = jobid
try:
time1, time2, iplist, min_time, max_time = compute_pre(
starttime, endtime, jobid)
except Exception as e:
print e
results_message = []
results_host = []
count_ip = len(iplist) / node_count
remainder = len(iplist) % node_count
if time1[8:10] == time2[8:10]:
index = [time1[0:4] + '.' + time1[5:7] + '.' + time1[8:10]]
else:
index = compute_index(time1, time2)
for lnd in xrange(len(index)):
if count_ip > 0:
for c1 in xrange(count_ip):
results_message_tmp, results_host_tmp = es_search.search(
time1, time2,
iplist[c1 * node_count:c1 * node_count + node_count],
index[lnd], host_t)
results_message += results_message_tmp
results_host += results_host_tmp
if remainder > 0:
results_message_tmp, results_host_tmp = es_search.search(
time1, time2,
iplist[count_ip * node_count:count_ip * node_count +
remainder], index[lnd], host_t)
else:
results_message_tmp, results_host_tmp = es_search.search(
time1, time2, iplist, index[lnd], host_t)
results_message += results_message_tmp
results_host += results_host_tmp
try:
resultr_band, resultw_band, resultr_iops, resultw_iops, resultr_open, resultw_close, resultr_size, resultw_size, dictr, dictw, file_all_count = deal_all_message(
results_message, results_host, min_time, max_time)
del results_message
del results_host
gc.collect()
except Exception as e:
print e
saveIOBANDWIDTH(resultr_band, resultw_band, title_IOmode, jobid,
program_name, corehour, min_time)
saveMDS(resultr_open, resultw_close, title_IOmode, jobid, program_name,
corehour, min_time)
saveprocess(dictr, 'r', title_IOmode, min_time, max_time, jobid,
program_name, corehour)
saveprocess(dictw, 'b', title_IOmode, min_time, max_time, jobid,
program_name, corehour)
savesize(resultr_size, 'READ', title, jobid, program_name)
savesize(resultw_size, 'WRITE', title, jobid, program_name)
max_PE_r = sum_process(dictr, 'r', min_time, max_time)
max_PE_w = sum_process(dictw, 'b', min_time, max_time)
sum_resultr, sum_resultw, average_resultr, average_resultw, count_resultr, count_resultw, count_resultrw, average_resultr, average_resultw = sum_IOBANDWIDTH(
resultr_band, resultw_band)
sum_IOPS_r, sum_IOPS_w, count_IOPS_r, count_IOPS_w, count_IOPS_rw, count_IOPS_rw_all, average_IOPS_r, average_IOPS_w = sum_IOPS(
resultr_iops, resultw_iops, resultr_band, resultw_band)
sum_MDS_o, sum_MDS_c, count_MDS_o, count_MDS_c, count_MDS_oc, average_MDS_o, average_MDS_c = sum_MDS(
resultr_open, resultw_close)
return (sum_resultr, sum_resultw, count_resultr, count_resultw,
count_resultrw, average_resultr, average_resultw, sum_IOPS_r,
sum_IOPS_w, count_IOPS_r, count_IOPS_w, count_IOPS_rw,
count_IOPS_rw_all, average_IOPS_r, average_IOPS_w, sum_MDS_o,
sum_MDS_c, count_MDS_o, count_MDS_c, count_MDS_oc, average_MDS_o,
average_MDS_c, max_PE_r, max_PE_w, file_all_count)
def show_job_all(jobid):
try:
time1, time2, iplist, min_time, max_time, fwd_list \
= compute_pre_with_jobid(jobid)
except Exception as e:
print e
print time1
print time2
print min_time
print max_time
if (time1[8:10] == time2[8:10]):
index = [time1[0:4] + "." + time1[5:7] + "." + time1[8:10]]
else:
index = compute_index(time1, time2)
split_time_start, split_time_end = split_time(time1, time2, index)
print "index: ", index
count_ip = len(iplist) / size_ip
remainder = len(iplist) % size_ip
results_message = []
results_host = []
print "count_ip: ", count_ip
print "remainder: ", remainder
# print len(index)
try:
for lnd in xrange(len(index)):
print "Now search index: ", index[lnd]
if (count_ip > 0):
for c1 in xrange(count_ip):
print "Now search ip iteration: ", c1
try:
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist[c1*size_ip:c1*size_ip+size_ip], index[lnd], 10)
results_message += results_message_tmp
results_host += results_host_tmp
except Exception as e:
print e
if (remainder > 0):
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist[count_ip*size_ip:count_ip*size_ip + remainder], index[lnd], 1)
else:
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist, index[lnd], 1)
except Exception as e:
print e
results_message_tmp = ""
results_host_tmp = ""
print "Warning, The ES index may be missing!!!!!"
results_message += results_message_tmp
results_host += results_host_tmp
print 'message length: ', len(results_message)
resultr_band,resultw_band,resultr_iops,resultw_iops,resultr_open,\
resultw_close, resultr_size, resultw_size, dictr, dictw, file_count, \
file_open, fd_info, fwd_file_map \
= deal_all_message(results_message, results_host, min_time, max_time)
print "JobID:", jobid
print 'file_count: ', file_count
show_IOBANDWIDTH(resultr_band, resultw_band, jobid)
show_IOPS(resultr_iops, resultw_iops, resultr_band, resultw_band, jobid)
show_MDS(resultr_open, resultw_close, jobid)
pe_r = show_process(dictr, 'r', min_time, max_time, jobid)
pe_w = show_process(dictw, 'b', min_time, max_time, jobid)
file_open_count = [len(file_open[i]) for i in range(len(file_open))]
if options.draw == True:
draw_2d(resultr_band, resultw_band, 'IOBW')
draw_2d(resultr_iops, resultw_iops, 'IOPS')
draw_2d(pe_r, pe_w, 'PE')
draw_d(file_open_count, 'Unique File Count')
tmp_file_name = './results_job_data/tmp_csv/' + jobid + '.csv'
print tmp_file_name
save_tmp(resultr_band, resultw_band, resultr_iops, resultw_iops,
resultr_open, resultw_close, pe_r, pe_w, tmp_file_name)
if options.trace == True:
trace_file_name = './results_job_data/job_trace/' + jobid + '.csv'
save_trace(results_message, results_host, trace_file_name)
if options.fd_info == True:
for fd in fd_info:
print "[FD: %s] [filename: %s] [total_time: %d] \n\
[sum_read_size: %f MB] [sum_write_size: %f MB] \n\
[start_time: %s] [end_time: %s]"\
%(fd, fd_info[fd]['file_name'], fd_info[fd]['total_time'], \
fd_info[fd]['sum_read_size'], fd_info[fd]['sum_write_size'], \
fd_info[fd]['start_time'], fd_info[fd]['end_time'])
if options.ost_info == True:
file_open_set = set()
for i in range(len(file_open)):
file_open_set |= file_open[i]
if ('NULL' in file_open_set):
file_open_set.remove('NULL')
file_list = ""
for file1 in file_open_set:
file_list += file1
file_list += ','
ost_list = search_ost(file_list)
if (len(ost_list) == 0):
print "Files have been deleted."
else:
host = 90
bandr, bandw = get_ost_data(time1, time2, ost_list, index, host)
show_ost_data(ost_list, bandr, bandw)
if options.fwd_info == True:
for fwd in fwd_file_map:
file_list = ''
for file1 in fwd_file_map[fwd]:
file_list += file1
file_list += ','
ost_list = search_ost(file_list)
if (len(ost_list) == 0):
print "Files have been deleted."
else:
host = 87
bandr, bandw = get_fwd_data(time1, time2, fwd, ost_list, index,
host)
print "fwd: %s info: \n" % fwd
print bandr
print bandw
def save_main(jobid, title):
# print jobid
try:
jobID, CNC ,run_time, corehour, time1, time2, \
iplist, min_time, max_time = compute_pre_with_jobid(jobid)
except Exception as e:
print e
results_message = []
results_host = []
count_ip = len(iplist) / node_count
remainder = len(iplist) % node_count
if (time1[8:10] == time2[8:10]):
index = [time1[0:4] + "." + time1[5:7] + "." + time1[8:10]]
else:
index = compute_index(time1, time2)
host_t = 1
try:
for lnd in xrange(len(index)):
if (count_ip > 0):
for c1 in xrange(count_ip):
try:
results_message_tmp, results_host_tmp \
= es_search.search(time1, time2,\
iplist[c1*node_count:c1*node_count+node_count], index[lnd], host_t)
results_message += results_message_tmp
results_host += results_host_tmp
except Exception as e:
print e
if (remainder > 0):
results_message_tmp, results_host_tmp = \
es_search.search(time1, time2,\
iplist[count_ip*node_count:count_ip*node_count+remainder], index[lnd], host_t)
else:
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist, index[lnd], host_t)
except Exception as e:
print e
results_message += results_message_tmp
results_host += results_host_tmp
try:
resultr_band,resultw_band,resultr_iops,resultw_iops,resultr_open, resultw_close, \
resultr_size, resultw_size, dictr, dictw, file_all_count, file_open \
= deal_all_message(results_message, results_host, min_time, max_time)
del results_message
del results_host
gc.collect()
except Exception as e:
print e
pe_r = process(dictr, 'r', min_time, max_time)
pe_w = process(dictw, 'b', min_time, max_time)
del dictr
del dictw
gc.collect()
save_all(jobID, resultr_band, resultw_band, \
resultr_iops, resultw_iops, resultr_open, resultw_close, pe_r, pe_w, file_open, title)
save_IOphase_mode(jobID, CNC ,run_time, corehour, resultr_band, resultw_band, \
resultr_iops, resultw_iops, resultr_open, resultw_close, resultr_size, resultw_size, \
pe_r, pe_w, file_all_count, title)
del file_open
del resultr_band
del resultw_band
del resultr_iops
del resultw_iops
del resultr_open
del resultw_close
del resultr_size
del resultw_size
del pe_r
del pe_w
gc.collect()
def compute_result(jobid):
try:
time1, time2, iplist, min_time, max_time \
= compute_pre_with_jobid(jobid)
except Exception as e:
print e
if (time1[8:10] == time2[8:10]):
index = [time1[0:4] + "." + time1[5:7] + "." + time1[8:10]]
else:
index = compute_index(time1, time2)
count_ip = len(iplist) / size_ip
remainder = len(iplist) % size_ip
results_message = []
results_host = []
try:
for lnd in xrange(len(index)):
if (count_ip > 0):
for c1 in xrange(count_ip):
try:
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist[c1*size_ip:c1*size_ip+size_ip], index[lnd], 10)
results_message += results_message_tmp
results_host += results_host_tmp
except Exception as e:
print e
if (remainder > 0):
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist[count_ip*size_ip:count_ip*size_ip + remainder], index[lnd], 1)
else:
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist, index[lnd], 1)
except Exception as e:
print e
results_message += results_message_tmp
results_host += results_host_tmp
try:
resultr_band,resultw_band,resultr_iops,resultw_iops,resultr_open,\
resultw_close, resultr_size, resultw_size, dictr, dictw, file_count, file_open\
= deal_all_message(results_message, results_host, min_time, max_time)
del results_message
del results_host
gc.collect()
except Exception as e:
print e
max_PE_r = sum_process(dictr, 'r', min_time, max_time)
max_PE_w = sum_process(dictw, 'b', min_time, max_time)
sum_resultr, sum_resultw, average_resultr, average_resultw, \
count_resultr, count_resultw, count_resultrw, \
average_resultr, average_resultw, \
= sum_IOBANDWIDTH(resultr_band,resultw_band)
sum_IOPS_r, sum_IOPS_w, count_IOPS_r, count_IOPS_w, \
count_IOPS_rw, count_IOPS_rw_all, \
average_IOPS_r, average_IOPS_w, \
= sum_IOPS(resultr_iops,resultw_iops,resultr_band,resultw_band)
sum_MDS_o, sum_MDS_c, \
count_MDS_o, count_MDS_c, count_MDS_oc, \
average_MDS_o, average_MDS_c, \
= sum_MDS(resultr_open,resultw_close)
return sum_resultr, sum_resultw,\
count_resultr, count_resultw, count_resultrw, average_resultr, average_resultw, \
sum_IOPS_r, sum_IOPS_w, count_IOPS_r, count_IOPS_w, \
count_IOPS_rw, count_IOPS_rw_all, \
average_IOPS_r, average_IOPS_w, \
sum_MDS_o, sum_MDS_c, \
count_MDS_o, count_MDS_c, count_MDS_oc, \
average_MDS_o, average_MDS_c, \
max_PE_r , max_PE_w, file_count
def detect(jobid):
try:
time1, time2, iplist, min_time, max_time, fwdlist = compute_pre_with_jobid(
jobid)
except Exception as e:
print e
if (time1[8:10] == time2[8:10]):
index = [time1[0:4] + "." + time1[5:7] + "." + time1[8:10]]
else:
index = compute_index(time1, time2)
#split_time_start, split_time_end = split_time(time1, time2, index)
print "index: ", index, " time1:", time1, " time2:", time2
job_cmp = dict()
for r in fwdlist:
iplist = list(fwdlist[r])
count_ip = len(iplist) / size_ip
remainder = len(iplist) % size_ip
results_message = []
results_host = []
print "Now fwd: ", r, " Now iplist: ", iplist
try:
for lnd in xrange(len(index)):
print "Now search index: ", index[lnd]
if (count_ip > 0):
print "count_ip ", count_ip
for c1 in xrange(count_ip):
print "Now search ip iteration: ", c1
try:
results_message_tmp, results_host_tmp = es_search.search(
time1, time2,
iplist[c1 * size_ip:c1 * size_ip + size_ip],
index[lnd], 10)
results_message += results_message_tmp
results_host += results_host_tmp
except Exception as e:
print e
if (remainder > 0):
results_message_tmp, results_host_tmp = es_search.search(
time1, time2,
iplist[count_ip * size_ip:count_ip * size_ip +
remainder], index[lnd], 1)
else:
results_message_tmp, results_host_tmp = es_search.search(
time1, time2, iplist, index[lnd], 1)
except Exception as e:
print e
results_message_tmp = ""
results_host_tmp = ""
print "Warning, The ES index may be missing!!!!!"
results_message += results_message_tmp
results_host += results_host_tmp
print 'message length: ', len(results_message)
(resultr_band, resultw_band, resultr_iops, resultw_iops, resultr_open,
resultw_close, resultr_size, resultw_size, dictr, dictw, file_count,
file_open1, fd_info,
fwd_file_map1) = deal_all_message(results_message, results_host,
min_time, max_time)
b1, b2, b3 = show_IOBANDWIDTH(resultr_band, resultw_band)
job_cmp[r] = [b1, b2, b3]
print "begin fwd search..."
print "fwdlist: ", fwdlist.keys()
print time1, time2
time1_f = time1
time2_f = time2
job_fwd = dict()
host = 87
ostlist = []
fwd_band_sum = 0.0
fwd_volume_sum = 0.0
fwd_count = 0
for fwd in fwdlist:
#print "fwd: ",fwd
fwd_count += 1
ostlist1, fwd_bandr, fwd_bandw = get_fwd_data(time1, time2, fwd, index,
host)
b1, b2, b3 = show_IOBANDWIDTH(fwd_bandr, fwd_bandw)
fwd_band_sum += b2
fwd_volume_sum += b3
job_fwd[fwd] = [b1, b2, b3]
for ostid in ostlist1:
if ostid not in ostlist:
ostlist.append(ostid)
fwdlist_issue_next = []
for fwd in fwdlist:
if job_fwd[fwd][0] < (
fwd_band_sum / fwd_count /
2) and job_fwd[fwd][1] < (fwd_band_sum / fwd_count / 2) and (
fwd_volume_sum / 5 * 3 / fwd_count < job_fwd[fwd][2]):
fwdlist_issue_next.append(fwd)
print "fwd anomaly check: ", fwdlist_issue_next
for ti in [1, 4, 12, 96]:
fwdlist_issue = []
for r in fwdlist_issue_next:
fwdlist_issue.append(r)
time1_std = time1_f[:10] + ' ' + time1_f[11:-5]
time2_std = time2_f[:10] + ' ' + time2_f[11:-5]
t1 = datetime.datetime.strptime(time1_std, '%Y-%m-%d %H:%M:%S')
t2 = datetime.datetime.strptime(time2_std, '%Y-%m-%d %H:%M:%S')
time_interval = datetime.timedelta(hours=ti)
tt1 = t1 - time_interval
tt2 = t2 + time_interval
time1_f = str(tt1)[:10] + 'T' + str(tt1)[11:] + '.000Z'
time2_f = str(tt2)[:10] + 'T' + str(tt2)[11:] + '.000Z'
fwd_band_sum = 0.0
fwd_volume_sum = 0.0
fwd_count = 0
for fwd in fwdlist_issue:
#print "fwd: ",fwd
fwd_count += 1
ostlist1, fwd_bandr, fwd_bandw = get_fwd_data(
time1_f, time2_f, fwd, index, host)
b1, b2, b3 = show_IOBANDWIDTH(fwd_bandr, fwd_bandw)
fwd_band_sum += b2
fwd_volume_sum += b3
job_fwd[fwd] = [b1, b2, b3]
for ostid in ostlist1:
if ostid not in ostlist:
ostlist.append(ostid)
fwdlist_issue_next = []
for fwd in fwdlist_issue:
if job_fwd[fwd][0] < (fwd_band_sum / fwd_count / 2) and job_fwd[
fwd][1] < (fwd_band_sum / fwd_count / 2) and (
fwd_volume_sum / 5 * 3 / fwd_count < job_fwd[fwd][2]):
fwdlist_issue_next.append(fwd)
print "Anomaly time: ", ti, " ", list(
set(fwdlist_issue).difference(set(fwdlist_issue_next)))
print "begin OST search..."
print "OSTlist: ", ostlist
job_ost = dict()
host = 88
time1_o = time1
time2_o = time2
ost_bandr, ost_bandw = get_ost_data(time1, time2, ostlist, index, host)
ost_band_sum = 0.0
ost_volume_sum = 0.0
ost_count = 0
for ostid in ostlist:
ost_count += 1
tmpr = []
tmpw = []
for j in range(len(ost_bandr[0])):
tmpr.append(ost_bandr[ostid][j])
tmpw.append(ost_bandw[ostid][j])
b1, b2, b3 = show_IOBANDWIDTH(tmpr, tmpw)
ost_band_sum += b2
ost_volume_sum += b3
job_ost[ostid] = [b1, b2, b3]
ostlist_issue_next = []
for ostid in ostlist:
try:
if job_ost[ostid][0] < (
ost_band_sum / ost_count / 2
) and job_ost[ostid][1] < (ost_band_sum / ost_count / 2) and (
ost_volume_sum / 5 * 3 / ost_count < job_ost[ostid][2]):
ostlist_issue_next.append(ostid)
except Exception as e:
print e
print "ost anomaly check: ", ostlist_issue_next
for ti in [1, 4, 12, 96]:
print "begin time: ", ti
ostlist_issue = []
for r in ostlist_issue_next:
ostlist_issue.append(r)
if len(ostlist_issue) < 1:
break
time1_std = time1_o[:10] + ' ' + time1_o[11:-5]
time2_std = time2_o[:10] + ' ' + time2_o[11:-5]
t1 = datetime.datetime.strptime(time1_std, '%Y-%m-%d %H:%M:%S')
t2 = datetime.datetime.strptime(time2_std, '%Y-%m-%d %H:%M:%S')
time_interval = datetime.timedelta(hours=ti)
tt1 = t1 - time_interval
tt2 = t2 + time_interval
time1_o = str(tt1)[:10] + 'T' + str(tt1)[11:] + '.000Z'
time2_o = str(tt2)[:10] + 'T' + str(tt2)[11:] + '.000Z'
ost_bandr, ost_bandw = get_ost_data(time1_o, time2_o, ostlist_issue,
index, host)
ost_band_sum = 0.0
ost_volume_sum = 0.0
ost_count = 0
for ostid in ostlist_issue:
#print "fwd: ",fwd
ost_count += 1
tmpr = []
tmpw = []
for j in range(len(ost_bandr[0])):
tmpr.append(ost_bandr[ostid][j])
tmpw.append(ost_bandw[ostid][j])
b1, b2, b3 = show_IOBANDWIDTH(tmpr, tmpw)
ost_band_sum += b2
ost_volume_sum += b3
job_ost[ostid] = [b1, b2, b3]
ostlist_issue_next = []
for ostid in ostlist_issue:
try:
if job_ost[ostid][0] < (ost_band_sum / ost_count /
2) and job_ost[ostid][1] < (
ost_band_sum / ost_count / 2) and (
ost_volume_sum / 5 * 3 /
ost_count < job_ost[ostid][2]):
ostlist_issue_next.append(ostid)
except Exception as e:
print e
print "Anomaly time: ", ti, " ", list(
set(ostlist_issue).difference(set(ostlist_issue_next)))
def show_job_all(jobid):
try:
time1, time2, iplist, min_time, max_time, fwd_list \
= compute_pre_with_jobid(jobid)
except Exception as e:
print e
try:
print time1
print time2
print min_time
print max_time
except Exception as e:
print 'Start_time or end_time are not updated.Please wait for job complete or give explicit time format'
sys.exit()
if (time1[8:10] == time2[8:10]):
index = [time1[0:4] + "." + time1[5:7] + "." + time1[8:10]]
else:
index = compute_index(time1, time2)
print "index: ", index
count_ip = len(iplist) / size_ip
remainder = len(iplist) % size_ip
results_message = []
results_host = []
print "count_ip: ", count_ip
print "remainder: ", remainder
# print len(index)
try:
for lnd in xrange(len(index)):
print "Now search index: ", index[lnd]
if (count_ip > 0):
for c1 in xrange(count_ip):
# print "Now search ip iteration: ", iplist[c1*size_ip:c1*size_ip+size_ip]
try:
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist[c1*size_ip:c1*size_ip+size_ip], index[lnd], 10)
results_message += results_message_tmp
results_host += results_host_tmp
except Exception as e:
print e
if (remainder > 0):
# print "Now search ip iteration: ", iplist[count_ip*size_ip:count_ip*size_ip + remainder]
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist[count_ip*size_ip:count_ip*size_ip + remainder], index[lnd], 1)
else:
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist, index[lnd], 1)
except Exception as e:
print e
results_message_tmp = ""
results_host_tmp = ""
print "Warning, The ES index may be missing!!!!!"
results_message += results_message_tmp
results_host += results_host_tmp
host_set1 = set(results_host)
host_set2 = set(iplist)
print 'message length: ', len(results_message)
resultr_band,resultw_band,resultr_iops,resultw_iops,resultr_open,\
resultw_close, resultr_size, resultw_size, dictr, dictw, file_count, \
file_open, fd_info, fwd_file_map \
= deal_all_message(results_message, results_host, min_time, max_time)
print 'file_count: ', file_count
show_IOBANDWIDTH(resultr_band, resultw_band, jobid)
show_IOPS(resultr_iops, resultw_iops, resultr_band, resultw_band, jobid)
show_MDS(resultr_open, resultw_close, jobid)
pe_r = show_process(dictr, 'r', min_time, max_time, jobid)
pe_w = show_process(dictw, 'b', min_time, max_time, jobid)
file_open_count = [len(file_open[i]) for i in range(len(file_open))]
if options.draw == True:
if (not os.path.exists(
'/home/export/mount_test/swstorage/results_job_data/job_trace/'
+ jobid)):
os.mkdir(
'/home/export/mount_test/swstorage/results_job_data/job_trace/'
+ jobid)
front_file_name = '/home/export/mount_test/swstorage/results_job_data/job_trace/' + jobid + '/front_end_bw' + '.csv'
tmp_file_name = '/home/export/mount_test/swstorage/results_job_data/job_trace/' + jobid + '/bw_iops_mds' + '.csv'
save_tmp(resultr_band, resultw_band, resultr_iops, resultw_iops, \
resultr_open, resultw_close, pe_r, pe_w, tmp_file_name)
save_front_bw(resultr_band, resultw_band, front_file_name)
draw_2d(resultr_band, resultw_band, 'IOBW')
draw_2d(resultr_iops, resultw_iops, 'IOPS')
draw_2d(pe_r, pe_w, 'PE')
draw_d(file_open_count, 'Unique File Count')
if options.trace == True:
trace_file_name = '/home/export/mount_test/swstorage/results_job_data/job_trace/' + jobid + '.csv'
save_trace(results_message, results_host, trace_file_name)
if options.fd_info == True:
for fd in fd_info:
print "[FD: %s] [filename: %s] [total_time: %d] \n\
[sum_read_size: %f MB] [sum_write_size: %f MB] \n\
[start_time: %s] [end_time: %s]"\
%(fd, fd_info[fd]['file_name'], fd_info[fd]['total_time'], \
fd_info[fd]['sum_read_size'], fd_info[fd]['sum_write_size'], \
fd_info[fd]['start_time'], fd_info[fd]['end_time'])
if options.ost_info == True:
host = 90
length = max_time - min_time
file_open_set = set()
for i in range(len(file_open)):
file_open_set |= file_open[i]
if ('NULL' in file_open_set):
file_open_set.remove('NULL')
file_open_list = list(file_open_set)
ost_list, target, file_ost_map = get_ost_list(file_open_list)
front_ost_bw_r, front_ost_bw_w = ost_bw_front(fd_info, file_ost_map,
length)
save_fd_info(fd_info, 'test2_job.log')
# results_message, results_host = search_interval_glb(time1, time2, index[0])
# resultr_band,resultw_band,resultr_iops,resultw_iops,resultr_open,\
# resultw_close, resultr_size, resultw_size, dictr, dictw, file_count, \
# file_open_glb, fd_info_glb, fwd_file_map \
# = deal_all_message(results_message, results_host, min_time, max_time)
#
# save_fd_info(fd_info_glb, 'test2_glb.log')
#
# print 'message length: ', len(results_message)
# file_open_set_glb = set()
# for i in range(len(file_open_glb)):
# file_open_set_glb |= file_open_glb[i]
# if('NULL' in file_open_set_glb):
# file_open_set_glb.remove('NULL')
# file_open_list_glb = list(file_open_set_glb)
# ost_list_glb, target_glb, file_ost_map_glb = get_ost_list(file_open_list_glb)
# glb_front_ost_bw_r, glb_front_ost_bw_w = ost_bw_front(fd_info_glb, file_ost_map_glb, length)
#
# for i in range(length):
# bw_r = 0
# bw_w = 0
# glb_bw_r = 0
# glb_bw_w = 0
# for ost in ost_list:
# ostid = int(ost)
# bw_r += front_ost_bw_r[ostid][i]
# bw_w += front_ost_bw_w[ostid][i]
# glb_bw_r += glb_front_ost_bw_r[ostid][i]
# glb_bw_w += glb_front_ost_bw_w[ostid][i]
# print "bw_r: %f glb_bw_r: %f bw_w: %f glb_bw_w: %f"%(bw_r, glb_bw_r, bw_w, glb_bw_w)
if (len(ost_list) == 0):
print "Files have been deleted."
else:
bandr, bandw = get_ost_data(time1, time2, ost_list, index, host)
if (not os.path.exists(
'/home/export/mount_test/swstorage/results_job_data/job_trace/'
+ jobid)):
os.mkdir(
'/home/export/mount_test/swstorage/results_job_data/job_trace/'
+ jobid)
ost_file_name = '/home/export/mount_test/swstorage/results_job_data/job_trace/' + jobid + '/back_end' + '.csv'
ost_agg_file_name = '/home/export/mount_test/swstorage/results_job_data/job_trace/' + jobid + '/back_end_agg' + '.csv'
save_back_bw(ost_list, bandr, bandw, ost_file_name)
save_back_bw_agg(ost_list, bandr, bandw, ost_agg_file_name)
if options.fwd_info == True:
if (not os.path.exists(
'/home/export/mount_test/swstorage/results_job_data/job_trace/'
+ jobid)):
os.mkdir(
'/home/export/mount_test/swstorage/results_job_data/job_trace/'
+ jobid)
fwd_file_name = '/home/export/mount_test/swstorage/results_job_data/job_trace/' + jobid + '/fwd' + '.csv'
for fwd in fwd_file_map:
try:
print fwd
print fwd_file_map[fwd]
file_list = ''
if ('NULL' in fwd_file_map[fwd]):
fwd_file_map[fwd].remove('NULL')
file_list = list(fwd_file_map[fwd])
ost_list, target, file_ost_map = get_ost_list(file_list)
if (len(ost_list) == 0):
print "Files have been deleted."
else:
if (target == 'online1'):
print "search online1.\n"
host = 87
bandr, bandw = get_fwd_data(time1, time2, fwd_map_1[fwd], \
ost_list, index, host)
save_fwd_bw(fwd, bandr, bandw, fwd_file_name)
print "fwd: %s info: \n" % fwd_map_1[fwd]
print bandr
print bandw
else:
print "search online2.\n"
host = 89
bandr, bandw = get_fwd_data(time1, time2, fwd_map_2[fwd], \
ost_list, index, host)
save_fwd_bw(fwd, bandr, bandw, fwd_file_name)
print "fwd: %s info: \n" % fwd_map_2[fwd]
print bandr
print bandw
except Exception as e:
print e
def show_job_all(start_time, end_time, jobid):
try:
time1, time2, iplist, min_time, max_time \
= compute_pre(start_time, end_time, jobid)
except Exception as e:
print e
print time1
print time2
print min_time
print max_time
# index = [time1[0:4] + "." + time1[5:7] + "." + time1[8:10]]
if (time1[8:10] == time2[8:10]):
index = [time1[0:4] + "." + time1[5:7] + "." + time1[8:10]]
else:
index = compute_index(time1, time2)
split_time_start, split_time_end = split_time(time1, time2, index)
print "index: ", index
count_ip = len(iplist) / size_ip
remainder = len(iplist) % size_ip
results_message = []
results_host = []
print "count_ip: ", count_ip
print "remainder: ", remainder
# print len(index)
try:
for lnd in xrange(len(index)):
print "Now search index: ", index[lnd]
if (count_ip > 0):
for c1 in xrange(count_ip):
print "Now search ip iteration: ", c1
try:
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist[c1*size_ip:c1*size_ip+size_ip], index[lnd], 10)
results_message += results_message_tmp
results_host += results_host_tmp
except Exception as e:
print e
if (remainder > 0):
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist[count_ip*size_ip:count_ip*size_ip + remainder], index[lnd], 1)
else:
results_message_tmp, results_host_tmp = es_search.search(time1, time2,\
iplist, index[lnd], 1)
except Exception as e:
print e
results_message += results_message_tmp
results_host += results_host_tmp
print len(results_message)
resultr_band,resultw_band,resultr_iops,resultw_iops,resultr_open,\
resultw_close, resultr_size, resultw_size, dictr, dictw, file_count, file_open\
= deal_all_message(results_message, results_host, min_time, max_time)
print 'file_count: ', file_count
show_IOBANDWIDTH(resultr_band, resultw_band, jobid)
show_IOPS(resultr_iops, resultw_iops, resultr_band, resultw_band, jobid)
show_MDS(resultr_open, resultw_close, jobid)
pe_r = show_process(dictr, 'r', min_time, max_time, jobid)
pe_w = show_process(dictw, 'b', min_time, max_time, jobid)
tmp_file_name = '/home/export/mount_test/swstorage/results_job_data/tmp_csv/' + jobid + '.csv'
print tmp_file_name
save_tmp(resultr_band, resultw_band, resultr_iops, resultw_iops,
resultr_open, resultw_close, pe_r, pe_w, tmp_file_name)
draw_2d(resultr_band, pe_r)
draw_2d(resultw_band, pe_w)
