def fwd_deal_message(ost_message, ost_time, start_time, end_time):
length = time_to_sec_fast(end_time) - time_to_sec_fast(start_time)
#print length
fwd_bandr = [0 for i in range(length)]
fwd_bandw = [0 for i in range(length)]
ostlist, bandr, bandw = ost_deal_message(ost_message, ost_time, start_time,
end_time)
for j in range(length):
for i in range(440):
fwd_bandr[j] += bandr[i][j]
fwd_bandw[j] += bandw[i][j]
return ostlist, fwd_bandr, fwd_bandw
def compute_pre_with_jobid(jobid):
# print time11, time12, jobid
UTC = datetime.timedelta(hours=8)
resu = []
resu = get_re_jobid(jobid)
for val in resu:
ti = val[0] + " " + val[1] + " " + val[2]
program_name = val[2]
CNC = int(val[5])
run_time = int(val[6])
corehour = float(val[7])
time21 = val[3]
time22 = val[4]
node = val[8]
t21 = datetime.datetime.strptime(time21, '%Y-%m-%d %H:%M:%S')
t22 = datetime.datetime.strptime(time22, '%Y-%m-%d %H:%M:%S')
min_time = time_to_sec_fast(time21)
max_time = time_to_sec_fast(time22)
tt1 = str(t21 - UTC)
tt2 = str(t22 - UTC)
time1 = tt1[:10] + "T" + tt1[11:] + ".000Z"
time2 = tt2[:10] + "T" + tt2[11:] + ".000Z"
iplist = []
for no in node:
a = no.split('-')
try:
int(a[0])
except Exception:
print a[0]
print 'null node!!!'
return
if len(a) > 1:
# print a[0],a[1]
for x in range(int(a[0]), int(a[1]) + 1):
w2 = x // 1024
w3 = (x - w2 * 1024) // 8
w4 = x - w2 * 1024 - w3 * 8 + 1
ip = "172." + str(w2) + "." + str(w3) + "." + str(w4)
iplist.append(ip)
elif len(a) == 1:
#print a[0]
w2 = int(a[0]) // 1024
w3 = (int(a[0]) - w2 * 1024) // 8
w4 = int(a[0]) - w2 * 1024 - w3 * 8 + 1
ip = "172." + str(w2) + "." + str(w3) + "." + str(w4)
iplist.append(ip)
return jobid, CNC, run_time, corehour, time1, time2, iplist, min_time, max_time
def fwd_deal_message(ost_message, ost_time, start_time, end_time):
length = time_to_sec_fast(end_time) - time_to_sec_fast(start_time)
fwd_bandr = [0 for i in range(length + 5)]
fwd_bandw = [0 for i in range(length + 5)]
bandr, bandw = ost_deal_message(ost_message, ost_time, start_time,
end_time)
for j in range(length):
for i in range(440):
# if(bandr[i][j] > 0 or bandw[i][j] > 0):
# print 'bandr[%d][%d]: %f bandw[%d][%d]: %f\n'\
# %(i, j, bandr[i][j], i, j, bandw[i][j])
fwd_bandr[j] += bandr[i][j]
fwd_bandw[j] += bandw[i][j]
return fwd_bandr, fwd_bandw
def compute_pre_with_jobid(jobid):
# print time11, time12, jobid
UTC = datetime.timedelta(hours=8)
resu = []
print jobid
if int(jobid) == 0:
resu = [[
'00000000', 'quanji', 'quanji', '2018-05-21 14:00:00',
'2018-05-21 15:00:00', 293L, 1L, 1L, ['0-40959']
]]
else:
resu = get_re_jobid_all(jobid)
print resu
for val in resu:
ti = val[0] + " " + val[1] + " " + val[2]
time21 = val[3]
time22 = val[4]
node = val[8]
t21 = datetime.datetime.strptime(time21, '%Y-%m-%d %H:%M:%S')
t22 = datetime.datetime.strptime(time22, '%Y-%m-%d %H:%M:%S')
min_time = time_to_sec_fast(time21)
max_time = time_to_sec_fast(time22)
tt1 = str(t21 - UTC)
tt2 = str(t22 - UTC)
time1 = tt1[:10] + "T" + tt1[11:] + ".000Z"
time2 = tt2[:10] + "T" + tt2[11:] + ".000Z"
iplist = []
fwd_list = dict()
for no in node:
a = no.split('-')
try:
int(a[0])
except Exception:
print a[0]
print 'null node!!!'
return
if len(a) > 1:
print a[0], a[1]
for x in range(int(a[0]), int(a[1]) + 1):
w2 = x // 1024
w3 = (x - w2 * 1024) // 8
w4 = x - w2 * 1024 - w3 * 8 + 1
ip = "172." + str(w2) + "." + str(w3) + "." + str(w4)
iplist.append(ip)
fwd_no = x // 512
if (fwd_no in fwd_list):
fwd_list[fwd_no].add(ip)
else:
fwd_list[fwd_no] = set([ip])
elif len(a) == 1:
#print a[0]
w2 = int(a[0]) // 1024
w3 = (int(a[0]) - w2 * 1024) // 8
w4 = int(a[0]) - w2 * 1024 - w3 * 8 + 1
ip = "172." + str(w2) + "." + str(w3) + "." + str(w4)
iplist.append(ip)
fwd_no = int(a[0]) // 512
if (fwd_no in fwd_list):
fwd_list[fwd_no].add(ip)
else:
fwd_list[fwd_no] = set([ip])
return time1, time2, iplist, min_time, max_time, fwd_list
def ost_deal_message(ost_message, ost_time, start_time, end_time):
ostlist = []
result = []
for i in range(len(ost_message)):
result.append(ost_message[i] + ' ' + ost_time[i])
result.sort()
length = time_to_sec_fast(end_time) - time_to_sec_fast(start_time)
#print length
bandr = [([0] * (length + 5)) for i in range(440)]
bandw = [([0] * (length + 5)) for i in range(440)]
t1 = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
print "Reshape and sort done..."
lg = len(result)
if lg < 1:
return ostlist, bandr, bandw
x = result[0].split(' ')
ost_p = x[0]
read_p = 0
write_p = 0
for i in range(len(x) / 3):
size = re.sub("\D", "", x[1 + 3 * i])
read_count = re.sub("\D", "", x[2 + 3 * i])
write_count = re.sub("\D", "", x[3 + 3 * i])
read_p += int(size) * int(read_count)
write_p += int(size) * int(write_count)
time_p = str(x[28])[:10] + " " + str(x[28])[11:-5]
t2 = datetime.datetime.strptime(time_p, '%Y-%m-%d %H:%M:%S')
time_interval = str(t2 - t1).split(':')
t_p = int(time_interval[0]) * 3600 + int(time_interval[1]) * 60 + int(
time_interval[2])
print "Deal data..."
read_count = 0
write_count = 0
for item in range(1, lg):
try:
y = result[item].split(' ')
ost = y[0]
read = 0
write = 0
for i in range(len(x) / 3):
size = re.sub("\D", "", y[1 + 3 * i])
read_count = re.sub("\D", "", y[2 + 3 * i])
write_count = re.sub("\D", "", y[3 + 3 * i])
read += int(size) * int(read_count)
write += int(size) * int(write_count)
time = str(y[28])[:10] + " " + str(y[28])[11:-5]
t3 = datetime.datetime.strptime(time, '%Y-%m-%d %H:%M:%S')
time_interval = str(t3 - t1).split(':')
t = int(time_interval[0]) * 3600 + int(
time_interval[1]) * 60 + int(time_interval[2])
if ost == ost_p:
interval = t - t_p
#print interval," ",t_p," ",band_number
OST_number = int('0x' + ost, 16)
read_value = int(read) - int(read_p)
write_value = int(write) - int(write_p)
if interval > 0 and interval < 1000:
for j in range(interval):
bandr[OST_number][
t_p + j] += read_value / 1024.0 * 4 / interval
bandw[OST_number][
t_p + j] += write_value / 1024.0 * 4 / interval
ost_p = ost
read_p = read
write_p = write
t_p = t
if OST_number not in ostlist:
ostlist.append(OST_number)
else:
x = result[item].split(' ')
ost_p = x[0]
read_p = 0
write_p = 0
for i in range(len(x) / 3):
size = re.sub("\D", "", x[1 + 3 * i])
read_count = re.sub("\D", "", x[2 + 3 * i])
write_count = re.sub("\D", "", x[3 + 3 * i])
read_p += int(size) * int(read_count)
write_p += int(size) * int(write_count)
time_p = str(x[28])[:10] + " " + str(x[28])[11:-5]
t2 = datetime.datetime.strptime(time_p, '%Y-%m-%d %H:%M:%S')
time_interval = str(t2 - t1).split(':')
t_p = int(time_interval[0]) * 3600 + int(
time_interval[1]) * 60 + int(time_interval[2])
except Exception as e:
print e, " size: ", size, " read_count: ", read_count, " write_count: ", write_count, " time_interval: ", time_interval
print y
continue
return ostlist, bandr, bandw
def compute_pre(starttime, endtime, jobid):
time11 = str(starttime)
time12 = str(endtime)
t11 = datetime.datetime.strptime(time11, '%Y-%m-%d %H:%M:%S')
t12 = datetime.datetime.strptime(time12, '%Y-%m-%d %H:%M:%S')
UTC = datetime.timedelta(hours=8)
resu = []
resu = get_re_jobid(jobid)
print_tag = 1
for val in resu:
ti = val[0] + ' ' + val[1] + ' ' + val[2]
time21 = val[3]
time22 = val[4]
if time22 == 'None':
time22 = time12
node = val[8]
time21 = time11
time22 = time12
t21 = datetime.datetime.strptime(time21, '%Y-%m-%d %H:%M:%S')
t22 = datetime.datetime.strptime(time22, '%Y-%m-%d %H:%M:%S')
if time21 <= time11:
time1 = time11
t1 = t11
else:
time1 = time21
t1 = t21
if time22 >= time12:
time2 = time12
t2 = t12
else:
time2 = time22
t2 = t22
min_time = time_to_sec_fast(time1)
max_time = time_to_sec_fast(time2)
tt1 = str(t1 - UTC)
tt2 = str(t2 - UTC)
time1 = tt1[:10] + 'T' + tt1[11:] + '.000Z'
time2 = tt2[:10] + 'T' + tt2[11:] + '.000Z'
iplist = []
for no in node:
a = no.split('-')
try:
int(a[0])
except Exception:
print a[0]
print 'null node!!!'
return
if len(a) > 1:
for x in range(int(a[0]), int(a[1]) + 1):
w2 = x // 1024
w3 = (x - w2 * 1024) // 8
w4 = x - w2 * 1024 - w3 * 8 + 1
ip = '172.' + str(w2) + '.' + str(w3) + '.' + str(w4)
iplist.append(ip)
elif len(a) == 1:
w2 = int(a[0]) // 1024
w3 = (int(a[0]) - w2 * 1024) // 8
w4 = int(a[0]) - w2 * 1024 - w3 * 8 + 1
ip = '172.' + str(w2) + '.' + str(w3) + '.' + str(w4)
iplist.append(ip)
return (time1, time2, iplist, min_time, max_time)
def deal_csv(results_message, results_host, min_time, max_time, time1, time2):
global compression_count
global no_count
resultr = np.array([0.0 for i in range(max_time - min_time + 5)])
resultw = np.array([0.0 for i in range(max_time - min_time + 5)])
result_open = np.array([0 for i in range(max_time - min_time + 5)])
result_close = np.array([0 for i in range(max_time - min_time + 5)])
resultr_ops = np.array([0 for i in range(max_time - min_time + 5)])
resultw_ops = np.array([0 for i in range(max_time - min_time + 5)])
resultr_size = []
resultw_size = []
dictr = []
dictw = []
compression_count = 0
no_count = 0
index_cc = 0
for i in xrange(len(results_message)):
x = results_message[i].split()
time = x[0] + ' ' + x[1]
c_time = time[1:-1]
ke = x[6]
kee = x[3]
if ke == 'OPEN':
try:
ctime = time_to_sec_fast(c_time)
index = ctime - min_time + 1
result_open[index] += 1
no_count += 1
compression_count += 1
except Exception as e:
c1 = 1
elif ke == 'RELEASE':
try:
ctime = time_to_sec_fast(c_time)
index = ctime - min_time + 1
result_close[index] += 1
no_count += 1
compression_count += 1
except Exception as e:
c1 = 1
elif kee == 'READ':
try:
if 'offset' not in results_message[i]:
offset_judge = 1
ip = results_host[i]
size = x[4].split('=')[1]
op_count = int(x[5].split('=')[1])
sumsize = int(size)
ctime = time_to_sec_fast(c_time)
index = ctime - min_time + 1
resultr[index] += sumsize / 1048576.0
resultr_ops[index] += op_count
resultr_size.append([sumsize / op_count * 1.0, op_count])
else:
offset_judge = 2
ip = results_host[i]
size = int(x[5].split('=')[1])
op_count = int(x[7].split('=')[1])
sumsize = int(size) * op_count
ctime = time_to_sec_fast(c_time)
index = ctime - min_time + 1
resultr[index] += sumsize / 1048576.0
resultr_ops[index] += op_count
resultr_size.append([size, op_count])
dictr.append([ip, index, sumsize, op_count])
no_count += op_count
compression_count += 1
except Exception as e:
c1 = 1
elif kee == 'WRITE':
try:
if 'offset' not in results_message[i]:
offset_judge = 1
ip = results_host[i]
size = x[4].split('=')[1]
op_count = int(x[5].split('=')[1])
sumsize = int(size)
ctime = time_to_sec_fast(c_time)
index = ctime - min_time + 1
resultw[index] += sumsize / 1048576.0
resultw_ops[index] += op_count
resultw_size.append([sumsize / op_count * 1.0, op_count])
else:
offset_judge = 2
ip = results_host[i]
size = int(x[5].split('=')[1])
op_count = int(x[7].split('=')[1])
sumsize = int(size) * op_count
ctime = time_to_sec_fast(c_time)
index = ctime - min_time + 1
resultw[index] += sumsize / 1048576.0
resultw_ops[index] += op_count
resultw_size.append([size, op_count])
dictw.append([ip, index, sumsize, op_count])
no_count += op_count
compression_count += 1
except Exception as e:
c1 = 1
return (resultr, resultw, resultr_ops, resultw_ops, result_open,
result_close, resultr_size, resultw_size, dictr, dictw)
def compute_pre_with_jobid(jobid):
fwd_map = [
81, 82, 105, 85, 87, 88, 89, 90, 93, 136, 97, 99, 101, 103, 91, 92,
130, 131, 140, 133, 134, 135, 96, 63, 100, 102, 128, 129, 104, 137, 56,
86, 83, 95, 30, 127, 141, 142, 143, 144, 107, 108, 109, 110, 111, 112,
113, 55, 121, 116, 117, 118, 119, 120, 122, 94, 124, 125, 138, 139, 23,
24, 25, 26, 27, 28, 29, 44, 31, 45, 33, 34, 35, 36, 37, 38, 39, 40, 41,
42, 126, 20, 43
]
UTC = datetime.timedelta(hours=8)
resu = []
resu = get_re_jobid_all(jobid)
for val in resu:
ti = val[0] + " " + val[1] + " " + val[2]
time21 = val[3]
time22 = val[4]
node = val[8]
t21 = datetime.datetime.strptime(time21, '%Y-%m-%d %H:%M:%S')
t22 = datetime.datetime.strptime(time22, '%Y-%m-%d %H:%M:%S')
min_time = time_to_sec_fast(time21)
max_time = time_to_sec_fast(time22)
tt1 = str(t21 - UTC)
tt2 = str(t22 - UTC)
time1 = tt1[:10] + "T" + tt1[11:] + ".000Z"
time2 = tt2[:10] + "T" + tt2[11:] + ".000Z"
iplist = []
fwd_list = dict()
for no in node:
a = no.split('-')
try:
int(a[0])
except Exception:
print a[0]
print 'null node!!!'
return
if len(a) > 1:
print a[0], a[1]
for x in range(int(a[0]), int(a[1]) + 1):
w2 = x // 1024
w3 = (x - w2 * 1024) // 8
w4 = x - w2 * 1024 - w3 * 8 + 1
ip = "172." + str(w2) + "." + str(w3) + "." + str(w4)
iplist.append(ip)
fwd_no = x // 512
if (fwd_map[fwd_no] in fwd_list):
fwd_list[fwd_map[fwd_no]].add(ip)
else:
fwd_list[fwd_map[fwd_no]] = set([ip])
elif len(a) == 1:
#print a[0]
w2 = int(a[0]) // 1024
w3 = (int(a[0]) - w2 * 1024) // 8
w4 = int(a[0]) - w2 * 1024 - w3 * 8 + 1
ip = "172." + str(w2) + "." + str(w3) + "." + str(w4)
iplist.append(ip)
fwd_no = int(a[0]) // 512
if (fwd_map[fwd_no] in fwd_list):
fwd_list[fwd_map[fwd_no]].add(ip)
else:
fwd_list[fwd_map[fwd_no]] = set([ip])
return time1, time2, iplist, min_time, max_time, fwd_list
def compute_pre_with_jobid(jobid):
UTC = datetime.timedelta(hours=8)
resu = []
resu = get_re_jobid_all(jobid)
print resu
for val in resu:
ti = val[0] + " " + val[1] + " " + val[2]
time21 = val[3]
time22 = val[4]
if s_time != 'NULL':
time21 = s_time
if e_time != 'NULL':
time22 = e_time
node = val[8]
iplist = []
fwd_list = dict()
for no in node:
a = no.split('-')
try:
int(a[0])
except Exception:
print a[0]
print 'null node!!!'
return
if len(a) > 1:
print a[0], a[1]
for x in range(int(a[0]), int(a[1]) + 1):
w2 = x // 1024
w3 = (x - w2 * 1024) // 8
w4 = x - w2 * 1024 - w3 * 8 + 1
ip = "172." + str(w2) + "." + str(w3) + "." + str(w4)
iplist.append(ip)
fwd_no = x // 512
if (fwd_no in fwd_list):
fwd_list[fwd_no].add(ip)
else:
fwd_list[fwd_no] = set([ip])
elif len(a) == 1:
#print a[0]
w2 = int(a[0]) // 1024
w3 = (int(a[0]) - w2 * 1024) // 8
w4 = int(a[0]) - w2 * 1024 - w3 * 8 + 1
ip = "172." + str(w2) + "." + str(w3) + "." + str(w4)
iplist.append(ip)
fwd_no = int(a[0]) // 512
if (fwd_no in fwd_list):
fwd_list[fwd_no].add(ip)
else:
fwd_list[fwd_no] = set([ip])
try:
t21 = datetime.datetime.strptime(time21, '%Y-%m-%d %H:%M:%S')
t22 = datetime.datetime.strptime(time22, '%Y-%m-%d %H:%M:%S')
except Exception as e:
print e
print 'Time format is incorrect. \
Job information is not updated in job database.'
sys.exit()
min_time = time_to_sec_fast(time21)
max_time = time_to_sec_fast(time22)
tt1 = str(t21 - UTC)
tt2 = str(t22 - UTC)
time1 = tt1[:10] + "T" + tt1[11:] + ".000Z"
time2 = tt2[:10] + "T" + tt2[11:] + ".000Z"
return time1, time2, iplist, min_time, max_time, fwd_list
