def fwd_deal_message(ost_message, ost_time, start_time, end_time): length = time_to_sec_fast(end_time) - time_to_sec_fast(start_time) #print length fwd_bandr = [0 for i in range(length)] fwd_bandw = [0 for i in range(length)] ostlist, bandr, bandw = ost_deal_message(ost_message, ost_time, start_time, end_time) for j in range(length): for i in range(440): fwd_bandr[j] += bandr[i][j] fwd_bandw[j] += bandw[i][j] return ostlist, fwd_bandr, fwd_bandw
def compute_pre_with_jobid(jobid): # print time11, time12, jobid UTC = datetime.timedelta(hours=8) resu = [] resu = get_re_jobid(jobid) for val in resu: ti = val[0] + " " + val[1] + " " + val[2] program_name = val[2] CNC = int(val[5]) run_time = int(val[6]) corehour = float(val[7]) time21 = val[3] time22 = val[4] node = val[8] t21 = datetime.datetime.strptime(time21, '%Y-%m-%d %H:%M:%S') t22 = datetime.datetime.strptime(time22, '%Y-%m-%d %H:%M:%S') min_time = time_to_sec_fast(time21) max_time = time_to_sec_fast(time22) tt1 = str(t21 - UTC) tt2 = str(t22 - UTC) time1 = tt1[:10] + "T" + tt1[11:] + ".000Z" time2 = tt2[:10] + "T" + tt2[11:] + ".000Z" iplist = [] for no in node: a = no.split('-') try: int(a[0]) except Exception: print a[0] print 'null node!!!' return if len(a) > 1: # print a[0],a[1] for x in range(int(a[0]), int(a[1]) + 1): w2 = x // 1024 w3 = (x - w2 * 1024) // 8 w4 = x - w2 * 1024 - w3 * 8 + 1 ip = "172." + str(w2) + "." + str(w3) + "." + str(w4) iplist.append(ip) elif len(a) == 1: #print a[0] w2 = int(a[0]) // 1024 w3 = (int(a[0]) - w2 * 1024) // 8 w4 = int(a[0]) - w2 * 1024 - w3 * 8 + 1 ip = "172." + str(w2) + "." + str(w3) + "." + str(w4) iplist.append(ip) return jobid, CNC, run_time, corehour, time1, time2, iplist, min_time, max_time
def fwd_deal_message(ost_message, ost_time, start_time, end_time): length = time_to_sec_fast(end_time) - time_to_sec_fast(start_time) fwd_bandr = [0 for i in range(length + 5)] fwd_bandw = [0 for i in range(length + 5)] bandr, bandw = ost_deal_message(ost_message, ost_time, start_time, end_time) for j in range(length): for i in range(440): # if(bandr[i][j] > 0 or bandw[i][j] > 0): # print 'bandr[%d][%d]: %f bandw[%d][%d]: %f\n'\ # %(i, j, bandr[i][j], i, j, bandw[i][j]) fwd_bandr[j] += bandr[i][j] fwd_bandw[j] += bandw[i][j] return fwd_bandr, fwd_bandw
def compute_pre_with_jobid(jobid): # print time11, time12, jobid UTC = datetime.timedelta(hours=8) resu = [] print jobid if int(jobid) == 0: resu = [[ '00000000', 'quanji', 'quanji', '2018-05-21 14:00:00', '2018-05-21 15:00:00', 293L, 1L, 1L, ['0-40959'] ]] else: resu = get_re_jobid_all(jobid) print resu for val in resu: ti = val[0] + " " + val[1] + " " + val[2] time21 = val[3] time22 = val[4] node = val[8] t21 = datetime.datetime.strptime(time21, '%Y-%m-%d %H:%M:%S') t22 = datetime.datetime.strptime(time22, '%Y-%m-%d %H:%M:%S') min_time = time_to_sec_fast(time21) max_time = time_to_sec_fast(time22) tt1 = str(t21 - UTC) tt2 = str(t22 - UTC) time1 = tt1[:10] + "T" + tt1[11:] + ".000Z" time2 = tt2[:10] + "T" + tt2[11:] + ".000Z" iplist = [] fwd_list = dict() for no in node: a = no.split('-') try: int(a[0]) except Exception: print a[0] print 'null node!!!' return if len(a) > 1: print a[0], a[1] for x in range(int(a[0]), int(a[1]) + 1): w2 = x // 1024 w3 = (x - w2 * 1024) // 8 w4 = x - w2 * 1024 - w3 * 8 + 1 ip = "172." + str(w2) + "." + str(w3) + "." + str(w4) iplist.append(ip) fwd_no = x // 512 if (fwd_no in fwd_list): fwd_list[fwd_no].add(ip) else: fwd_list[fwd_no] = set([ip]) elif len(a) == 1: #print a[0] w2 = int(a[0]) // 1024 w3 = (int(a[0]) - w2 * 1024) // 8 w4 = int(a[0]) - w2 * 1024 - w3 * 8 + 1 ip = "172." + str(w2) + "." + str(w3) + "." + str(w4) iplist.append(ip) fwd_no = int(a[0]) // 512 if (fwd_no in fwd_list): fwd_list[fwd_no].add(ip) else: fwd_list[fwd_no] = set([ip]) return time1, time2, iplist, min_time, max_time, fwd_list
def ost_deal_message(ost_message, ost_time, start_time, end_time): ostlist = [] result = [] for i in range(len(ost_message)): result.append(ost_message[i] + ' ' + ost_time[i]) result.sort() length = time_to_sec_fast(end_time) - time_to_sec_fast(start_time) #print length bandr = [([0] * (length + 5)) for i in range(440)] bandw = [([0] * (length + 5)) for i in range(440)] t1 = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') print "Reshape and sort done..." lg = len(result) if lg < 1: return ostlist, bandr, bandw x = result[0].split(' ') ost_p = x[0] read_p = 0 write_p = 0 for i in range(len(x) / 3): size = re.sub("\D", "", x[1 + 3 * i]) read_count = re.sub("\D", "", x[2 + 3 * i]) write_count = re.sub("\D", "", x[3 + 3 * i]) read_p += int(size) * int(read_count) write_p += int(size) * int(write_count) time_p = str(x[28])[:10] + " " + str(x[28])[11:-5] t2 = datetime.datetime.strptime(time_p, '%Y-%m-%d %H:%M:%S') time_interval = str(t2 - t1).split(':') t_p = int(time_interval[0]) * 3600 + int(time_interval[1]) * 60 + int( time_interval[2]) print "Deal data..." read_count = 0 write_count = 0 for item in range(1, lg): try: y = result[item].split(' ') ost = y[0] read = 0 write = 0 for i in range(len(x) / 3): size = re.sub("\D", "", y[1 + 3 * i]) read_count = re.sub("\D", "", y[2 + 3 * i]) write_count = re.sub("\D", "", y[3 + 3 * i]) read += int(size) * int(read_count) write += int(size) * int(write_count) time = str(y[28])[:10] + " " + str(y[28])[11:-5] t3 = datetime.datetime.strptime(time, '%Y-%m-%d %H:%M:%S') time_interval = str(t3 - t1).split(':') t = int(time_interval[0]) * 3600 + int( time_interval[1]) * 60 + int(time_interval[2]) if ost == ost_p: interval = t - t_p #print interval," ",t_p," ",band_number OST_number = int('0x' + ost, 16) read_value = int(read) - int(read_p) write_value = int(write) - int(write_p) if interval > 0 and interval < 1000: for j in range(interval): bandr[OST_number][ t_p + j] += read_value / 1024.0 * 4 / interval bandw[OST_number][ t_p + j] += write_value / 1024.0 * 4 / interval ost_p = ost read_p = read write_p = write t_p = t if OST_number not in ostlist: ostlist.append(OST_number) else: x = result[item].split(' ') ost_p = x[0] read_p = 0 write_p = 0 for i in range(len(x) / 3): size = re.sub("\D", "", x[1 + 3 * i]) read_count = re.sub("\D", "", x[2 + 3 * i]) write_count = re.sub("\D", "", x[3 + 3 * i]) read_p += int(size) * int(read_count) write_p += int(size) * int(write_count) time_p = str(x[28])[:10] + " " + str(x[28])[11:-5] t2 = datetime.datetime.strptime(time_p, '%Y-%m-%d %H:%M:%S') time_interval = str(t2 - t1).split(':') t_p = int(time_interval[0]) * 3600 + int( time_interval[1]) * 60 + int(time_interval[2]) except Exception as e: print e, " size: ", size, " read_count: ", read_count, " write_count: ", write_count, " time_interval: ", time_interval print y continue return ostlist, bandr, bandw
def compute_pre(starttime, endtime, jobid): time11 = str(starttime) time12 = str(endtime) t11 = datetime.datetime.strptime(time11, '%Y-%m-%d %H:%M:%S') t12 = datetime.datetime.strptime(time12, '%Y-%m-%d %H:%M:%S') UTC = datetime.timedelta(hours=8) resu = [] resu = get_re_jobid(jobid) print_tag = 1 for val in resu: ti = val[0] + ' ' + val[1] + ' ' + val[2] time21 = val[3] time22 = val[4] if time22 == 'None': time22 = time12 node = val[8] time21 = time11 time22 = time12 t21 = datetime.datetime.strptime(time21, '%Y-%m-%d %H:%M:%S') t22 = datetime.datetime.strptime(time22, '%Y-%m-%d %H:%M:%S') if time21 <= time11: time1 = time11 t1 = t11 else: time1 = time21 t1 = t21 if time22 >= time12: time2 = time12 t2 = t12 else: time2 = time22 t2 = t22 min_time = time_to_sec_fast(time1) max_time = time_to_sec_fast(time2) tt1 = str(t1 - UTC) tt2 = str(t2 - UTC) time1 = tt1[:10] + 'T' + tt1[11:] + '.000Z' time2 = tt2[:10] + 'T' + tt2[11:] + '.000Z' iplist = [] for no in node: a = no.split('-') try: int(a[0]) except Exception: print a[0] print 'null node!!!' return if len(a) > 1: for x in range(int(a[0]), int(a[1]) + 1): w2 = x // 1024 w3 = (x - w2 * 1024) // 8 w4 = x - w2 * 1024 - w3 * 8 + 1 ip = '172.' + str(w2) + '.' + str(w3) + '.' + str(w4) iplist.append(ip) elif len(a) == 1: w2 = int(a[0]) // 1024 w3 = (int(a[0]) - w2 * 1024) // 8 w4 = int(a[0]) - w2 * 1024 - w3 * 8 + 1 ip = '172.' + str(w2) + '.' + str(w3) + '.' + str(w4) iplist.append(ip) return (time1, time2, iplist, min_time, max_time)
def deal_csv(results_message, results_host, min_time, max_time, time1, time2): global compression_count global no_count resultr = np.array([0.0 for i in range(max_time - min_time + 5)]) resultw = np.array([0.0 for i in range(max_time - min_time + 5)]) result_open = np.array([0 for i in range(max_time - min_time + 5)]) result_close = np.array([0 for i in range(max_time - min_time + 5)]) resultr_ops = np.array([0 for i in range(max_time - min_time + 5)]) resultw_ops = np.array([0 for i in range(max_time - min_time + 5)]) resultr_size = [] resultw_size = [] dictr = [] dictw = [] compression_count = 0 no_count = 0 index_cc = 0 for i in xrange(len(results_message)): x = results_message[i].split() time = x[0] + ' ' + x[1] c_time = time[1:-1] ke = x[6] kee = x[3] if ke == 'OPEN': try: ctime = time_to_sec_fast(c_time) index = ctime - min_time + 1 result_open[index] += 1 no_count += 1 compression_count += 1 except Exception as e: c1 = 1 elif ke == 'RELEASE': try: ctime = time_to_sec_fast(c_time) index = ctime - min_time + 1 result_close[index] += 1 no_count += 1 compression_count += 1 except Exception as e: c1 = 1 elif kee == 'READ': try: if 'offset' not in results_message[i]: offset_judge = 1 ip = results_host[i] size = x[4].split('=')[1] op_count = int(x[5].split('=')[1]) sumsize = int(size) ctime = time_to_sec_fast(c_time) index = ctime - min_time + 1 resultr[index] += sumsize / 1048576.0 resultr_ops[index] += op_count resultr_size.append([sumsize / op_count * 1.0, op_count]) else: offset_judge = 2 ip = results_host[i] size = int(x[5].split('=')[1]) op_count = int(x[7].split('=')[1]) sumsize = int(size) * op_count ctime = time_to_sec_fast(c_time) index = ctime - min_time + 1 resultr[index] += sumsize / 1048576.0 resultr_ops[index] += op_count resultr_size.append([size, op_count]) dictr.append([ip, index, sumsize, op_count]) no_count += op_count compression_count += 1 except Exception as e: c1 = 1 elif kee == 'WRITE': try: if 'offset' not in results_message[i]: offset_judge = 1 ip = results_host[i] size = x[4].split('=')[1] op_count = int(x[5].split('=')[1]) sumsize = int(size) ctime = time_to_sec_fast(c_time) index = ctime - min_time + 1 resultw[index] += sumsize / 1048576.0 resultw_ops[index] += op_count resultw_size.append([sumsize / op_count * 1.0, op_count]) else: offset_judge = 2 ip = results_host[i] size = int(x[5].split('=')[1]) op_count = int(x[7].split('=')[1]) sumsize = int(size) * op_count ctime = time_to_sec_fast(c_time) index = ctime - min_time + 1 resultw[index] += sumsize / 1048576.0 resultw_ops[index] += op_count resultw_size.append([size, op_count]) dictw.append([ip, index, sumsize, op_count]) no_count += op_count compression_count += 1 except Exception as e: c1 = 1 return (resultr, resultw, resultr_ops, resultw_ops, result_open, result_close, resultr_size, resultw_size, dictr, dictw)
def compute_pre_with_jobid(jobid): fwd_map = [ 81, 82, 105, 85, 87, 88, 89, 90, 93, 136, 97, 99, 101, 103, 91, 92, 130, 131, 140, 133, 134, 135, 96, 63, 100, 102, 128, 129, 104, 137, 56, 86, 83, 95, 30, 127, 141, 142, 143, 144, 107, 108, 109, 110, 111, 112, 113, 55, 121, 116, 117, 118, 119, 120, 122, 94, 124, 125, 138, 139, 23, 24, 25, 26, 27, 28, 29, 44, 31, 45, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 126, 20, 43 ] UTC = datetime.timedelta(hours=8) resu = [] resu = get_re_jobid_all(jobid) for val in resu: ti = val[0] + " " + val[1] + " " + val[2] time21 = val[3] time22 = val[4] node = val[8] t21 = datetime.datetime.strptime(time21, '%Y-%m-%d %H:%M:%S') t22 = datetime.datetime.strptime(time22, '%Y-%m-%d %H:%M:%S') min_time = time_to_sec_fast(time21) max_time = time_to_sec_fast(time22) tt1 = str(t21 - UTC) tt2 = str(t22 - UTC) time1 = tt1[:10] + "T" + tt1[11:] + ".000Z" time2 = tt2[:10] + "T" + tt2[11:] + ".000Z" iplist = [] fwd_list = dict() for no in node: a = no.split('-') try: int(a[0]) except Exception: print a[0] print 'null node!!!' return if len(a) > 1: print a[0], a[1] for x in range(int(a[0]), int(a[1]) + 1): w2 = x // 1024 w3 = (x - w2 * 1024) // 8 w4 = x - w2 * 1024 - w3 * 8 + 1 ip = "172." + str(w2) + "." + str(w3) + "." + str(w4) iplist.append(ip) fwd_no = x // 512 if (fwd_map[fwd_no] in fwd_list): fwd_list[fwd_map[fwd_no]].add(ip) else: fwd_list[fwd_map[fwd_no]] = set([ip]) elif len(a) == 1: #print a[0] w2 = int(a[0]) // 1024 w3 = (int(a[0]) - w2 * 1024) // 8 w4 = int(a[0]) - w2 * 1024 - w3 * 8 + 1 ip = "172." + str(w2) + "." + str(w3) + "." + str(w4) iplist.append(ip) fwd_no = int(a[0]) // 512 if (fwd_map[fwd_no] in fwd_list): fwd_list[fwd_map[fwd_no]].add(ip) else: fwd_list[fwd_map[fwd_no]] = set([ip]) return time1, time2, iplist, min_time, max_time, fwd_list
def compute_pre_with_jobid(jobid): UTC = datetime.timedelta(hours=8) resu = [] resu = get_re_jobid_all(jobid) print resu for val in resu: ti = val[0] + " " + val[1] + " " + val[2] time21 = val[3] time22 = val[4] if s_time != 'NULL': time21 = s_time if e_time != 'NULL': time22 = e_time node = val[8] iplist = [] fwd_list = dict() for no in node: a = no.split('-') try: int(a[0]) except Exception: print a[0] print 'null node!!!' return if len(a) > 1: print a[0], a[1] for x in range(int(a[0]), int(a[1]) + 1): w2 = x // 1024 w3 = (x - w2 * 1024) // 8 w4 = x - w2 * 1024 - w3 * 8 + 1 ip = "172." + str(w2) + "." + str(w3) + "." + str(w4) iplist.append(ip) fwd_no = x // 512 if (fwd_no in fwd_list): fwd_list[fwd_no].add(ip) else: fwd_list[fwd_no] = set([ip]) elif len(a) == 1: #print a[0] w2 = int(a[0]) // 1024 w3 = (int(a[0]) - w2 * 1024) // 8 w4 = int(a[0]) - w2 * 1024 - w3 * 8 + 1 ip = "172." + str(w2) + "." + str(w3) + "." + str(w4) iplist.append(ip) fwd_no = int(a[0]) // 512 if (fwd_no in fwd_list): fwd_list[fwd_no].add(ip) else: fwd_list[fwd_no] = set([ip]) try: t21 = datetime.datetime.strptime(time21, '%Y-%m-%d %H:%M:%S') t22 = datetime.datetime.strptime(time22, '%Y-%m-%d %H:%M:%S') except Exception as e: print e print 'Time format is incorrect. \ Job information is not updated in job database.' sys.exit() min_time = time_to_sec_fast(time21) max_time = time_to_sec_fast(time22) tt1 = str(t21 - UTC) tt2 = str(t22 - UTC) time1 = tt1[:10] + "T" + tt1[11:] + ".000Z" time2 = tt2[:10] + "T" + tt2[11:] + ".000Z" return time1, time2, iplist, min_time, max_time, fwd_list