def main(): global tablet global normal_set global TP global FP global FN global label_y global predict_y file_dir = '/cbs_trace1/sample_6/' #load Cache MemoryCache = LRUCache(memory_size) SSDCache = LRUCache(cache_size) #load Classifier clf = Classifier() current_time = 1538323200 last = 0 for i in range(1, 31): ssd_hit_num = 0 memory_hit_num = 0 total_read = 0 total_write = 0 memory_write_num = 0 memory_write_hit = 0 ssd_write_num = 0 hdd_write_num = 0 predict_time = 0 predict_cnt = 0 data_x = [] data_y = [] if clf.load("%d.clf" % (last)): predict_flag = 1 else: predict_flag = 0 print('----------------------------------') cudate = time.strftime("%Y-%m-%d", time.localtime(current_time)) print(cudate) current_time += 86400 print('----------------------------------') file_name = file_dir + str(i) f = open(file_name, "r") for line in f.readlines(): io_record = line.split(",") #update tablet key = io_record[4][:-2] + ',' + str(int(io_record[1]) >> 11) if key in tablet: a = tablet[key] else: a = [0 for i in range(0, feature_num)] tablet[key] = a if int(io_record[3]) == 1: a[info_table['WriteSize']] += int(io_record[2]) a[info_table['WriteCnt']] += 1 if a[info_table['LastWrite']] != 0: a[info_table['WriteInterval']] += abs( int(io_record[0]) - a[info_table['LastWrite']]) a[info_table['LastWrite']] = int(io_record[0]) if int(io_record[2]) > 64: a[info_table['BigWrite']] += 1 elif int(io_record[2]) <= 8: a[info_table['SmallWrite']] += 1 else: a[info_table['ReadSize']] += int(io_record[2]) a[info_table['ReadCnt']] += 1 if a[info_table['LastRead']] != 0: a[info_table['ReadInterval']] += abs( int(io_record[0]) - a[info_table['LastRead']]) a[info_table['LastRead']] = int(io_record[0]) if int(io_record[2]) > 64: a[info_table['BigRead']] += 1 elif int(io_record[2]) <= 8: a[info_table['SmallRead']] += 1 #update Cache block = io_record[4][:-2] + ',' + str(int(io_record[1]) >> 6) if int(io_record[3]) == 1: #write total_write += 1 if MemoryCache.get(block) != None: memory_write_num += 1 memory_write_hit += 1 else: #reach condition to flush, flush first! flush_data = MemoryCache.flush() if flush_data != None: feature_tmp = [] flush_key = [] for key in flush_data: flush_key.append(key[0]) feature_tmp.append(process_eviction(key, 0)[0]) if predict_flag == 1: oldtime = datetime.now() rst = clf.predict(feature_tmp) newtime = datetime.now() predict_cnt += len(flush_key) predict_time += int( (newtime - oldtime).microseconds) for key, ft in zip(flush_key, rst): tablet_key = "%s,%d" % (key.split( ",")[0], int(key.split(",")[1]) >> 5) if key in SSDCache.cache: ssd_write_num += 1 else: if ft == 1: #only write hdd_write_num += 1 tablet[tablet_key][ info_table['Predict']] = 1 else: tablet[tablet_key][ info_table['Predict']] = 2 ssd_write_num += 1 ssd_evict = SSDCache.set(key, 1) if ssd_evict != None: hdd_write_num += 1 label = process_eviction( ssd_evict, 1) data_x.append(label[0]) data_y.append(label[1]) else: for key in flush_key: ssd_write_num += 1 ssd_evict = SSDCache.set(key, 1) if ssd_evict != None: hdd_write_num += 1 label = process_eviction(ssd_evict, 1) data_x.append(label[0]) data_y.append(label[1]) MemoryCache.set(block, 1) else: #read total_read += 1 if MemoryCache.get(block) != None: #hit memory_hit_num += 1 else: if SSDCache.get(block) != None: #hit ssd_hit_num += 1 a[info_table['Hit']] += 1 else: ssd_evict = SSDCache.set(block, 1) ssd_write_num += 1 if ssd_evict != None: hdd_write_num += 1 label = process_eviction(ssd_evict, 1) data_x.append(label[0]) data_y.append(label[1]) print("SSD footprint : %.2f %%" % (len(SSDCache.cache) * 100 / cache_size)) print("Memory footprint : %.2f %%" % (len(MemoryCache.cache) * 100 / memory_size)) print("memory_write : %.2f %%" % (memory_write_num * 100 / total_write)) print("memory_write_hit : %.2f %%" % (memory_write_hit * 100 / total_write)) print("ssd_write : %.2f %%" % ((ssd_write_num) * 100 / total_write)) print("hdd_write : %.2f %%" % (hdd_write_num * 100 / total_write)) print("Memory Hit Ratio : %.2f %%" % (memory_hit_num * 100 / total_read)) print("SSD Hit Ratio : %.2f %%" % ((memory_hit_num + ssd_hit_num) * 100 / total_read)) oldtime = datetime.now() #training model for next day last = i #save Classifier if len(data_x) > 0: clf = Classifier() clf.load("%d.clf" % (last)) clf.fit(data_x, data_y) clf.save("%d.clf" % (last)) newtime = datetime.now() train_time = int((newtime - oldtime).microseconds) #result file = 'result/hwp_result' f = open(file, 'a+') #time,memory_write,ssd_write,hdd_write,ssd_write_ratio,hdd_write_ratio,memory_hit,ssd_hit,accuracy,recall,predict_time,auc,train_time,recall2,positive_ratio if TP + FP == 0: f.write( "%d\t%d\t%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.4f\t%d\t%.2f\t%.2f\n" % (current_time - 86400, total_write, ssd_write_num, hdd_write_num, ssd_write_num * 100 / total_write, hdd_write_num * 100 / total_write, memory_hit_num * 100 / total_read, (memory_hit_num + ssd_hit_num) * 100 / total_read, 0, 0, 0, 0, train_time, 0, 0)) else: auc = metrics.roc_auc_score(label_y, predict_y) total = len(label_y) right = 0 pos = 0 t1 = 0 for l, p in zip(label_y, predict_y): if l == p: right += 1 if l == 1: pos += 1 if p == 1: t1 += 1 f.write( "%d\t%d\t%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.4f\t%d\t%.2f\t%.2f\n" % (current_time - 86400, total_write, ssd_write_num, hdd_write_num, ssd_write_num * 100 / total_write, hdd_write_num * 100 / total_write, memory_hit_num * 100 / total_read, (memory_hit_num + ssd_hit_num) * 100 / total_read, right * 100 / total, TP * 100 / (TP + FN), predict_time / predict_cnt, auc, train_time, t1 * 100 / pos, pos * 100 / total)) f.close()
def main(): file_dir = '/cbs_trace1/sample_6/' MemoryCache = LRUCache(memory_size) SSDCache = LRUCache(cache_size) current_time = 1538323200 for i in range(1,31): ssd_hit_num = 0 memory_hit_num = 0 total_read = 0 total_write = 0 memory_write_num = 0 ssd_write_num = 0 hdd_write_num = 0 print('----------------------------------') cudate = time.strftime("%Y-%m-%d", time.localtime(current_time)) print(cudate) current_time += 86400 print('----------------------------------') file_name = file_dir + str(i) f = open(file_name,"r") print(file_name) for line in f.readlines(): io_record = line.split(",") #update tablet key = io_record[4][:-2]+','+ str(int(io_record[1]) >> 11) if key in tablet: a = tablet[key] a[0] = a[0] + 1 else: a = [1 for i in range(0,feature_num)] tablet[key] = a block = io_record[4][:-2]+','+ str(int(io_record[1]) >> 6) if int(io_record[3]) == 1: #write total_write += 1 if MemoryCache.get(block) != None: memory_write_num += 1 else: #reach condition to flush, flush first! flush_data = MemoryCache.flush() if flush_data != None: for key in flush_data: ssd_write_num += 1 ssd_evict = SSDCache.set(key[0],1) if ssd_evict != None: hdd_write_num += 1 memory_write_num += 1 MemoryCache.set(block,1) else: #read total_read += 1 if MemoryCache.get(block) != None: #hit memory_hit_num += 1 else: if SSDCache.get(block) != None: #命中 ssd_hit_num += 1 else: evict = SSDCache.set(block,1) ssd_write_num += 1 if evict != None: hdd_write_num += 1 print("SSD footprint : %.2f %%"%(len(SSDCache.cache) * 100 / cache_size)) print("memory_write : %.2f %%"%(memory_write_num * 100 / total_write)) #need to write ssd log once when write to memory once print("ssd_write : %.2f %%"%((ssd_write_num+memory_write_num) * 100 / total_write)) print("hdd_write : %.2f %%"%(hdd_write_num * 100 / total_write)) print("Memory Hit Ratio : %.2f %%"%(memory_hit_num * 100 / total_read)) print("SSD Hit Ratio : %.2f %%"%( (memory_hit_num + ssd_hit_num) * 100 / total_read)) #result file = 'result/write_back' f = open(file, 'a+') #time,ssd_write,hdd_write,memory_hit,ssd_hit f.write("%d\t%d\t%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\n"%(current_time - 86400,total_write, ssd_write_num , hdd_write_num , (ssd_write_num) * 100 / total_write, hdd_write_num * 100 / total_write, memory_hit_num * 100 / total_read , (memory_hit_num + ssd_hit_num) * 100 / total_read)) f.close()