def output_statistic_CSV(self,data,CSV_path): """Csv files Output.""" if not data: return parse_type = utils.get_parse_type(os.path.basename(CSV_path)) print("") print(os.path.basename(CSV_path),parse_type) logging.debug("+ %s" % os.path.basename(CSV_path)) logging.debug(" %s" % parse_type) read_datas = [] write_datas = [] ioD_datas = [] csvfile = file(CSV_path,'wb') writer = csv.writer(csvfile) for block_name, blockData_dic in data.items(): if 'read_block_str' == block_name: writer.writerow(myconf.FIELDS_READ) for name,data in blockData_dic.items(): line = [] line.append(name) line.extend(data) read_datas.append(tuple(line)) writer.writerows(read_datas) writer.writerow("") if 'write_block_str' == block_name: writer.writerow(myconf.FIELDS_WRITE) for name,data in blockData_dic.items(): line = [] line.append(name) line.extend(data) write_datas.append(tuple(line)) writer.writerows(write_datas) writer.writerow("") if 'ioDistrbution_block_str' == block_name: writer.writerow(myconf.FIELDS_ioD) for name,data in blockData_dic.items(): line = [] line.append(name) line.extend(data) ioD_datas.append(tuple(line)) writer.writerows(ioD_datas) writer.writerow("") if None == parse_type: print("@csv generated with warn@ " + CSV_path) logging.debug("@csv generated with warn@ %s" % CSV_path) else: print("@csv generated successfully@ " + CSV_path) logging.debug("@csv generated successfully@ %s" % CSV_path) csvfile.close()
def output_batch_CSV(self,logFile_list,CSV_path): """Csv files Output.""" parse_type = utils.get_parse_type(os.path.basename(CSV_path)) print(os.path.basename(CSV_path),parse_type) logging.debug("+ %s" % os.path.basename(CSV_path)) logging.debug(" %s" % parse_type) read_datas = [] write_datas = [] lat_datas = [] ioD_datas = [] for logObj in logFile_list: print(" "+logObj.fileName) logging.debug(" - %s" % logObj.fileName) if myconf.PARSE_TYPE["mix"] == parse_type: read_datas.append(self.get_readField_data(logObj)) write_datas.append(self.get_writeField_data(logObj)) if myconf.PARSE_TYPE["pure_r"] == parse_type: read_datas.append(self.get_readField_data(logObj)) write_datas = [""] if myconf.PARSE_TYPE["pure_w"] == parse_type: write_datas.append(self.get_writeField_data(logObj)) read_datas = [""] lat_datas.append(self.get_latField_data(logObj,myconf.FIELDS_LAT)) ioD_datas.append(self.get_ioDField_data(logObj,myconf.FIELDS_ioD)) csvfile = file(CSV_path,'wb') writer = csv.writer(csvfile) writer.writerow(myconf.FIELDS_READ) writer.writerows(read_datas) writer.writerow("") writer.writerow(myconf.FIELDS_WRITE) writer.writerows(write_datas) writer.writerow("") writer.writerow(myconf.FIELDS_LAT) writer.writerows(lat_datas) writer.writerow("") writer.writerow(myconf.FIELDS_ioD) writer.writerows(ioD_datas) if None == parse_type: print("@csv generated with warn@ " + CSV_path) logging.debug("@csv generated with warn@ %s" % CSV_path) else: print("@csv generated successfully@ " + CSV_path) logging.debug("@csv generated successfully@ %s" % CSV_path) csvfile.close()
def get_statistic(self,file_path): """Get statistic from the csv file_path given. """ if self.prohibition == True: print("Sorry ,the power is illegal !") return raw_lines = utils.cut_file(file_path) lines = [] for l in raw_lines: if '\n' != l and '' != l and None != l: lines.append(l.replace("\r","")) parse_type = utils.get_parse_type(file_path) blocks_str = self.split_file(lines,parse_type) file_stat = {} for block_name, blockData_list in blocks_str.items(): if None == blockData_list: continue dic = {} num = {} for item in blockData_list: line_list = item.split(",") match = re.search("(.*)/vm", line_list[0]) if match: group = match.group(1) num[group] = 0; dic[group] = [0.0]*(len(line_list)-1) for item in blockData_list: line_list = item.split(",") match = re.search("(.*)/vm", line_list[0]) if match: group = match.group(1) num[group] += 1; for i in range(1,len(line_list)): dic[group][i-1] += round(string.atof(line_list[i]), 2) if 'read_block_str' == block_name or 'write_block_str' == block_name: for group_name ,group_data_list in dic.items(): group_data_list[2] = round(group_data_list[2] / num[group_name],2) group_data_list[3] = round(group_data_list[3] / num[group_name],2) group_data_list[4] = round(group_data_list[4] / num[group_name],2) #print ("######",round(group_data_list[4] / num[group_name],2)) file_stat[block_name] = dic return file_stat
def log_file_maker(self,file_path): """Most important parsing fio-log logic. 1.cuting lines 2.parsing string into several string blocks 3.puting string blocks into a dic blocks 4.generating LogFile object with its arributes 5.return LogFile object """ lines = utils.cut_file(file_path) blocks_str = self.split_report(lines) blocks = self.get_blocks(blocks_str) parse_type = utils.get_parse_type(file_path) log_file = LogFile(file_path,parse_type,blocks["read"],blocks["write"],blocks["latPercent"],blocks["ioDistribution"]) return log_file
def getTimeAxis(self,dataLogFile_list): parserT = utils.get_parse_type(dataLogFile_list[0].get_fileName()) if myconf.PARSE_TYPE["mix"] == parserT or myconf.PARSE_TYPE["pure_r"] == parserT: timeArea = self.getNzeroLines(dataLogFile_list[0].get_readBlock()) for dataLogObj in dataLogFile_list: listlen = self.getNzeroLines(dataLogObj.get_readBlock()) if listlen < timeArea: timeArea = listlen return timeArea elif myconf.PARSE_TYPE["pure_w"] == parserT: timeArea = self.getNzeroLines(dataLogFile_list[0].get_writeBlock()) for dataLogObj in dataLogFile_list: listlen = self.getNzeroLines(dataLogObj.get_writeBlock()) if listlen < timeArea: timeArea = listlen return timeArea
def output_batch_data_CSV(self,dataLogFile_list,CSV_path): """Csv files Output.""" timeNum = self.getTimeAxis(dataLogFile_list) parserType = utils.get_parse_type(CSV_path) colNum = len(dataLogFile_list) dataLines = [] fields = ["time"] statisticsMin = ["Min"] statisticsMax = ["Max"] statisticsAvg = ["Avg"] statisticsVar = ["Var"] if myconf.PARSE_TYPE["mix"] == parserType: for i in range(1, timeNum): line = [] line.append(i*5000) for obj in dataLogFile_list: name = obj.get_fileName() if 0 == fields.count(os.path.basename(os.path.dirname(name)) + "-read"): fields.append(os.path.basename(os.path.dirname(name)) + "-read") if 0 == fields.count(os.path.basename(os.path.dirname(name)) + "-write"): fields.append(os.path.basename(os.path.dirname(name)) + "-write") line.append(obj.get_readBlock()[i]) line.append(obj.get_writeBlock()[i]) dataLines.append(line) for obj in dataLogFile_list: obj.init_statics(timeNum) statisticsMin.append(obj.get_min_r()) statisticsMax.append(obj.get_max_r()) statisticsAvg.append(obj.get_avg_r()) statisticsVar.append(obj.get_var_r()) statisticsMin.append(obj.get_min_w()) statisticsMax.append(obj.get_max_w()) statisticsAvg.append(obj.get_avg_w()) statisticsVar.append(obj.get_var_w()) if myconf.PARSE_TYPE["pure_r"] == parserType: for i in range(1, timeNum): line = [] line.append(i*5000) for obj in dataLogFile_list: name = obj.get_fileName() if 0 == fields.count(os.path.basename(os.path.dirname(name)) + "-read"): fields.append(os.path.basename(os.path.dirname(name)) + "-read") line.append(obj.get_readBlock()[i]) dataLines.append(line) for obj in dataLogFile_list: obj.init_statics(timeNum) statisticsMin.append(obj.get_min_r()) statisticsMax.append(obj.get_max_r()) statisticsAvg.append(obj.get_avg_r()) statisticsVar.append(obj.get_var_r()) if myconf.PARSE_TYPE["pure_w"] == parserType: for i in range(1, timeNum): line = [] line.append(i*5000) for obj in dataLogFile_list: name = obj.get_fileName() if 0 == fields.count(os.path.basename(os.path.dirname(name)) + "-write"): fields.append(os.path.basename(os.path.dirname(name)) + "-write") line.append(obj.get_writeBlock()[i]) dataLines.append(line) for obj in dataLogFile_list: obj.init_statics(timeNum) statisticsMin.append(obj.get_min_w()) statisticsMax.append(obj.get_max_w()) statisticsAvg.append(obj.get_avg_w()) statisticsVar.append(obj.get_var_w()) #print(os.path.basename(CSV_path),parserType) logging.debug("+ %s" % os.path.basename(CSV_path)) logging.debug(" %s" % parserType) csvfile = file(CSV_path,'wb') writer = csv.writer(csvfile) #print(fields) writer.writerow(fields) for line in dataLines: writer.writerow(line) writer.writerow(statisticsMin) writer.writerow(statisticsMax) writer.writerow(statisticsAvg) writer.writerow(statisticsVar) if None == parserType: print("@csv generated with warn@ " + CSV_path) logging.debug("@csv generated with warn@ %s" % CSV_path) else: print("@csv generated successfully@ " + CSV_path) logging.debug("@csv generated successfully@ %s" % CSV_path) csvfile.close()
def data_log_file_maker(self,file_path): """Most important parsing data-log logic. 1.cuting lines 2.tracing the lines in the log 3.generating the readBlock and writeBlock 4.generating DataLogFile object with its arributes """ #print(file_path) parseType = utils.get_parse_type(file_path) lines = utils.cut_file(file_path) newlines = [] for l in lines: if '\n' != l and '' != l and None != l: newlines.append(l) #tailLine = '' #i = 0 #while '' == tailLine: # i -= 1 # tailLine = ''.join(lines[i].split()) #tailLineItems = tailLine.split(",") #num = int(round(int(tailLineItems[0])/5000.0)) + 1 num = len(newlines) + 1 readBlock = [0] * num writeBlock = [0] * num lastIndex_r = 1 lastIndex_w = 1 for line in lines: line = ''.join(line.split()) #print("###@ line:%s" % line) if '' == line: continue lineItems = line.split(",") index = int(round(int(lineItems[0])/5000.0)) #Ensure the current - last == 1, set the repair to fix repair = 500 if '0' == lineItems[2]: while index - lastIndex_r > 1: index = int(round((int(lineItems[0]) - repair)/5000.0)) repair += 500 lastIndex_r = index if 0 == readBlock[index]: readBlock[index] = int(lineItems[1]) if '1' == lineItems[2]: while index - lastIndex_w > 1: index = int(round((int(lineItems[0]) - repair)/5000.0)) repair += 500 lastIndex_w = index if 0 == writeBlock[index]: writeBlock[index] = int(lineItems[1]) #if '0' == lineItems[2]: # if 0 == readBlock[index]: # readBlock[index] = int(lineItems[1]) #elif '1' == lineItems[2]: # if 0 == writeBlock[index]: # writeBlock[index] = int(lineItems[1]) data_log_file = DataLogFile(file_path,parseType,readBlock,writeBlock) return data_log_file