def processMDF(fileName, channelist, resampleFlag, resampleValue, convertFlag, convertSelection): # Will process file according to defined options yop = mdfreader.Mdf() yop.multiProc = False # already multiprocessed yop.convertAfterRead = True yop.read(fileName) # reads complete file yop.keep_channels(channelist) # removes unnecessary channels if resampleFlag: yop.resample(resampleValue) if convertFlag: if convertSelection == 'Matlab': yop.export_to_matlab() elif convertSelection == 'csv': yop.export_to_csv() elif convertSelection == 'netcdf': yop.export_to_NetCDF() elif convertSelection == 'hdf5': yop.export_to_hdf5() elif convertSelection == 'excel': yop.export_to_excel() elif convertSelection == 'excel2010': yop.export_to_xlsx() elif convertSelection == 'mdf3': yop.write(fileName + '_new') yopPicklable = {} # picklable dict and not object for channel in list(yop.keys()): yopPicklable[channel] = yop[channel] return [yopPicklable, yop.masterChannelList, yop.fileName]
def __init__(self, parent=None): """ Constructor """ QMainWindow.__init__(self, parent) self.setupUi(self) self.fileNames = [] # files to convert self.mdfClass = mdfreader.Mdf() # instance of mdf self.mdfinfoClass = mdfreader.MdfInfo() # instance of mdfinfo self.convertSelection = 'Matlab' # by default Matlab conversion is selected self.MergeFiles = False # by default self.labFileName = [] # .lab file name self.defaultPath = None # default path to open for browsing files self.actionPlotSelectedChannel = QAction( "Plot", self.SelectedChannelList) # context menu to allow plot of channel self.SelectedChannelList.addAction(self.actionPlotSelectedChannel) self.actionPlotSelectedChannel.triggered.connect(self.plotSelected) self.actionPlotChannel = QAction( "Plot", self.channelList) # context menu to allow plot of channel self.channelList.addAction(self.actionPlotChannel) self.actionPlotChannel.triggered.connect(self.plot) self.actionFileRemove = QAction( "Delete", self.FileList) # context menu to remove selected file from list self.FileList.addAction(self.actionFileRemove) self.actionFileRemove.triggered.connect(self.FileRemove)
def dyn_dat(file, channel_list, raster): '''load the dat file and return in pandas dataframe''' df = pd.DataFrame(columns=channel_list) yop = mdfreader.Mdf(file) yop.resample(raster) for channel in channel_list: values = yop.get_channel_data(channel) df[channel] = values return df
def read_data(fileName): #counter = 0 # read data from an mdf file # and print value for every key raw_dict = mdfreader.Mdf(fileName) #for key in raw_dict.keys(): #print(key) #print(raw_dict[key]) #print(type(raw_dict[key]['data'])) #print(raw_dict[key]['data']) #counter+=1 #print(counter) return raw_dict
def findOcc(cycle, nan, zero): occ = {} for file in cycle: dic = mdfreader.Mdf(file) if nan: dic = dropAttNan(dic, 0.5) if zero: dic = dropAttZeros(dic, 0.99) keys = dic.keys() for key in keys: if key in occ.keys(): occ[key] += 1 else: occ[key] = 1 return occ
def read_mdf( self, event ): wildcard = 'INCA Files (*.dat)|*.dat' dlg0 = wx.FileDialog(None,message = 'Choose a MDF file', defaultDir = os.getcwd(), defaultFile="", wildcard = wildcard, style = wx.FD_OPEN) if dlg0.ShowModal() == wx.ID_OK: self.file_path = dlg0.GetPaths() else: return self.rawdata = mdfreader.Mdf(self.file_path[0]) key = list(self.rawdata.keys()) self.listBox_unselected.Clear() self.listBox_selected.Clear() for i in range(len(key)): self.listBox_unselected.Append(key[i])
def mdf_resample( self, event ): try: wb = load_workbook(self.config_path[0]) except AttributeError: dlg1 = wx.MessageDialog(None,message = 'Please read config file first',caption = 'Message') if dlg1.ShowModal() == wx.ID_OK: return try: ws = wb['variable'] #the required resample variables is in sheet named 'variable' except KeyError: dlg = wx.MessageDialog(None,message = 'Please read mdf config file',caption = 'Message') if dlg.ShowModal() == wx.ID_OK: return sample_rate = ws['A2'].value / 1000 #required resample rate # column_max = ws.max_column row_max = ws.max_row var_tuple = ws['B2':'B%d' % row_max] var_value = [] for i in range(len(var_tuple)): var_value.append(var_tuple[i][0].value) #delete None data in list var_value = [i for i in var_value if i != None] var_list = list(set(var_value)) # delete duplicated data,but change the order var_list.sort(key = var_value.index) # var_list is ordered by var_value var_len = len(var_list) # get the filepath wildcard = 'INCA Files (*.dat)|*.dat|All Files(*.*)|*.*' dlg0 = wx.FileDialog(None,message = 'Choose MDF files', defaultDir = os.getcwd(), defaultFile="", wildcard = wildcard, style = wx.FD_OPEN|wx.FD_MULTIPLE) if dlg0.ShowModal() == wx.ID_OK: file_path = dlg0.GetPaths() #include filepath and filename # file_fullname = dlg0.GetFilename() #only include filename else: return file_num = len(file_path) file_name = [] for i in range(file_num): num = file_path[i].rfind('\\') file_name.append(file_path[i][num + 1:-4]) #delete '.dat' def newlist(num): newlist = [] for i in range(num): newlist.append([]) return newlist # read MDF file rawdata = [] time_data = newlist(file_num) time_start = newlist(file_num) time_end = newlist(file_num) var_data = newlist(file_num) for i in range(file_num): rawdata.append(mdfreader.Mdf(file_path[i])) for i in range(file_num): for j in range(var_len): rawdata_varlist = list(rawdata[i].keys()) rawdata_varlist_len = len(rawdata_varlist) for k in range(rawdata_varlist_len): match_check = re.match(var_list[j],rawdata_varlist[k]) if match_check != None: match_end = match_check.end() if (rawdata_varlist[k][match_end:] in [':CAN1',':CCP1',':XCP1','']): time_data[i].append(rawdata[i][rawdata[i][rawdata_varlist[k]]['master']]['data']) var_data[i].append(rawdata[i][rawdata_varlist[k]]['data']) break if k == rawdata_varlist_len - 1: dlg = wx.MessageDialog(None,'File %s has no variable named \"%s\"' % (file_name[i],var_list[j]),'Error',wx.ICON_ERROR|wx.OK) if dlg.ShowModal() == wx.ID_OK: return for i in range(file_num): for j in range(var_len): time_start[i].append(time_data[i][j][0]) time_end[i].append(time_data[i][j][-1]) #data resample time_max = [] time_min = [] time_sample = [] for i in range(file_num): time_max.append(min(time_end[i])) time_min.append(max(time_start[i])) for i in range(file_num): time_sample.append([int(time_min[i]) + 1,int(time_max[i]) - 1]) inter = newlist(file_num) interval = [] time_resample = [] data_resample = newlist(file_num) for i in range(file_num): interval.append(int((time_sample[i][1] - time_sample[i][0])/sample_rate + 1)) #resample according to the required rate for i in range(file_num): time_resample.append(np.linspace(time_sample[i][0],time_sample[i][1],num = interval[i])) #Resampling target sequences for i in range(file_num): for j in range(var_len): inter[i].append(interp1d(time_data[i][j],var_data[i][j],kind='linear')) for i in range(file_num): for j in range(var_len): data_resample[i].append(inter[i][j](time_resample[i])) #writing excel file #dir_cur = os.getcwd() # dict_data = [{}] * file_num df = [] # python global variable dictionary access names = globals() for i in range(file_num): names['dict_data%d' % i] = {} # for i in range(file_num): # num = file_path[i].rfind('\\') # file_name.append(file_path[i][num + 1:-4]) #delete '.dat' for i in range(file_num): names['dict_data%d' % i]['time_resample'] = time_resample[i] for j in range(var_len): names['dict_data%d' % i][var_list[j]] = data_resample[i][j] for i in range(file_num): df.append(pd.DataFrame(names['dict_data%d' % i])) path1 = 'excel_resample\\' isExists = os.path.exists(path1) if not isExists: os.mkdir(path1) else: pass for i in range(file_num): try: wb_new = Workbook() wb_new.save('excel_resample\\%s.xlsx' % file_name[i]) writer = pd.ExcelWriter('excel_resample\\%s.xlsx' % file_name[i]) df[i].to_excel(writer,sheet_name = 'resampled data',startcol = 0,index=False) writer.save() except PermissionError: dlg = wx.MessageDialog(None,message = 'please close the excel output file before writing',caption = 'Message') if dlg.ShowModal() == wx.ID_OK: return #writing excel file dlg = wx.MessageDialog(None,message = 'Resample Complete',caption = 'Message') #dlg.ShowModal() : wx.ID_YES, wx.ID_NO, wx.ID_CANCEL, wx.ID_OK if dlg.ShowModal() == wx.ID_OK: return
# -*- coding: utf-8 -*- """ Created on Sun Jun 21 21:59:11 2020 @author: PHH2KOR """ import mdfreader import pandas as pd #Loading dat file yop = mdfreader.Mdf('EM59_Cold_MIDC_Veh6/23012020_Cold_ET_Veh6_EM59.dat' ) #This is the dat file yop.resample(0.1) #resampling the data to 0.1s channel_list = list(yop.keys()) df = pd.DataFrame( columns=channel_list) #pandas dataframe to store dat file content for channel in channel_list: values = yop.get_channel_data( channel) #to load entire dat file , an expensive operation df[channel] = values channel_list_filter = [ 'Epm_nEng', 'InjCtl_qSetUnBal', 'CEngDsT_t', 'InjCrv_qPiI1Des_mp', 'InjCrv_phiMI1Des', 'InjCrv_tiPiI1Des', 'RailP_pFlt', 'AFS_mAirPerCyl', 'EnvP_p' ] df_filter = pd.DataFrame(
def resample(): app = wx.App() global mes mes = 'Program Start' frame = MyWindow(None) frame.Show() path = os.getcwd() os.chdir(path) wb = load_workbook('Config_resample.xlsx') ws = wb.active sample_rate = ws['A2'].value / 1000 #需求的采样频率 # column_max = ws.max_column row_max = ws.max_row var_tuple = ws['B2':'B%d' % row_max] var_value = [] for i in range(len(var_tuple)): var_value.append(var_tuple[i][0].value) #去除列表中的None值 var_value = [i for i in var_value if i != None] var_len = len(var_value) #import and get required rawdata #root = tk.Tk() # 创建一个Tkinter.Tk()实例 #root.withdraw() # 将Tkinter.Tk()实例隐藏 #获取选取的文件路径 wildcard = 'INCA Files (*.dat)|*.dat|All Files(*.*)|*.*' dlg0 = wx.FileDialog(None,message = 'Choose a MDF file', defaultDir = os.getcwd(), defaultFile="", wildcard = wildcard, style = wx.FD_OPEN|wx.FD_MULTIPLE) if dlg0.ShowModal() == wx.ID_OK: file_path = dlg0.GetPaths() #包括路径和文件名 # file_fullname = dlg0.GetFilename() #只包括了文件名 else: time.sleep(1) wx.Exit() #退出wxPython sys.exit() file_num = len(file_path) def newlist(num): newlist = [] for i in range(num): newlist.append([]) return newlist #读取MDF文件 time.sleep(2) mes = 'Reading Mdf File...' frame.stbox.SetLabel(mes) rawdata = [] time_data = newlist(file_num) time_start = newlist(file_num) time_end = newlist(file_num) var_data = newlist(file_num) for i in range(file_num): rawdata.append(mdfreader.Mdf(file_path[i])) for i in range(file_num): for j in range(var_len): rawdata_varlist = list(rawdata[i].keys()) rawdata_varlist_len = len(rawdata_varlist) for k in range(rawdata_varlist_len): match_check = re.match(var_list[j], rawdata_varlist[k]) if match_check != None: match_end = match_check.end() if (rawdata_varlist[k][match_end:] in [':CAN1', ':CCP1', ':XCP1', '']): time_data[i].append(rawdata[i][rawdata[i][rawdata_varlist[k]]['master']]['data']) var_data[i].append(rawdata[i][rawdata_varlist[k]]['data']) break if k == rawdata_varlist_len - 1: dlg = wx.MessageDialog(None, 'File %s has no variable named \"%s\"' % (file_name[i], var_list[j]),'Error', wx.ICON_ERROR | wx.OK) if dlg.ShowModal() == wx.ID_OK: time.sleep(1) wx.Exit() sys.exit() #正常退出程序 for i in range(file_num): for j in range(var_len): time_start[i].append(time_data[i][j][0]) time_end[i].append(time_data[i][j][-1]) #数据重采样 time.sleep(2) mes = 'Resampling Data...' frame.stbox.SetLabel(mes) time_max = [] time_min = [] time_sample = [] for i in range(file_num): time_max.append(min(time_end[i])) time_min.append(max(time_start[i])) for i in range(file_num): time_sample.append([int(time_min[i]) + 1,int(time_max[i]) - 1]) inter = newlist(file_num) interval = [] time_resample = [] data_resample = newlist(file_num) for i in range(file_num): interval.append(int((time_sample[i][1] - time_sample[i][0])/sample_rate + 1)) #按要求频率重采样 for i in range(file_num): time_resample.append(np.linspace(time_sample[i][0],time_sample[i][1],num = interval[i])) #重采样目标时间序列 for i in range(file_num): for j in range(var_len): inter[i].append(interp1d(time_data[i][j],var_data[i][j],kind='linear')) for i in range(file_num): for j in range(var_len): data_resample[i].append(inter[i][j](time_resample[i])) #writing excel file time.sleep(2) mes = 'Writing Excel...' frame.stbox.SetLabel(mes) file_name = [] #dir_cur = os.getcwd() # dict_data = [{}] * file_num df = [] #python全局变量字典访问 names = globals() for i in range(file_num): names['dict_data%d' % i] = {} for i in range(file_num): num = file_path[i].rfind('\\') file_name.append(file_path[i][num + 1:-4]) #去掉.dat # for i in range(file_num): # ws_new.append(wb_new.create_sheet(file_name[i])) for i in range(file_num): names['dict_data%d' % i]['time_resample'] = time_resample[i] for j in range(var_len): names['dict_data%d' % i][var_value[j]] = data_resample[i][j] for i in range(file_num): df.append(pd.DataFrame(names['dict_data%d' % i])) path1 = 'excel\\' isExists = os.path.exists(path1) if not isExists: os.mkdir(path1) else: pass for i in range(file_num): wb_new = Workbook() wb_new.save('excel\\%s.xlsx' % file_name[i]) writer = pd.ExcelWriter('excel\\%s.xlsx' % file_name[i]) df[i].to_excel(writer,sheet_name = 'resampled data',startcol = 0,index=False) writer.save() #writing excel file time.sleep(2) mes = 'Program End' frame.stbox.SetLabel(mes) dlg = wx.MessageDialog(None,message = 'Resample Completed',caption = 'Message') #dlg.ShowModal() : wx.ID_YES, wx.ID_NO, wx.ID_CANCEL, wx.ID_OK if dlg.ShowModal() == wx.ID_OK: time.sleep(1) frame.Close() dlg.Destroy() app.MainLoop() del app
from __future__ import print_function from warnings import warn import sys import mdfreader warn(sys.argv[1:][0]) mdfreader.Mdf(sys.argv[1:][0])
""" Created on Wed May 20 17:42:05 2020 @author: Gaurav """ import mdfreader import pandas as pd #Loading dat file yop = mdfreader.Mdf('jan_hot.dat') #This is the dat file yop.resample(0.1) #resampling the data to 0.1s channel_list = list(yop.keys()) df = pd.DataFrame( columns=channel_list) #pandas dataframe to store dat file content for channel in channel_list: values = yop.get_channel_data( channel) #to load entire dat file , an expensive operation df[channel] = values channel_list_filter = [ 'Epm_nEng', 'InjCtl_qSetUnBal', 'CEngDsT_t', 'InjCrv_qPiI1Des_mp', 'InjCrv_phiMI1Des', 'InjCrv_tiPiI1Des', 'RailP_pFlt', 'AFS_mAirPerCyl', 'EnvP_p' ] df_filter = pd.DataFrame( columns=channel_list_filter) #pandas dataframe to store dat file content for channel in channel_list_filter:
'NSCRgn_tUsTPrim_mp', 'NSCRgn_tPrimMaxPrdcIntr_mp', 'ASMod_tEGFld_[17]', 'InjSys_facMI1CmbCor_mp', 'ExhMod_mfNoxEngOutp', 'PFltPOp_stRng_[1]', 'NSCRgn_stRlsLogic', 'SCRFFC_dmNH3Cnv', 'ETCTl_facPoI1Cor_mp', 'ASMod_tEGFld_[4]', 'InjVlv_tiET_[4]', 'Hegn_mFlowNoxB1S2', 'PFltPOp_stRng_[3]', 'Exh_tExhTMon1_mp', 'NSCRgn_tDeOSC_mp', 'NSCRgn_stRlsDSOxWd', 'InjVlv_tiET_[3]', 'NSCRgn_mSOx_mp', 'PFltPOp_stRng_[2]', 'Exh_tExhTMon3_mp', 'NSCRgn_tPrim_mp', 'ASMod_tEGFld_[5]', 'SCR_tUCatDsT', 'InjSys_facMI1CmbBas_mp', 'CoEng_st', 'NSCRgn_stRlsHtgWd_mp', 'EngPrt_trqNLim', 'NSCRgn_tScnd_mp', 'SCRFFC_dmNOxUs', 'Exh_tExhTMon6_mp', 'NSCRgn_stRlsDNOxWd_mp', 'Exh_tExhTMon4_mp', 'OxiCat_stStrtCondPas_mp', 'OxiCat_dmFlCmb_mp', 'ASMod_tEGFld_[16]', 'Hegn_mFlowNoxB1S1', 'SCR_tSensUCatUsT', 'NSCRgn_stDem', 'CALC_phi_Clash', 'CALC_Lambda', 'CALC_Trq_Derate', 'CALC_LNT_Exo', 'CALC_Delta_tTrb', 'CALC_Delta_pThr', '$ActiveCalibrationPage', 'ASMod_dmExhMnfDs', '$EVENT_COMMENTS' ] for i in [1]: dff = pd.DataFrame() for file in fileNames[i]: dic = mdfreader.Mdf(file) ndic = selectChannels(dic, conserve) target = 'ECUdata\\csvCycle' + str(i) + '\\' df = sampleData(ndic, pd.DataFrame(), 2) dff = pd.concat([dff, df]) filetab = file.split('\\') print(filetab[len(filetab) - 1] + 'concatened') #Let's write it dff.to_csv(target + 'ALL.csv') print('finished')
def traverse_channel(self): if WRITE_EXCEL: time_str = time.strftime("%m%d", time.localtime()) workbook = xlsxwriter.Workbook(PATH + '/{}_测试数据统计.xlsx'.format(time_str)) worksheet = workbook.add_worksheet('汇总表') ecu_channel = ECUChannel() i = 1 count=0 for xm1 in self.datas: count=count+1 data_len=len(self.datas) self.dat_name = os.path.splitext(os.path.basename(xm1))[0] self.dat_path = os.path.dirname(xm1) dat=os.path.basename(xm1) self.current_dat = xm1 yop = mdfreader.Mdf(xm1) self.channels = yop.keys() self.time_channel_deal(yop) print("一共{}个dat文件,正在解析第{}个dat文件。".format(data_len,count)) for channel in self.channels: ecu_channel.set_channel_name(channel) values = yop.get_channel_data(channel) size=sys.getsizeof(list(values)) lenx=len(values) ecu_channel.set_channel_values(values) unit = yop.get_channel_unit(channel) ecu_channel.set_channel_unit(unit) self.get_time_channel(ecu_channel) ecu_channel = self.clean_data(ecu_channel) if channel not in self.time_channel_info.keys(): # 画折线图 if ecu_channel.channel_values and DRAW_PIC: self.draw_picture(ecu_channel) # 数据写到excel表格 if WRITE_EXCEL: maxvalue = max(values) minvalue = min(values) try: meanx = np.mean(values) except TypeError: meanx = None count_dict = Counter(values) max_count = max(count_dict.values()) for key, value in count_dict.items(): if value == max_count: manyx = key manyxproportion = '{:.4%}'.format(value / len(values)) worksheet.write(0, 0, '文件路径') worksheet.write(0, 1, '通道名称') worksheet.write(0, 2, '最小值') worksheet.write(0, 3, '最大值') worksheet.write(0, 4, '平均数') worksheet.write(0, 5, '众数') worksheet.write(0, 6, '众数比例') # self.worksheet.write(0,7,"折线图") worksheet.write(i, 0, self.current_dat) worksheet.write(i, 1, channel) worksheet.write(i, 2, str(minvalue)) worksheet.write(i, 3, str(maxvalue)) worksheet.write(i, 4, str(meanx)) worksheet.write(i, 5, str(manyx)) worksheet.write(i, 6, str(manyxproportion)) # self.worksheet.insert_chart(i, 7, chart) i = i + 1 if WRITE_EXCEL: workbook.close()