def processMDF(fileName, channelist, resampleFlag, resampleValue, convertFlag,
convertSelection):
# Will process file according to defined options
yop = mdfreader.Mdf()
yop.multiProc = False # already multiprocessed
yop.convertAfterRead = True
yop.read(fileName) # reads complete file
yop.keep_channels(channelist) # removes unnecessary channels
if resampleFlag:
yop.resample(resampleValue)
if convertFlag:
if convertSelection == 'Matlab':
yop.export_to_matlab()
elif convertSelection == 'csv':
yop.export_to_csv()
elif convertSelection == 'netcdf':
yop.export_to_NetCDF()
elif convertSelection == 'hdf5':
yop.export_to_hdf5()
elif convertSelection == 'excel':
yop.export_to_excel()
elif convertSelection == 'excel2010':
yop.export_to_xlsx()
elif convertSelection == 'mdf3':
yop.write(fileName + '_new')
yopPicklable = {} # picklable dict and not object
for channel in list(yop.keys()):
yopPicklable[channel] = yop[channel]
return [yopPicklable, yop.masterChannelList, yop.fileName]
def __init__(self, parent=None):
"""
Constructor
"""
QMainWindow.__init__(self, parent)
self.setupUi(self)
self.fileNames = [] # files to convert
self.mdfClass = mdfreader.Mdf() # instance of mdf
self.mdfinfoClass = mdfreader.MdfInfo() # instance of mdfinfo
self.convertSelection = 'Matlab' # by default Matlab conversion is selected
self.MergeFiles = False # by default
self.labFileName = [] # .lab file name
self.defaultPath = None # default path to open for browsing files
self.actionPlotSelectedChannel = QAction(
"Plot",
self.SelectedChannelList) # context menu to allow plot of channel
self.SelectedChannelList.addAction(self.actionPlotSelectedChannel)
self.actionPlotSelectedChannel.triggered.connect(self.plotSelected)
self.actionPlotChannel = QAction(
"Plot", self.channelList) # context menu to allow plot of channel
self.channelList.addAction(self.actionPlotChannel)
self.actionPlotChannel.triggered.connect(self.plot)
self.actionFileRemove = QAction(
"Delete",
self.FileList) # context menu to remove selected file from list
self.FileList.addAction(self.actionFileRemove)
self.actionFileRemove.triggered.connect(self.FileRemove)
def dyn_dat(file, channel_list, raster):
'''load the dat file and return in pandas dataframe'''
df = pd.DataFrame(columns=channel_list)
yop = mdfreader.Mdf(file)
yop.resample(raster)
for channel in channel_list:
values = yop.get_channel_data(channel)
df[channel] = values
return df
def read_data(fileName):
#counter = 0
# read data from an mdf file
# and print value for every key
raw_dict = mdfreader.Mdf(fileName)
#for key in raw_dict.keys():
#print(key)
#print(raw_dict[key])
#print(type(raw_dict[key]['data']))
#print(raw_dict[key]['data'])
#counter+=1
#print(counter)
return raw_dict
def findOcc(cycle, nan, zero):
occ = {}
for file in cycle:
dic = mdfreader.Mdf(file)
if nan:
dic = dropAttNan(dic, 0.5)
if zero:
dic = dropAttZeros(dic, 0.99)
keys = dic.keys()
for key in keys:
if key in occ.keys():
occ[key] += 1
else:
occ[key] = 1
return occ
def read_mdf( self, event ):
wildcard = 'INCA Files (*.dat)|*.dat'
dlg0 = wx.FileDialog(None,message = 'Choose a MDF file',
defaultDir = os.getcwd(),
defaultFile="",
wildcard = wildcard,
style = wx.FD_OPEN)
if dlg0.ShowModal() == wx.ID_OK:
self.file_path = dlg0.GetPaths()
else:
return
self.rawdata = mdfreader.Mdf(self.file_path[0])
key = list(self.rawdata.keys())
self.listBox_unselected.Clear()
self.listBox_selected.Clear()
for i in range(len(key)):
self.listBox_unselected.Append(key[i])
def mdf_resample( self, event ):
try:
wb = load_workbook(self.config_path[0])
except AttributeError:
dlg1 = wx.MessageDialog(None,message = 'Please read config file first',caption = 'Message')
if dlg1.ShowModal() == wx.ID_OK:
return
try:
ws = wb['variable'] #the required resample variables is in sheet named 'variable'
except KeyError:
dlg = wx.MessageDialog(None,message = 'Please read mdf config file',caption = 'Message')
if dlg.ShowModal() == wx.ID_OK:
return
sample_rate = ws['A2'].value / 1000 #required resample rate
# column_max = ws.max_column
row_max = ws.max_row
var_tuple = ws['B2':'B%d' % row_max]
var_value = []
for i in range(len(var_tuple)):
var_value.append(var_tuple[i][0].value)
#delete None data in list
var_value = [i for i in var_value if i != None]
var_list = list(set(var_value)) # delete duplicated data,but change the order
var_list.sort(key = var_value.index) # var_list is ordered by var_value
var_len = len(var_list)
# get the filepath
wildcard = 'INCA Files (*.dat)|*.dat|All Files(*.*)|*.*'
dlg0 = wx.FileDialog(None,message = 'Choose MDF files',
defaultDir = os.getcwd(),
defaultFile="",
wildcard = wildcard,
style = wx.FD_OPEN|wx.FD_MULTIPLE)
if dlg0.ShowModal() == wx.ID_OK:
file_path = dlg0.GetPaths() #include filepath and filename
# file_fullname = dlg0.GetFilename() #only include filename
else:
return
file_num = len(file_path)
file_name = []
for i in range(file_num):
num = file_path[i].rfind('\\')
file_name.append(file_path[i][num + 1:-4]) #delete '.dat'
def newlist(num):
newlist = []
for i in range(num):
newlist.append([])
return newlist
# read MDF file
rawdata = []
time_data = newlist(file_num)
time_start = newlist(file_num)
time_end = newlist(file_num)
var_data = newlist(file_num)
for i in range(file_num):
rawdata.append(mdfreader.Mdf(file_path[i]))
for i in range(file_num):
for j in range(var_len):
rawdata_varlist = list(rawdata[i].keys())
rawdata_varlist_len = len(rawdata_varlist)
for k in range(rawdata_varlist_len):
match_check = re.match(var_list[j],rawdata_varlist[k])
if match_check != None:
match_end = match_check.end()
if (rawdata_varlist[k][match_end:] in [':CAN1',':CCP1',':XCP1','']):
time_data[i].append(rawdata[i][rawdata[i][rawdata_varlist[k]]['master']]['data'])
var_data[i].append(rawdata[i][rawdata_varlist[k]]['data'])
break
if k == rawdata_varlist_len - 1:
dlg = wx.MessageDialog(None,'File %s has no variable named \"%s\"' % (file_name[i],var_list[j]),'Error',wx.ICON_ERROR|wx.OK)
if dlg.ShowModal() == wx.ID_OK:
return
for i in range(file_num):
for j in range(var_len):
time_start[i].append(time_data[i][j][0])
time_end[i].append(time_data[i][j][-1])
#data resample
time_max = []
time_min = []
time_sample = []
for i in range(file_num):
time_max.append(min(time_end[i]))
time_min.append(max(time_start[i]))
for i in range(file_num):
time_sample.append([int(time_min[i]) + 1,int(time_max[i]) - 1])
inter = newlist(file_num)
interval = []
time_resample = []
data_resample = newlist(file_num)
for i in range(file_num):
interval.append(int((time_sample[i][1] - time_sample[i][0])/sample_rate + 1)) #resample according to the required rate
for i in range(file_num):
time_resample.append(np.linspace(time_sample[i][0],time_sample[i][1],num = interval[i])) #Resampling target sequences
for i in range(file_num):
for j in range(var_len):
inter[i].append(interp1d(time_data[i][j],var_data[i][j],kind='linear'))
for i in range(file_num):
for j in range(var_len):
data_resample[i].append(inter[i][j](time_resample[i]))
#writing excel file
#dir_cur = os.getcwd()
# dict_data = [{}] * file_num
df = []
# python global variable dictionary access
names = globals()
for i in range(file_num):
names['dict_data%d' % i] = {}
# for i in range(file_num):
# num = file_path[i].rfind('\\')
# file_name.append(file_path[i][num + 1:-4]) #delete '.dat'
for i in range(file_num):
names['dict_data%d' % i]['time_resample'] = time_resample[i]
for j in range(var_len):
names['dict_data%d' % i][var_list[j]] = data_resample[i][j]
for i in range(file_num):
df.append(pd.DataFrame(names['dict_data%d' % i]))
path1 = 'excel_resample\\'
isExists = os.path.exists(path1)
if not isExists:
os.mkdir(path1)
else:
pass
for i in range(file_num):
try:
wb_new = Workbook()
wb_new.save('excel_resample\\%s.xlsx' % file_name[i])
writer = pd.ExcelWriter('excel_resample\\%s.xlsx' % file_name[i])
df[i].to_excel(writer,sheet_name = 'resampled data',startcol = 0,index=False)
writer.save()
except PermissionError:
dlg = wx.MessageDialog(None,message = 'please close the excel output file before writing',caption = 'Message')
if dlg.ShowModal() == wx.ID_OK:
return
#writing excel file
dlg = wx.MessageDialog(None,message = 'Resample Complete',caption = 'Message')
#dlg.ShowModal() : wx.ID_YES, wx.ID_NO, wx.ID_CANCEL, wx.ID_OK
if dlg.ShowModal() == wx.ID_OK:
return
# -*- coding: utf-8 -*-
"""
Created on Sun Jun 21 21:59:11 2020
@author: PHH2KOR
"""
import mdfreader
import pandas as pd
#Loading dat file
yop = mdfreader.Mdf('EM59_Cold_MIDC_Veh6/23012020_Cold_ET_Veh6_EM59.dat'
) #This is the dat file
yop.resample(0.1) #resampling the data to 0.1s
channel_list = list(yop.keys())
df = pd.DataFrame(
columns=channel_list) #pandas dataframe to store dat file content
for channel in channel_list:
values = yop.get_channel_data(
channel) #to load entire dat file , an expensive operation
df[channel] = values
channel_list_filter = [
'Epm_nEng', 'InjCtl_qSetUnBal', 'CEngDsT_t', 'InjCrv_qPiI1Des_mp',
'InjCrv_phiMI1Des', 'InjCrv_tiPiI1Des', 'RailP_pFlt', 'AFS_mAirPerCyl',
'EnvP_p'
]
df_filter = pd.DataFrame(
def resample():
app = wx.App()
global mes
mes = 'Program Start'
frame = MyWindow(None)
frame.Show()
path = os.getcwd()
os.chdir(path)
wb = load_workbook('Config_resample.xlsx')
ws = wb.active
sample_rate = ws['A2'].value / 1000 #需求的采样频率
# column_max = ws.max_column
row_max = ws.max_row
var_tuple = ws['B2':'B%d' % row_max]
var_value = []
for i in range(len(var_tuple)):
var_value.append(var_tuple[i][0].value)
#去除列表中的None值
var_value = [i for i in var_value if i != None]
var_len = len(var_value)
#import and get required rawdata
#root = tk.Tk() # 创建一个Tkinter.Tk()实例
#root.withdraw() # 将Tkinter.Tk()实例隐藏
#获取选取的文件路径
wildcard = 'INCA Files (*.dat)|*.dat|All Files(*.*)|*.*'
dlg0 = wx.FileDialog(None,message = 'Choose a MDF file',
defaultDir = os.getcwd(),
defaultFile="",
wildcard = wildcard,
style = wx.FD_OPEN|wx.FD_MULTIPLE)
if dlg0.ShowModal() == wx.ID_OK:
file_path = dlg0.GetPaths() #包括路径和文件名
# file_fullname = dlg0.GetFilename() #只包括了文件名
else:
time.sleep(1)
wx.Exit() #退出wxPython
sys.exit()
file_num = len(file_path)
def newlist(num):
newlist = []
for i in range(num):
newlist.append([])
return newlist
#读取MDF文件
time.sleep(2)
mes = 'Reading Mdf File...'
frame.stbox.SetLabel(mes)
rawdata = []
time_data = newlist(file_num)
time_start = newlist(file_num)
time_end = newlist(file_num)
var_data = newlist(file_num)
for i in range(file_num):
rawdata.append(mdfreader.Mdf(file_path[i]))
for i in range(file_num):
for j in range(var_len):
rawdata_varlist = list(rawdata[i].keys())
rawdata_varlist_len = len(rawdata_varlist)
for k in range(rawdata_varlist_len):
match_check = re.match(var_list[j], rawdata_varlist[k])
if match_check != None:
match_end = match_check.end()
if (rawdata_varlist[k][match_end:] in [':CAN1', ':CCP1', ':XCP1', '']):
time_data[i].append(rawdata[i][rawdata[i][rawdata_varlist[k]]['master']]['data'])
var_data[i].append(rawdata[i][rawdata_varlist[k]]['data'])
break
if k == rawdata_varlist_len - 1:
dlg = wx.MessageDialog(None, 'File %s has no variable named \"%s\"' % (file_name[i], var_list[j]),'Error', wx.ICON_ERROR | wx.OK)
if dlg.ShowModal() == wx.ID_OK:
time.sleep(1)
wx.Exit()
sys.exit() #正常退出程序
for i in range(file_num):
for j in range(var_len):
time_start[i].append(time_data[i][j][0])
time_end[i].append(time_data[i][j][-1])
#数据重采样
time.sleep(2)
mes = 'Resampling Data...'
frame.stbox.SetLabel(mes)
time_max = []
time_min = []
time_sample = []
for i in range(file_num):
time_max.append(min(time_end[i]))
time_min.append(max(time_start[i]))
for i in range(file_num):
time_sample.append([int(time_min[i]) + 1,int(time_max[i]) - 1])
inter = newlist(file_num)
interval = []
time_resample = []
data_resample = newlist(file_num)
for i in range(file_num):
interval.append(int((time_sample[i][1] - time_sample[i][0])/sample_rate + 1)) #按要求频率重采样
for i in range(file_num):
time_resample.append(np.linspace(time_sample[i][0],time_sample[i][1],num = interval[i])) #重采样目标时间序列
for i in range(file_num):
for j in range(var_len):
inter[i].append(interp1d(time_data[i][j],var_data[i][j],kind='linear'))
for i in range(file_num):
for j in range(var_len):
data_resample[i].append(inter[i][j](time_resample[i]))
#writing excel file
time.sleep(2)
mes = 'Writing Excel...'
frame.stbox.SetLabel(mes)
file_name = []
#dir_cur = os.getcwd()
# dict_data = [{}] * file_num
df = []
#python全局变量字典访问
names = globals()
for i in range(file_num):
names['dict_data%d' % i] = {}
for i in range(file_num):
num = file_path[i].rfind('\\')
file_name.append(file_path[i][num + 1:-4]) #去掉.dat
# for i in range(file_num):
# ws_new.append(wb_new.create_sheet(file_name[i]))
for i in range(file_num):
names['dict_data%d' % i]['time_resample'] = time_resample[i]
for j in range(var_len):
names['dict_data%d' % i][var_value[j]] = data_resample[i][j]
for i in range(file_num):
df.append(pd.DataFrame(names['dict_data%d' % i]))
path1 = 'excel\\'
isExists = os.path.exists(path1)
if not isExists:
os.mkdir(path1)
else:
pass
for i in range(file_num):
wb_new = Workbook()
wb_new.save('excel\\%s.xlsx' % file_name[i])
writer = pd.ExcelWriter('excel\\%s.xlsx' % file_name[i])
df[i].to_excel(writer,sheet_name = 'resampled data',startcol = 0,index=False)
writer.save()
#writing excel file
time.sleep(2)
mes = 'Program End'
frame.stbox.SetLabel(mes)
dlg = wx.MessageDialog(None,message = 'Resample Completed',caption = 'Message')
#dlg.ShowModal() : wx.ID_YES, wx.ID_NO, wx.ID_CANCEL, wx.ID_OK
if dlg.ShowModal() == wx.ID_OK:
time.sleep(1)
frame.Close()
dlg.Destroy()
app.MainLoop()
del app
from __future__ import print_function from warnings import warn import sys import mdfreader warn(sys.argv[1:][0]) mdfreader.Mdf(sys.argv[1:][0])
"""
Created on Wed May 20 17:42:05 2020
@author: Gaurav
"""
import mdfreader
import pandas as pd
#Loading dat file
yop = mdfreader.Mdf('jan_hot.dat') #This is the dat file
yop.resample(0.1) #resampling the data to 0.1s
channel_list = list(yop.keys())
df = pd.DataFrame(
columns=channel_list) #pandas dataframe to store dat file content
for channel in channel_list:
values = yop.get_channel_data(
channel) #to load entire dat file , an expensive operation
df[channel] = values
channel_list_filter = [
'Epm_nEng', 'InjCtl_qSetUnBal', 'CEngDsT_t', 'InjCrv_qPiI1Des_mp',
'InjCrv_phiMI1Des', 'InjCrv_tiPiI1Des', 'RailP_pFlt', 'AFS_mAirPerCyl',
'EnvP_p'
]
df_filter = pd.DataFrame(
columns=channel_list_filter) #pandas dataframe to store dat file content
for channel in channel_list_filter:
'NSCRgn_tUsTPrim_mp', 'NSCRgn_tPrimMaxPrdcIntr_mp', 'ASMod_tEGFld_[17]',
'InjSys_facMI1CmbCor_mp', 'ExhMod_mfNoxEngOutp', 'PFltPOp_stRng_[1]',
'NSCRgn_stRlsLogic', 'SCRFFC_dmNH3Cnv', 'ETCTl_facPoI1Cor_mp',
'ASMod_tEGFld_[4]', 'InjVlv_tiET_[4]', 'Hegn_mFlowNoxB1S2',
'PFltPOp_stRng_[3]', 'Exh_tExhTMon1_mp', 'NSCRgn_tDeOSC_mp',
'NSCRgn_stRlsDSOxWd', 'InjVlv_tiET_[3]', 'NSCRgn_mSOx_mp',
'PFltPOp_stRng_[2]', 'Exh_tExhTMon3_mp', 'NSCRgn_tPrim_mp',
'ASMod_tEGFld_[5]', 'SCR_tUCatDsT', 'InjSys_facMI1CmbBas_mp', 'CoEng_st',
'NSCRgn_stRlsHtgWd_mp', 'EngPrt_trqNLim', 'NSCRgn_tScnd_mp',
'SCRFFC_dmNOxUs', 'Exh_tExhTMon6_mp', 'NSCRgn_stRlsDNOxWd_mp',
'Exh_tExhTMon4_mp', 'OxiCat_stStrtCondPas_mp', 'OxiCat_dmFlCmb_mp',
'ASMod_tEGFld_[16]', 'Hegn_mFlowNoxB1S1', 'SCR_tSensUCatUsT',
'NSCRgn_stDem', 'CALC_phi_Clash', 'CALC_Lambda', 'CALC_Trq_Derate',
'CALC_LNT_Exo', 'CALC_Delta_tTrb', 'CALC_Delta_pThr',
'$ActiveCalibrationPage', 'ASMod_dmExhMnfDs', '$EVENT_COMMENTS'
]
for i in [1]:
dff = pd.DataFrame()
for file in fileNames[i]:
dic = mdfreader.Mdf(file)
ndic = selectChannels(dic, conserve)
target = 'ECUdata\\csvCycle' + str(i) + '\\'
df = sampleData(ndic, pd.DataFrame(), 2)
dff = pd.concat([dff, df])
filetab = file.split('\\')
print(filetab[len(filetab) - 1] + 'concatened')
#Let's write it
dff.to_csv(target + 'ALL.csv')
print('finished')
def traverse_channel(self):
if WRITE_EXCEL:
time_str = time.strftime("%m%d", time.localtime())
workbook = xlsxwriter.Workbook(PATH + '/{}_测试数据统计.xlsx'.format(time_str))
worksheet = workbook.add_worksheet('汇总表')
ecu_channel = ECUChannel()
i = 1
count=0
for xm1 in self.datas:
count=count+1
data_len=len(self.datas)
self.dat_name = os.path.splitext(os.path.basename(xm1))[0]
self.dat_path = os.path.dirname(xm1)
dat=os.path.basename(xm1)
self.current_dat = xm1
yop = mdfreader.Mdf(xm1)
self.channels = yop.keys()
self.time_channel_deal(yop)
print("一共{}个dat文件，正在解析第{}个dat文件。".format(data_len,count))
for channel in self.channels:
ecu_channel.set_channel_name(channel)
values = yop.get_channel_data(channel)
size=sys.getsizeof(list(values))
lenx=len(values)
ecu_channel.set_channel_values(values)
unit = yop.get_channel_unit(channel)
ecu_channel.set_channel_unit(unit)
self.get_time_channel(ecu_channel)
ecu_channel = self.clean_data(ecu_channel)
if channel not in self.time_channel_info.keys():
# 画折线图
if ecu_channel.channel_values and DRAW_PIC:
self.draw_picture(ecu_channel)
# 数据写到excel表格
if WRITE_EXCEL:
maxvalue = max(values)
minvalue = min(values)
try:
meanx = np.mean(values)
except TypeError:
meanx = None
count_dict = Counter(values)
max_count = max(count_dict.values())
for key, value in count_dict.items():
if value == max_count:
manyx = key
manyxproportion = '{:.4%}'.format(value / len(values))
worksheet.write(0, 0, '文件路径')
worksheet.write(0, 1, '通道名称')
worksheet.write(0, 2, '最小值')
worksheet.write(0, 3, '最大值')
worksheet.write(0, 4, '平均数')
worksheet.write(0, 5, '众数')
worksheet.write(0, 6, '众数比例')
# self.worksheet.write(0,7,"折线图")
worksheet.write(i, 0, self.current_dat)
worksheet.write(i, 1, channel)
worksheet.write(i, 2, str(minvalue))
worksheet.write(i, 3, str(maxvalue))
worksheet.write(i, 4, str(meanx))
worksheet.write(i, 5, str(manyx))
worksheet.write(i, 6, str(manyxproportion))
# self.worksheet.insert_chart(i, 7, chart)
i = i + 1
if WRITE_EXCEL:
workbook.close()
