def GetDatafromMdf_asDF(filename, SigList, SampleTime=0.01, EncodeEnums=True):
## Local helper
def cleanEnumString(string):
return string.decode().split(r'\x00')[0].strip().strip('\x00')
## Function body
tempMDF = MDF()
sigs = []
with MDF(filename, remove_source_from_channel_names=True) as mdf:
for var in SigList:
try:
# get group and Index from channel_db
grp_idx = mdf.channels_db[var][0]
# Fetch signal as data object
sigs.append(mdf.get(group=grp_idx[0], index=grp_idx[1]))
# Append to mdf
except:
continue
tempMDF.append(sigs)
df = tempMDF.to_dataframe(raster=SampleTime)
types = df.apply(lambda x: pd.api.types.infer_dtype(x.values))
for col in types[types == 'bytes'].index: # String/Enum
df[col] = df[col].apply(cleanEnumString)
if (EncodeEnums == True):
types = df.apply(lambda x: pd.api.types.infer_dtype(x.values))
for col in types[types == 'string'].index: # String/Enum
df[col] = df[col].astype('category')
df[col] = df[col].cat.codes
return df
def to_hdf5(ts, id, mf4_dir, hdf_dir):
hdf_file = os.path.join(hdf_dir, '%s.hdf' % id[:5])
for t in ts:
file = os.path.join(mf4_dir, '%s_%s.mf4' % (t, id))
mdf_file = MDF(file)
data = mdf_file.to_dataframe()
data.index = (data.index * 1000000000 + float(t))
data.index = data.index.values.astype('datetime64[ns]')
data.index = pandas.to_datetime(data.index)
data.to_hdf(hdf_file, id.replace('-', '_'), append=True)
def test_to_dataframe(self):
dfs = [
DataFrame({
f'df_{i}_column_0': np.ones(5) * i,
f'df_{i}_column_1': np.arange(5) * i
}) for i in range(5)
]
mdf = MDF()
for df in dfs:
mdf.append(df)
target = {}
for i in range(5):
target[f'df_{i}_column_0'] = np.ones(5) * i
target[f'df_{i}_column_1'] = np.arange(5) * i
target = DataFrame(target)
self.assertTrue(target.equals(mdf.to_dataframe()))
measuredSignals = []
#remove signals with CPP in it
for signal in allSignals:
if not(("CCP" in signal) or ("_sword" in signal) or "$" in signal):
measuredSignals.append(signal)
#creating an empty list of important signals
impSignals = ['cps_n_engine', 'egr_b_operate_valve',
'egr_T_exhaust_temperature', 'egr_T_oil_temperature',
'egr_T_limiting_temp_low', 'egr_T_limiting_temp_high',
'egr_P_exhaustp']
df = mdf.to_dataframe(
channels=impSignals,
raster= 'egr_b_operate_valve',
time_from_zero=True,
empty_channels="zeros",
# keep_arrays=False,
use_display_names=True,
time_as_date=True,
reduce_memory_usage=True,
raw=False,
ignore_value2text_conversions=False)
#remove \CCP from column names of dataframe
df.columns = [col.split('\\')[0] for col in df.columns]
#resampling the data to 1 second
df = df.resample('S').mean()
n_i = list(range(1,len(df['egr_b_operate_valve'])+1))
df['Sl.no'] = n_i
df.set_index('Sl.no',inplace=True)
mdf.close()