def process_analog(filelist):
''' execute a series of operation on a dataframe of files defined by process_directory
Parameters:
----------
filelist - dlist of files to process
figoutput - name of file for saving image
return combined_df: dataframe with sensor signal
'''
first = True
for fullname in filelist:
print 'processing:', fullname
data_analog, start_date, stop_date = reformat.load_data(fullname)
Start_Time = time.mktime(start_date.timetuple())
Stop_Time = time.mktime(stop_date.timetuple())
diff = stop_date - start_date
time_diff_second = diff.seconds
len_data = len(data_analog)
analog_period = time_diff_second / np.float(len_data)
# analog dataframe
analog_df = pd.DataFrame(data_analog)
analog_df.columns = ['sensor_amp']
ts = np.arange(Start_Time, Stop_Time, analog_period)
analog_df['timestamp'] = ts[:len_data]
#normalization
analog_df['sensor_amp'] = analog_df['sensor_amp'] - np.mean(
analog_df['sensor_amp'])
array_length1 = len(analog_df['timestamp'])
Fs = int(np.float(len_data) /
time_diff_second) # can be calculated based on time stamp.
rng1 = pd.date_range(analog_df['timestamp'][0],
periods=array_length1,
freq=str(int(1000000 / Fs)) + 'u')
analog_df.index = rng1
if first:
combined_df = analog_df
first = False
else:
combined_df = pd.concat([combined_df, analog_df])
return combined_df, Fs
def process_analog(filelist):
''' execute a series of operation on a dataframe of files defined by process_directory
Parameters:
----------
filelist - dlist of files to process
figoutput - name of file for saving image
return combined_df: dataframe with sensor signal
'''
first = True
for fullname in filelist:
print 'processing:', fullname
data_analog, start_date, stop_date = reformat.load_data(fullname)
Start_Time = time.mktime(start_date.timetuple())
Stop_Time=time.mktime(stop_date.timetuple())
diff = stop_date-start_date
time_diff_second = diff.seconds
len_data= len(data_analog)
analog_period = time_diff_second/np.float(len_data)
# analog dataframe
analog_df = pd.DataFrame(data_analog)
analog_df.columns = ['sensor_amp']
ts = np.arange(Start_Time,Stop_Time,analog_period)
analog_df['timestamp'] = ts[:len_data]
#normalization
analog_df['sensor_amp'] = analog_df['sensor_amp'] - np.mean(analog_df['sensor_amp'])
array_length1 = len(analog_df['timestamp'])
Fs = int(np.float(len_data)/time_diff_second) # can be calculated based on time stamp.
rng1 = pd.date_range(analog_df['timestamp'][0], periods = array_length1,freq = str(int(1000000/Fs))+'u')
analog_df.index = rng1
if first:
combined_df = analog_df
first = False
else:
combined_df =pd.concat([combined_df, analog_df])
return combined_df, Fs
def correlation_analog_digital(name):
''' compute the correlation between the analog and the digital signal
@param name: name of the file to analyze
@todo different frequency for analog_periodg and digital - need to convert to same timescale
'''
#name = r'C:\Users\home\Documents\Bob\Cascade Experiments\CascadeKitchen_A_Nov01_1829'
title = name.split('\\')[-1]
file_analog = name + '_analog.txt'
file_digital = name + '_digital.txt'
# load analog data
data_analog, start_date, stop_date = reformat.load_data(file_analog)
Start_Time = time.mktime(start_date.timetuple())
Stop_Time=time.mktime(stop_date.timetuple())
diff = stop_date-start_date
time_diff_second = diff.seconds
len_data= len(data_analog)
analog_period = time_diff_second/np.float(len_data)
# analog dataframe
dataframe1 = pd.DataFrame(data_analog)
dataframe1.columns = ['sensor_amp']
ts = np.arange(Start_Time,Stop_Time,analog_period)
dataframe1['timestamp'] = ts[:len_data]
#normalization
dataframe1['sensor_amp'] = dataframe1['sensor_amp'] / np.mean(dataframe1['sensor_amp']) - 1.0
# load digital data
time_stamp, switch, flow = reformat.reformatdigfilex(file_digital, save= False)
digit_time_span = time_stamp[-1]-time_stamp[0]
#digital datafrme
dataframe2 = pd.DataFrame(time_stamp,columns = ['timestamp'])
dataframe2['state']= switch
dataframe2['flow'] = flow
array_length1 = len(dataframe1['timestamp'])
array_length2 = len(dataframe2['timestamp'])
Fs = int(np.float(len_data)/time_diff_second) # can be calculated based on time stamp.
Fs2 = int(array_length2/digit_time_span) # can be calculated based on time stamp.
rng1 = pd.date_range(dataframe1['timestamp'][0], periods = array_length1,freq = str(int(1000000/Fs))+'u')
rng2 = pd.date_range(dataframe2['timestamp'][0], periods = array_length2,freq = str(int(1000000/Fs2))+'u')
dataframe1.index = rng1
dataframe2.index = rng2
plt.close()
plt.subplot(4,1,2)
plt.xlabel('time')
plt.ylabel('sensor')
# plt.plot(dataframe1['timestamp'],dataframe1['sensor_amp'])
plt.plot(rng1,dataframe1['sensor_amp'])
plt.subplot(4,1,3)
plt.xlabel('time')
plt.ylabel('flow')
plt.plot(rng2,dataframe2['flow'])
timewindow = 1.0
ts1 = np.arange(0,array_length1)/(Fs*timewindow)
ts2 = np.arange(0,array_length2)/(Fs2*timewindow)
# new time range
width = Fs*timewindow
img, frq, nline = countour_fft(dataframe1['sensor_amp'],Fs = Fs, timewindow = timewindow)
# refrmating image in 2D array
rng = pd.date_range(dataframe1['timestamp'][0], periods = nline,freq = str(int(timewindow))+'s')
b = int(img.shape[0]/nline)
img2 = img[0:b*nline].reshape((nline,b))
mat_reduced, U_reduced, S_reduced, s_sum = svd_reduction(img2.T,20)
# plt.subplot(3,1,1)
# plt.imshow(img2[:,1:].T) # remove first point (very high)
z_reduced = np.dot(U_reduced.T,img2.T)
for i in [0,1,2,3]:
plt.subplot(4, 1, 1)
plt.title(title)
plt.plot(frq, U_reduced[:,i])
plt.ylabel('ampl')
plt.subplot(4, 1, 4)
plt.ylabel('spec. ampl.')
plt.plot(rng,z_reduced[i])
plt.xlabel('time')
output = r'C:\Users\home\Documents\Bob\steve\correlation_'+title+'.png'
plt.savefig(output)
#plt.show()
plt.close()
# resampling with the same period as the fft
flow = dataframe2.resample(str(int(timewindow))+'s', how='mean')
x = flow['flow'].values[:nline]
plt.ylabel('spect. amplitude')
plt.xlabel('flow amplitude')
xrg = np.arange(np.min(x),np.max(x),(np.max(x)-np.min(x))/30.0)
for i in [0,1,2,3]:
fit = np.polyfit(x,z_reduced[i],1)
fit_fn = np.poly1d(fit)
plt.plot(x,z_reduced[i],'.', xrg,fit_fn(xrg),'-')
#plt.ylim(-0.03, 0.01)
output = r'C:\Users\home\Documents\Bob\steve\correlation_flow_spectrum_quad_'+title+'.png'
plt.savefig(output)
return
def correlation_analog_digital(name):
''' compute the correlation between the analog and the digital signal
@param name: name of the file to analyze
@todo different frequency for analog_periodg and digital - need to convert to same timescale
'''
#name = r'C:\Users\home\Documents\Bob\Cascade Experiments\CascadeKitchen_A_Nov01_1829'
title = name.split('\\')[-1]
file_analog = name + '_analog.txt'
file_digital = name + '_digital.txt'
# load analog data
data_analog, start_date, stop_date = reformat.load_data(file_analog)
Start_Time = time.mktime(start_date.timetuple())
Stop_Time = time.mktime(stop_date.timetuple())
diff = stop_date - start_date
time_diff_second = diff.seconds
len_data = len(data_analog)
analog_period = time_diff_second / np.float(len_data)
# analog dataframe
dataframe1 = pd.DataFrame(data_analog)
dataframe1.columns = ['sensor_amp']
ts = np.arange(Start_Time, Stop_Time, analog_period)
dataframe1['timestamp'] = ts[:len_data]
#normalization
dataframe1['sensor_amp'] = dataframe1['sensor_amp'] / np.mean(
dataframe1['sensor_amp']) - 1.0
# load digital data
time_stamp, switch, flow = reformat.reformatdigfilex(file_digital,
save=False)
digit_time_span = time_stamp[-1] - time_stamp[0]
#digital datafrme
dataframe2 = pd.DataFrame(time_stamp, columns=['timestamp'])
dataframe2['state'] = switch
dataframe2['flow'] = flow
array_length1 = len(dataframe1['timestamp'])
array_length2 = len(dataframe2['timestamp'])
Fs = int(np.float(len_data) /
time_diff_second) # can be calculated based on time stamp.
Fs2 = int(array_length2 /
digit_time_span) # can be calculated based on time stamp.
rng1 = pd.date_range(dataframe1['timestamp'][0],
periods=array_length1,
freq=str(int(1000000 / Fs)) + 'u')
rng2 = pd.date_range(dataframe2['timestamp'][0],
periods=array_length2,
freq=str(int(1000000 / Fs2)) + 'u')
dataframe1.index = rng1
dataframe2.index = rng2
plt.close()
plt.subplot(4, 1, 2)
plt.xlabel('time')
plt.ylabel('sensor')
# plt.plot(dataframe1['timestamp'],dataframe1['sensor_amp'])
plt.plot(rng1, dataframe1['sensor_amp'])
plt.subplot(4, 1, 3)
plt.xlabel('time')
plt.ylabel('flow')
plt.plot(rng2, dataframe2['flow'])
timewindow = 1.0
ts1 = np.arange(0, array_length1) / (Fs * timewindow)
ts2 = np.arange(0, array_length2) / (Fs2 * timewindow)
# new time range
width = Fs * timewindow
img, frq, nline = countour_fft(dataframe1['sensor_amp'],
Fs=Fs,
timewindow=timewindow)
# refrmating image in 2D array
rng = pd.date_range(dataframe1['timestamp'][0],
periods=nline,
freq=str(int(timewindow)) + 's')
b = int(img.shape[0] / nline)
img2 = img[0:b * nline].reshape((nline, b))
mat_reduced, U_reduced, S_reduced, s_sum = svd_reduction(img2.T, 20)
# plt.subplot(3,1,1)
# plt.imshow(img2[:,1:].T) # remove first point (very high)
z_reduced = np.dot(U_reduced.T, img2.T)
for i in [0, 1, 2, 3]:
plt.subplot(4, 1, 1)
plt.title(title)
plt.plot(frq, U_reduced[:, i])
plt.ylabel('ampl')
plt.subplot(4, 1, 4)
plt.ylabel('spec. ampl.')
plt.plot(rng, z_reduced[i])
plt.xlabel('time')
output = r'C:\Users\home\Documents\Bob\steve\correlation_' + title + '.png'
plt.savefig(output)
#plt.show()
plt.close()
# resampling with the same period as the fft
flow = dataframe2.resample(str(int(timewindow)) + 's', how='mean')
x = flow['flow'].values[:nline]
plt.ylabel('spect. amplitude')
plt.xlabel('flow amplitude')
xrg = np.arange(np.min(x), np.max(x), (np.max(x) - np.min(x)) / 30.0)
for i in [0, 1, 2, 3]:
fit = np.polyfit(x, z_reduced[i], 1)
fit_fn = np.poly1d(fit)
plt.plot(x, z_reduced[i], '.', xrg, fit_fn(xrg), '-')
#plt.ylim(-0.03, 0.01)
output = r'C:\Users\home\Documents\Bob\steve\correlation_flow_spectrum_quad_' + title + '.png'
plt.savefig(output)
return