def process_digital(table_sort, directory, target_directory =r'C:\Sandbox\coursera\cascade\data',letter='A', time_start = '2015-01-01',time_stop = '2015-01-15',width = 900,Fs = 200.0, timewindow = 4.0 ):
''' execute a series of operation on a dataframe of files defined by process_directory
Parameters:
----------
table_sort - dataframe with a table of files
directory - directory for the location of the silder
target_directory - location to save images
'''
table_digital = table_sort[(table_sort.typef=='digital') & (table_sort.letter == letter) & (table_sort.timestamp < time_stop) & (table_sort.timestamp > time_start)]
img = []
first = True
for filename in table_digital.filename:
fullname = os.path.join(directory,filename)
print filename
summary = summary_digital(fullname,Fs = Fs, timewindow = timewindow)
#image_series(fullname,Fs = 4000.0, timewindow = 10.0 )
try:
time_stamp, switch, flow = reformat.reformatdigfilex(fullname, save= False)
digit_time_span = time_stamp[-1]-time_stamp[0]
#digital datafrme
dataframe2 = pd.DataFrame(time_stamp,columns = ['timestamp'])
dataframe2['state']= switch
dataframe2['flow'] = flow
array_length2 = len(dataframe2['timestamp'])
Fs2 = int(array_length2/digit_time_span) # can be calculated based on time stamp.
rng2 = pd.date_range(dataframe2['timestamp'][0], periods = array_length2,freq = str(int(1000000/Fs2))+'u')
dataframe2.index = rng2
#reduction of size. timewindow
df_reduced = dataframe2.resample(str(int(timewindow))+'s', how='mean')
if first:
combined_df = df_reduced
first = False
else:
combined_df =pd.concat([combined_df, df_reduced])
except:
print 'issue with:',filename
print len(summary),' ',np.sum(summary)
img = np.concatenate((img,summary))
a = np.int(np.sqrt(len(img)))
b = len(img)/width
img2 = img[0:(a*a)].reshape((a,a))
img2 = img[0:(b*width)].reshape((b,width))
plt.clf()
### plt.axis([0,1000,0,1000])
plt.imshow(img2)
#plt.show()
fname = 'digital_'+letter +'_'+ time_start +'_'+time_stop+'_'+str(int(timewindow))+'s'+'.png'
output = os.path.join(target_directory,fname)
print('Saving frame', output)
plt.savefig(output)
return img2, combined_df
def process_digital(filelist):
''' execute a series of operation on a set digital files and conver them into a dataframe
Parameters:
----------
filelist - dlist of files to process
figoutput - name of file for saving image
return combined_df: dataframe with timestamp, flow and state of valve
'''
first = True
for fullname in filelist:
print 'processing:',fullname
try:
time_stamp, switch, flow = reformat.reformatdigfilex(fullname, save= False)
digit_time_span = time_stamp[-1]-time_stamp[0]
#digital datafrme
digital_df = pd.DataFrame(time_stamp,columns = ['timestamp'])
digital_df['state']= switch
digital_df['flow'] = flow
array_length2 = len(digital_df['timestamp'])
Fs2 = int(array_length2/digit_time_span) # can be calculated based on time stamp.
rng2 = pd.date_range(digital_df['timestamp'][0], periods = array_length2,freq = str(int(1000000/Fs2))+'u')
digital_df.index = rng2
if first:
combined_df = digital_df
first = False
else:
combined_df =pd.concat([combined_df, digital_df])
except:
print 'issue with:',fullname
return combined_df, Fs2
def process_analog(filelist):
''' execute a series of operation on a dataframe of files defined by process_directory
Parameters:
----------
filelist - dlist of files to process
figoutput - name of file for saving image
return combined_df: dataframe with sensor signal
'''
first = True
for fullname in filelist:
print 'processing:', fullname
data_analog, start_date, stop_date = reformat.load_data(fullname)
Start_Time = time.mktime(start_date.timetuple())
Stop_Time = time.mktime(stop_date.timetuple())
diff = stop_date - start_date
time_diff_second = diff.seconds
len_data = len(data_analog)
analog_period = time_diff_second / np.float(len_data)
# analog dataframe
analog_df = pd.DataFrame(data_analog)
analog_df.columns = ['sensor_amp']
ts = np.arange(Start_Time, Stop_Time, analog_period)
analog_df['timestamp'] = ts[:len_data]
#normalization
analog_df['sensor_amp'] = analog_df['sensor_amp'] - np.mean(
analog_df['sensor_amp'])
array_length1 = len(analog_df['timestamp'])
Fs = int(np.float(len_data) /
time_diff_second) # can be calculated based on time stamp.
rng1 = pd.date_range(analog_df['timestamp'][0],
periods=array_length1,
freq=str(int(1000000 / Fs)) + 'u')
analog_df.index = rng1
if first:
combined_df = analog_df
first = False
else:
combined_df = pd.concat([combined_df, analog_df])
return combined_df, Fs
def process_analog(filelist):
''' execute a series of operation on a dataframe of files defined by process_directory
Parameters:
----------
filelist - dlist of files to process
figoutput - name of file for saving image
return combined_df: dataframe with sensor signal
'''
first = True
for fullname in filelist:
print 'processing:', fullname
data_analog, start_date, stop_date = reformat.load_data(fullname)
Start_Time = time.mktime(start_date.timetuple())
Stop_Time=time.mktime(stop_date.timetuple())
diff = stop_date-start_date
time_diff_second = diff.seconds
len_data= len(data_analog)
analog_period = time_diff_second/np.float(len_data)
# analog dataframe
analog_df = pd.DataFrame(data_analog)
analog_df.columns = ['sensor_amp']
ts = np.arange(Start_Time,Stop_Time,analog_period)
analog_df['timestamp'] = ts[:len_data]
#normalization
analog_df['sensor_amp'] = analog_df['sensor_amp'] - np.mean(analog_df['sensor_amp'])
array_length1 = len(analog_df['timestamp'])
Fs = int(np.float(len_data)/time_diff_second) # can be calculated based on time stamp.
rng1 = pd.date_range(analog_df['timestamp'][0], periods = array_length1,freq = str(int(1000000/Fs))+'u')
analog_df.index = rng1
if first:
combined_df = analog_df
first = False
else:
combined_df =pd.concat([combined_df, analog_df])
return combined_df, Fs
def process_digital(filelist):
''' execute a series of operation on a set digital files and conver them into a dataframe
Parameters:
----------
filelist - dlist of files to process
figoutput - name of file for saving image
return combined_df: dataframe with timestamp, flow and state of valve
'''
first = True
for fullname in filelist:
print 'processing:', fullname
try:
time_stamp, switch, flow = reformat.reformatdigfilex(fullname,
save=False)
digit_time_span = time_stamp[-1] - time_stamp[0]
#digital datafrme
digital_df = pd.DataFrame(time_stamp, columns=['timestamp'])
digital_df['state'] = switch
digital_df['flow'] = flow
array_length2 = len(digital_df['timestamp'])
Fs2 = int(
array_length2 /
digit_time_span) # can be calculated based on time stamp.
rng2 = pd.date_range(digital_df['timestamp'][0],
periods=array_length2,
freq=str(int(1000000 / Fs2)) + 'u')
digital_df.index = rng2
if first:
combined_df = digital_df
first = False
else:
combined_df = pd.concat([combined_df, digital_df])
except:
print 'issue with:', fullname
return combined_df, Fs2
def correlation_analog_digital(name):
''' compute the correlation between the analog and the digital signal
@param name: name of the file to analyze
@todo different frequency for analog_periodg and digital - need to convert to same timescale
'''
#name = r'C:\Users\home\Documents\Bob\Cascade Experiments\CascadeKitchen_A_Nov01_1829'
title = name.split('\\')[-1]
file_analog = name + '_analog.txt'
file_digital = name + '_digital.txt'
# load analog data
data_analog, start_date, stop_date = reformat.load_data(file_analog)
Start_Time = time.mktime(start_date.timetuple())
Stop_Time=time.mktime(stop_date.timetuple())
diff = stop_date-start_date
time_diff_second = diff.seconds
len_data= len(data_analog)
analog_period = time_diff_second/np.float(len_data)
# analog dataframe
dataframe1 = pd.DataFrame(data_analog)
dataframe1.columns = ['sensor_amp']
ts = np.arange(Start_Time,Stop_Time,analog_period)
dataframe1['timestamp'] = ts[:len_data]
#normalization
dataframe1['sensor_amp'] = dataframe1['sensor_amp'] / np.mean(dataframe1['sensor_amp']) - 1.0
# load digital data
time_stamp, switch, flow = reformat.reformatdigfilex(file_digital, save= False)
digit_time_span = time_stamp[-1]-time_stamp[0]
#digital datafrme
dataframe2 = pd.DataFrame(time_stamp,columns = ['timestamp'])
dataframe2['state']= switch
dataframe2['flow'] = flow
array_length1 = len(dataframe1['timestamp'])
array_length2 = len(dataframe2['timestamp'])
Fs = int(np.float(len_data)/time_diff_second) # can be calculated based on time stamp.
Fs2 = int(array_length2/digit_time_span) # can be calculated based on time stamp.
rng1 = pd.date_range(dataframe1['timestamp'][0], periods = array_length1,freq = str(int(1000000/Fs))+'u')
rng2 = pd.date_range(dataframe2['timestamp'][0], periods = array_length2,freq = str(int(1000000/Fs2))+'u')
dataframe1.index = rng1
dataframe2.index = rng2
plt.close()
plt.subplot(4,1,2)
plt.xlabel('time')
plt.ylabel('sensor')
# plt.plot(dataframe1['timestamp'],dataframe1['sensor_amp'])
plt.plot(rng1,dataframe1['sensor_amp'])
plt.subplot(4,1,3)
plt.xlabel('time')
plt.ylabel('flow')
plt.plot(rng2,dataframe2['flow'])
timewindow = 1.0
ts1 = np.arange(0,array_length1)/(Fs*timewindow)
ts2 = np.arange(0,array_length2)/(Fs2*timewindow)
# new time range
width = Fs*timewindow
img, frq, nline = countour_fft(dataframe1['sensor_amp'],Fs = Fs, timewindow = timewindow)
# refrmating image in 2D array
rng = pd.date_range(dataframe1['timestamp'][0], periods = nline,freq = str(int(timewindow))+'s')
b = int(img.shape[0]/nline)
img2 = img[0:b*nline].reshape((nline,b))
mat_reduced, U_reduced, S_reduced, s_sum = svd_reduction(img2.T,20)
# plt.subplot(3,1,1)
# plt.imshow(img2[:,1:].T) # remove first point (very high)
z_reduced = np.dot(U_reduced.T,img2.T)
for i in [0,1,2,3]:
plt.subplot(4, 1, 1)
plt.title(title)
plt.plot(frq, U_reduced[:,i])
plt.ylabel('ampl')
plt.subplot(4, 1, 4)
plt.ylabel('spec. ampl.')
plt.plot(rng,z_reduced[i])
plt.xlabel('time')
output = r'C:\Users\home\Documents\Bob\steve\correlation_'+title+'.png'
plt.savefig(output)
#plt.show()
plt.close()
# resampling with the same period as the fft
flow = dataframe2.resample(str(int(timewindow))+'s', how='mean')
x = flow['flow'].values[:nline]
plt.ylabel('spect. amplitude')
plt.xlabel('flow amplitude')
xrg = np.arange(np.min(x),np.max(x),(np.max(x)-np.min(x))/30.0)
for i in [0,1,2,3]:
fit = np.polyfit(x,z_reduced[i],1)
fit_fn = np.poly1d(fit)
plt.plot(x,z_reduced[i],'.', xrg,fit_fn(xrg),'-')
#plt.ylim(-0.03, 0.01)
output = r'C:\Users\home\Documents\Bob\steve\correlation_flow_spectrum_quad_'+title+'.png'
plt.savefig(output)
return
def correlation_analog_digital(name):
''' compute the correlation between the analog and the digital signal
@param name: name of the file to analyze
@todo different frequency for analog_periodg and digital - need to convert to same timescale
'''
#name = r'C:\Users\home\Documents\Bob\Cascade Experiments\CascadeKitchen_A_Nov01_1829'
title = name.split('\\')[-1]
file_analog = name + '_analog.txt'
file_digital = name + '_digital.txt'
# load analog data
data_analog, start_date, stop_date = reformat.load_data(file_analog)
Start_Time = time.mktime(start_date.timetuple())
Stop_Time = time.mktime(stop_date.timetuple())
diff = stop_date - start_date
time_diff_second = diff.seconds
len_data = len(data_analog)
analog_period = time_diff_second / np.float(len_data)
# analog dataframe
dataframe1 = pd.DataFrame(data_analog)
dataframe1.columns = ['sensor_amp']
ts = np.arange(Start_Time, Stop_Time, analog_period)
dataframe1['timestamp'] = ts[:len_data]
#normalization
dataframe1['sensor_amp'] = dataframe1['sensor_amp'] / np.mean(
dataframe1['sensor_amp']) - 1.0
# load digital data
time_stamp, switch, flow = reformat.reformatdigfilex(file_digital,
save=False)
digit_time_span = time_stamp[-1] - time_stamp[0]
#digital datafrme
dataframe2 = pd.DataFrame(time_stamp, columns=['timestamp'])
dataframe2['state'] = switch
dataframe2['flow'] = flow
array_length1 = len(dataframe1['timestamp'])
array_length2 = len(dataframe2['timestamp'])
Fs = int(np.float(len_data) /
time_diff_second) # can be calculated based on time stamp.
Fs2 = int(array_length2 /
digit_time_span) # can be calculated based on time stamp.
rng1 = pd.date_range(dataframe1['timestamp'][0],
periods=array_length1,
freq=str(int(1000000 / Fs)) + 'u')
rng2 = pd.date_range(dataframe2['timestamp'][0],
periods=array_length2,
freq=str(int(1000000 / Fs2)) + 'u')
dataframe1.index = rng1
dataframe2.index = rng2
plt.close()
plt.subplot(4, 1, 2)
plt.xlabel('time')
plt.ylabel('sensor')
# plt.plot(dataframe1['timestamp'],dataframe1['sensor_amp'])
plt.plot(rng1, dataframe1['sensor_amp'])
plt.subplot(4, 1, 3)
plt.xlabel('time')
plt.ylabel('flow')
plt.plot(rng2, dataframe2['flow'])
timewindow = 1.0
ts1 = np.arange(0, array_length1) / (Fs * timewindow)
ts2 = np.arange(0, array_length2) / (Fs2 * timewindow)
# new time range
width = Fs * timewindow
img, frq, nline = countour_fft(dataframe1['sensor_amp'],
Fs=Fs,
timewindow=timewindow)
# refrmating image in 2D array
rng = pd.date_range(dataframe1['timestamp'][0],
periods=nline,
freq=str(int(timewindow)) + 's')
b = int(img.shape[0] / nline)
img2 = img[0:b * nline].reshape((nline, b))
mat_reduced, U_reduced, S_reduced, s_sum = svd_reduction(img2.T, 20)
# plt.subplot(3,1,1)
# plt.imshow(img2[:,1:].T) # remove first point (very high)
z_reduced = np.dot(U_reduced.T, img2.T)
for i in [0, 1, 2, 3]:
plt.subplot(4, 1, 1)
plt.title(title)
plt.plot(frq, U_reduced[:, i])
plt.ylabel('ampl')
plt.subplot(4, 1, 4)
plt.ylabel('spec. ampl.')
plt.plot(rng, z_reduced[i])
plt.xlabel('time')
output = r'C:\Users\home\Documents\Bob\steve\correlation_' + title + '.png'
plt.savefig(output)
#plt.show()
plt.close()
# resampling with the same period as the fft
flow = dataframe2.resample(str(int(timewindow)) + 's', how='mean')
x = flow['flow'].values[:nline]
plt.ylabel('spect. amplitude')
plt.xlabel('flow amplitude')
xrg = np.arange(np.min(x), np.max(x), (np.max(x) - np.min(x)) / 30.0)
for i in [0, 1, 2, 3]:
fit = np.polyfit(x, z_reduced[i], 1)
fit_fn = np.poly1d(fit)
plt.plot(x, z_reduced[i], '.', xrg, fit_fn(xrg), '-')
#plt.ylim(-0.03, 0.01)
output = r'C:\Users\home\Documents\Bob\steve\correlation_flow_spectrum_quad_' + title + '.png'
plt.savefig(output)
return
def process_digital(table_sort,
directory,
target_directory=r'C:\Sandbox\coursera\cascade\data',
letter='A',
time_start='2015-01-01',
time_stop='2015-01-15',
width=900,
Fs=200.0,
timewindow=4.0):
''' execute a series of operation on a dataframe of files defined by process_directory
Parameters:
----------
table_sort - dataframe with a table of files
directory - directory for the location of the silder
target_directory - location to save images
'''
table_digital = table_sort[(table_sort.typef == 'digital')
& (table_sort.letter == letter) &
(table_sort.timestamp < time_stop) &
(table_sort.timestamp > time_start)]
img = []
first = True
for filename in table_digital.filename:
fullname = os.path.join(directory, filename)
print filename
summary = summary_digital(fullname, Fs=Fs, timewindow=timewindow)
#image_series(fullname,Fs = 4000.0, timewindow = 10.0 )
try:
time_stamp, switch, flow = reformat.reformatdigfilex(fullname,
save=False)
digit_time_span = time_stamp[-1] - time_stamp[0]
#digital datafrme
dataframe2 = pd.DataFrame(time_stamp, columns=['timestamp'])
dataframe2['state'] = switch
dataframe2['flow'] = flow
array_length2 = len(dataframe2['timestamp'])
Fs2 = int(
array_length2 /
digit_time_span) # can be calculated based on time stamp.
rng2 = pd.date_range(dataframe2['timestamp'][0],
periods=array_length2,
freq=str(int(1000000 / Fs2)) + 'u')
dataframe2.index = rng2
#reduction of size. timewindow
df_reduced = dataframe2.resample(str(int(timewindow)) + 's',
how='mean')
if first:
combined_df = df_reduced
first = False
else:
combined_df = pd.concat([combined_df, df_reduced])
except:
print 'issue with:', filename
print len(summary), ' ', np.sum(summary)
img = np.concatenate((img, summary))
a = np.int(np.sqrt(len(img)))
b = len(img) / width
img2 = img[0:(a * a)].reshape((a, a))
img2 = img[0:(b * width)].reshape((b, width))
plt.clf()
### plt.axis([0,1000,0,1000])
plt.imshow(img2)
#plt.show()
fname = 'digital_' + letter + '_' + time_start + '_' + time_stop + '_' + str(
int(timewindow)) + 's' + '.png'
output = os.path.join(target_directory, fname)
print('Saving frame', output)
plt.savefig(output)
return img2, combined_df