示例#1
0
def process_digital(table_sort, directory, target_directory =r'C:\Sandbox\coursera\cascade\data',letter='A', time_start = '2015-01-01',time_stop = '2015-01-15',width = 900,Fs = 200.0, timewindow = 4.0 ):
    ''' execute a series of operation on a dataframe of files defined by process_directory
    Parameters:
    ----------
    table_sort - dataframe with a table of files 
    directory - directory for the location of the silder
    target_directory - location to save images
    '''
    table_digital = table_sort[(table_sort.typef=='digital') & (table_sort.letter == letter) & (table_sort.timestamp < time_stop) & (table_sort.timestamp > time_start)]
    img = []
    first = True

    for filename in table_digital.filename:
        fullname = os.path.join(directory,filename)
        print filename
        summary = summary_digital(fullname,Fs = Fs, timewindow = timewindow)
        #image_series(fullname,Fs = 4000.0, timewindow = 10.0 )
        try:
            time_stamp, switch, flow = reformat.reformatdigfilex(fullname, save= False)
            digit_time_span = time_stamp[-1]-time_stamp[0]
    
        #digital datafrme
            dataframe2 = pd.DataFrame(time_stamp,columns = ['timestamp'])
            dataframe2['state']= switch
            dataframe2['flow'] = flow
            array_length2 = len(dataframe2['timestamp'])
            
            Fs2 = int(array_length2/digit_time_span)   # can be calculated based on time stamp.
            rng2 = pd.date_range(dataframe2['timestamp'][0], periods = array_length2,freq = str(int(1000000/Fs2))+'u')
            dataframe2.index = rng2
            
            #reduction of size. timewindow        
            df_reduced  = dataframe2.resample(str(int(timewindow))+'s', how='mean')
            if first:
                combined_df = df_reduced
                first = False
            else:
                combined_df =pd.concat([combined_df, df_reduced]) 
        except:
            print 'issue with:',filename
        
        print len(summary),' ',np.sum(summary)
        img = np.concatenate((img,summary))
        
        
    a = np.int(np.sqrt(len(img)))
    b = len(img)/width
    img2 = img[0:(a*a)].reshape((a,a))
    img2 = img[0:(b*width)].reshape((b,width))
    plt.clf()
###    plt.axis([0,1000,0,1000])
    plt.imshow(img2)
    #plt.show()
    fname = 'digital_'+letter +'_'+ time_start +'_'+time_stop+'_'+str(int(timewindow))+'s'+'.png'
    output = os.path.join(target_directory,fname)
    print('Saving frame', output)
    plt.savefig(output)
    
    
    return img2, combined_df
示例#2
0
def process_digital(filelist):
    ''' execute a series of operation on a set digital files and conver them into a dataframe
    Parameters:
    ----------
    filelist - dlist of files to process 
    figoutput - name of file for saving image
    return   combined_df: dataframe with timestamp, flow and state of valve
    
    '''
    first = True
    for fullname in filelist:
        print 'processing:',fullname
        try:
            time_stamp, switch, flow = reformat.reformatdigfilex(fullname, save= False)
            digit_time_span = time_stamp[-1]-time_stamp[0]
    
        #digital datafrme
            digital_df = pd.DataFrame(time_stamp,columns = ['timestamp'])
            digital_df['state']= switch
            digital_df['flow'] = flow
            array_length2 = len(digital_df['timestamp'])
            
            Fs2 = int(array_length2/digit_time_span)   # can be calculated based on time stamp.
            rng2 = pd.date_range(digital_df['timestamp'][0], periods = array_length2,freq = str(int(1000000/Fs2))+'u')
            digital_df.index = rng2
            
            if first:
                combined_df = digital_df
                first = False
            else:
                combined_df =pd.concat([combined_df, digital_df]) 
        except:
            print 'issue with:',fullname
        
    return combined_df, Fs2
示例#3
0
def process_analog(filelist):
    ''' execute a series of operation on a dataframe of files defined by process_directory
    Parameters:
    ----------
    filelist - dlist of files to process 
    figoutput - name of file for saving image
    return   combined_df: dataframe with sensor signal
    
    '''
    first = True

    for fullname in filelist:
        print 'processing:', fullname
        data_analog, start_date, stop_date = reformat.load_data(fullname)
        Start_Time = time.mktime(start_date.timetuple())
        Stop_Time = time.mktime(stop_date.timetuple())

        diff = stop_date - start_date
        time_diff_second = diff.seconds
        len_data = len(data_analog)
        analog_period = time_diff_second / np.float(len_data)

        # analog dataframe
        analog_df = pd.DataFrame(data_analog)
        analog_df.columns = ['sensor_amp']
        ts = np.arange(Start_Time, Stop_Time, analog_period)
        analog_df['timestamp'] = ts[:len_data]
        #normalization
        analog_df['sensor_amp'] = analog_df['sensor_amp'] - np.mean(
            analog_df['sensor_amp'])
        array_length1 = len(analog_df['timestamp'])

        Fs = int(np.float(len_data) /
                 time_diff_second)  # can be calculated based on time stamp.

        rng1 = pd.date_range(analog_df['timestamp'][0],
                             periods=array_length1,
                             freq=str(int(1000000 / Fs)) + 'u')
        analog_df.index = rng1
        if first:
            combined_df = analog_df
            first = False
        else:
            combined_df = pd.concat([combined_df, analog_df])

    return combined_df, Fs
示例#4
0
def process_analog(filelist):
    ''' execute a series of operation on a dataframe of files defined by process_directory
    Parameters:
    ----------
    filelist - dlist of files to process 
    figoutput - name of file for saving image
    return   combined_df: dataframe with sensor signal
    
    '''
    first = True

    for fullname in filelist:
        print 'processing:', fullname
        data_analog, start_date, stop_date = reformat.load_data(fullname)
        Start_Time = time.mktime(start_date.timetuple())
        Stop_Time=time.mktime(stop_date.timetuple())
    
        diff = stop_date-start_date
        time_diff_second = diff.seconds
        len_data= len(data_analog)
        analog_period = time_diff_second/np.float(len_data)
    
        # analog dataframe
        analog_df = pd.DataFrame(data_analog)
        analog_df.columns = ['sensor_amp']
        ts = np.arange(Start_Time,Stop_Time,analog_period)
        analog_df['timestamp'] = ts[:len_data]
        #normalization
        analog_df['sensor_amp'] = analog_df['sensor_amp'] - np.mean(analog_df['sensor_amp'])
        array_length1 = len(analog_df['timestamp'])
 
        Fs = int(np.float(len_data)/time_diff_second) # can be calculated based on time stamp.
    
        rng1 = pd.date_range(analog_df['timestamp'][0], periods = array_length1,freq = str(int(1000000/Fs))+'u')
        analog_df.index = rng1
        if first:
            combined_df = analog_df
            first = False
        else:
            combined_df =pd.concat([combined_df, analog_df]) 
        
    return combined_df, Fs
示例#5
0
def process_digital(filelist):
    ''' execute a series of operation on a set digital files and conver them into a dataframe
    Parameters:
    ----------
    filelist - dlist of files to process 
    figoutput - name of file for saving image
    return   combined_df: dataframe with timestamp, flow and state of valve
    
    '''
    first = True
    for fullname in filelist:
        print 'processing:', fullname
        try:
            time_stamp, switch, flow = reformat.reformatdigfilex(fullname,
                                                                 save=False)
            digit_time_span = time_stamp[-1] - time_stamp[0]

            #digital datafrme
            digital_df = pd.DataFrame(time_stamp, columns=['timestamp'])
            digital_df['state'] = switch
            digital_df['flow'] = flow
            array_length2 = len(digital_df['timestamp'])

            Fs2 = int(
                array_length2 /
                digit_time_span)  # can be calculated based on time stamp.
            rng2 = pd.date_range(digital_df['timestamp'][0],
                                 periods=array_length2,
                                 freq=str(int(1000000 / Fs2)) + 'u')
            digital_df.index = rng2

            if first:
                combined_df = digital_df
                first = False
            else:
                combined_df = pd.concat([combined_df, digital_df])
        except:
            print 'issue with:', fullname

    return combined_df, Fs2
示例#6
0
def correlation_analog_digital(name):
    ''' compute the correlation between the analog and the digital signal
    @param name: name of the file to analyze
    
    @todo different frequency for analog_periodg and digital - need to convert to same timescale
    '''
    
    #name = r'C:\Users\home\Documents\Bob\Cascade Experiments\CascadeKitchen_A_Nov01_1829'
    
    title = name.split('\\')[-1]
    file_analog = name + '_analog.txt'
    file_digital = name + '_digital.txt'
    
    # load analog data
    data_analog, start_date, stop_date = reformat.load_data(file_analog)
    Start_Time = time.mktime(start_date.timetuple())
    Stop_Time=time.mktime(stop_date.timetuple())

    diff = stop_date-start_date
    time_diff_second = diff.seconds
    len_data= len(data_analog)
    analog_period = time_diff_second/np.float(len_data)

    # analog dataframe
    dataframe1 = pd.DataFrame(data_analog)
    dataframe1.columns = ['sensor_amp']
    ts = np.arange(Start_Time,Stop_Time,analog_period)
    dataframe1['timestamp'] = ts[:len_data]
    #normalization
    dataframe1['sensor_amp'] = dataframe1['sensor_amp'] / np.mean(dataframe1['sensor_amp']) - 1.0

    # load digital data
    time_stamp, switch, flow = reformat.reformatdigfilex(file_digital, save= False)
    digit_time_span = time_stamp[-1]-time_stamp[0]

    #digital datafrme
    dataframe2 = pd.DataFrame(time_stamp,columns = ['timestamp'])
    dataframe2['state']= switch
    dataframe2['flow'] = flow

    array_length1 = len(dataframe1['timestamp'])
    array_length2 = len(dataframe2['timestamp'])

    Fs = int(np.float(len_data)/time_diff_second) # can be calculated based on time stamp.
    Fs2 = int(array_length2/digit_time_span)   # can be calculated based on time stamp.

    rng1 = pd.date_range(dataframe1['timestamp'][0], periods = array_length1,freq = str(int(1000000/Fs))+'u')
    rng2 = pd.date_range(dataframe2['timestamp'][0], periods = array_length2,freq = str(int(1000000/Fs2))+'u')
    dataframe1.index = rng1
    dataframe2.index = rng2
    plt.close()
    plt.subplot(4,1,2)
    plt.xlabel('time')
    plt.ylabel('sensor')
#    plt.plot(dataframe1['timestamp'],dataframe1['sensor_amp'])
    plt.plot(rng1,dataframe1['sensor_amp'])
    plt.subplot(4,1,3)
    plt.xlabel('time')
    plt.ylabel('flow')
    plt.plot(rng2,dataframe2['flow'])

    timewindow = 1.0
    ts1 = np.arange(0,array_length1)/(Fs*timewindow)
    ts2 = np.arange(0,array_length2)/(Fs2*timewindow)
    # new time range
    width = Fs*timewindow
    img, frq, nline = countour_fft(dataframe1['sensor_amp'],Fs = Fs, timewindow = timewindow)
    # refrmating image in 2D array
    rng = pd.date_range(dataframe1['timestamp'][0], periods = nline,freq = str(int(timewindow))+'s')
    b = int(img.shape[0]/nline)
    img2 = img[0:b*nline].reshape((nline,b))
    mat_reduced, U_reduced, S_reduced, s_sum = svd_reduction(img2.T,20)


#    plt.subplot(3,1,1)
#    plt.imshow(img2[:,1:].T)  # remove first point (very high)
    z_reduced = np.dot(U_reduced.T,img2.T)
    for i in [0,1,2,3]:
        plt.subplot(4, 1, 1)
        plt.title(title)
        plt.plot(frq, U_reduced[:,i])
        plt.ylabel('ampl')
        plt.subplot(4, 1, 4)
        plt.ylabel('spec. ampl.')
        plt.plot(rng,z_reduced[i])
        plt.xlabel('time')
    output = r'C:\Users\home\Documents\Bob\steve\correlation_'+title+'.png'
    plt.savefig(output)
    #plt.show()
    plt.close()
    # resampling with the same period as the fft 
    flow  = dataframe2.resample(str(int(timewindow))+'s', how='mean')
    
    x = flow['flow'].values[:nline]
    plt.ylabel('spect. amplitude')
    plt.xlabel('flow amplitude')
    xrg = np.arange(np.min(x),np.max(x),(np.max(x)-np.min(x))/30.0)
    for i in [0,1,2,3]:
        fit = np.polyfit(x,z_reduced[i],1)
        fit_fn = np.poly1d(fit)
        plt.plot(x,z_reduced[i],'.', xrg,fit_fn(xrg),'-')
    #plt.ylim(-0.03, 0.01)
    output = r'C:\Users\home\Documents\Bob\steve\correlation_flow_spectrum_quad_'+title+'.png'
    plt.savefig(output)
    return
示例#7
0
def correlation_analog_digital(name):
    ''' compute the correlation between the analog and the digital signal
    @param name: name of the file to analyze
    
    @todo different frequency for analog_periodg and digital - need to convert to same timescale
    '''

    #name = r'C:\Users\home\Documents\Bob\Cascade Experiments\CascadeKitchen_A_Nov01_1829'

    title = name.split('\\')[-1]
    file_analog = name + '_analog.txt'
    file_digital = name + '_digital.txt'

    # load analog data
    data_analog, start_date, stop_date = reformat.load_data(file_analog)
    Start_Time = time.mktime(start_date.timetuple())
    Stop_Time = time.mktime(stop_date.timetuple())

    diff = stop_date - start_date
    time_diff_second = diff.seconds
    len_data = len(data_analog)
    analog_period = time_diff_second / np.float(len_data)

    # analog dataframe
    dataframe1 = pd.DataFrame(data_analog)
    dataframe1.columns = ['sensor_amp']
    ts = np.arange(Start_Time, Stop_Time, analog_period)
    dataframe1['timestamp'] = ts[:len_data]
    #normalization
    dataframe1['sensor_amp'] = dataframe1['sensor_amp'] / np.mean(
        dataframe1['sensor_amp']) - 1.0

    # load digital data
    time_stamp, switch, flow = reformat.reformatdigfilex(file_digital,
                                                         save=False)
    digit_time_span = time_stamp[-1] - time_stamp[0]

    #digital datafrme
    dataframe2 = pd.DataFrame(time_stamp, columns=['timestamp'])
    dataframe2['state'] = switch
    dataframe2['flow'] = flow

    array_length1 = len(dataframe1['timestamp'])
    array_length2 = len(dataframe2['timestamp'])

    Fs = int(np.float(len_data) /
             time_diff_second)  # can be calculated based on time stamp.
    Fs2 = int(array_length2 /
              digit_time_span)  # can be calculated based on time stamp.

    rng1 = pd.date_range(dataframe1['timestamp'][0],
                         periods=array_length1,
                         freq=str(int(1000000 / Fs)) + 'u')
    rng2 = pd.date_range(dataframe2['timestamp'][0],
                         periods=array_length2,
                         freq=str(int(1000000 / Fs2)) + 'u')
    dataframe1.index = rng1
    dataframe2.index = rng2
    plt.close()
    plt.subplot(4, 1, 2)
    plt.xlabel('time')
    plt.ylabel('sensor')
    #    plt.plot(dataframe1['timestamp'],dataframe1['sensor_amp'])
    plt.plot(rng1, dataframe1['sensor_amp'])
    plt.subplot(4, 1, 3)
    plt.xlabel('time')
    plt.ylabel('flow')
    plt.plot(rng2, dataframe2['flow'])

    timewindow = 1.0
    ts1 = np.arange(0, array_length1) / (Fs * timewindow)
    ts2 = np.arange(0, array_length2) / (Fs2 * timewindow)
    # new time range
    width = Fs * timewindow
    img, frq, nline = countour_fft(dataframe1['sensor_amp'],
                                   Fs=Fs,
                                   timewindow=timewindow)
    # refrmating image in 2D array
    rng = pd.date_range(dataframe1['timestamp'][0],
                        periods=nline,
                        freq=str(int(timewindow)) + 's')
    b = int(img.shape[0] / nline)
    img2 = img[0:b * nline].reshape((nline, b))
    mat_reduced, U_reduced, S_reduced, s_sum = svd_reduction(img2.T, 20)

    #    plt.subplot(3,1,1)
    #    plt.imshow(img2[:,1:].T)  # remove first point (very high)
    z_reduced = np.dot(U_reduced.T, img2.T)
    for i in [0, 1, 2, 3]:
        plt.subplot(4, 1, 1)
        plt.title(title)
        plt.plot(frq, U_reduced[:, i])
        plt.ylabel('ampl')
        plt.subplot(4, 1, 4)
        plt.ylabel('spec. ampl.')
        plt.plot(rng, z_reduced[i])
        plt.xlabel('time')
    output = r'C:\Users\home\Documents\Bob\steve\correlation_' + title + '.png'
    plt.savefig(output)
    #plt.show()
    plt.close()
    # resampling with the same period as the fft
    flow = dataframe2.resample(str(int(timewindow)) + 's', how='mean')

    x = flow['flow'].values[:nline]
    plt.ylabel('spect. amplitude')
    plt.xlabel('flow amplitude')
    xrg = np.arange(np.min(x), np.max(x), (np.max(x) - np.min(x)) / 30.0)
    for i in [0, 1, 2, 3]:
        fit = np.polyfit(x, z_reduced[i], 1)
        fit_fn = np.poly1d(fit)
        plt.plot(x, z_reduced[i], '.', xrg, fit_fn(xrg), '-')
    #plt.ylim(-0.03, 0.01)
    output = r'C:\Users\home\Documents\Bob\steve\correlation_flow_spectrum_quad_' + title + '.png'
    plt.savefig(output)
    return
示例#8
0
def process_digital(table_sort,
                    directory,
                    target_directory=r'C:\Sandbox\coursera\cascade\data',
                    letter='A',
                    time_start='2015-01-01',
                    time_stop='2015-01-15',
                    width=900,
                    Fs=200.0,
                    timewindow=4.0):
    ''' execute a series of operation on a dataframe of files defined by process_directory
    Parameters:
    ----------
    table_sort - dataframe with a table of files 
    directory - directory for the location of the silder
    target_directory - location to save images
    '''
    table_digital = table_sort[(table_sort.typef == 'digital')
                               & (table_sort.letter == letter) &
                               (table_sort.timestamp < time_stop) &
                               (table_sort.timestamp > time_start)]
    img = []
    first = True

    for filename in table_digital.filename:
        fullname = os.path.join(directory, filename)
        print filename
        summary = summary_digital(fullname, Fs=Fs, timewindow=timewindow)
        #image_series(fullname,Fs = 4000.0, timewindow = 10.0 )
        try:
            time_stamp, switch, flow = reformat.reformatdigfilex(fullname,
                                                                 save=False)
            digit_time_span = time_stamp[-1] - time_stamp[0]

            #digital datafrme
            dataframe2 = pd.DataFrame(time_stamp, columns=['timestamp'])
            dataframe2['state'] = switch
            dataframe2['flow'] = flow
            array_length2 = len(dataframe2['timestamp'])

            Fs2 = int(
                array_length2 /
                digit_time_span)  # can be calculated based on time stamp.
            rng2 = pd.date_range(dataframe2['timestamp'][0],
                                 periods=array_length2,
                                 freq=str(int(1000000 / Fs2)) + 'u')
            dataframe2.index = rng2

            #reduction of size. timewindow
            df_reduced = dataframe2.resample(str(int(timewindow)) + 's',
                                             how='mean')
            if first:
                combined_df = df_reduced
                first = False
            else:
                combined_df = pd.concat([combined_df, df_reduced])
        except:
            print 'issue with:', filename

        print len(summary), ' ', np.sum(summary)
        img = np.concatenate((img, summary))

    a = np.int(np.sqrt(len(img)))
    b = len(img) / width
    img2 = img[0:(a * a)].reshape((a, a))
    img2 = img[0:(b * width)].reshape((b, width))
    plt.clf()
    ###    plt.axis([0,1000,0,1000])
    plt.imshow(img2)
    #plt.show()
    fname = 'digital_' + letter + '_' + time_start + '_' + time_stop + '_' + str(
        int(timewindow)) + 's' + '.png'
    output = os.path.join(target_directory, fname)
    print('Saving frame', output)
    plt.savefig(output)

    return img2, combined_df