예제 #1
0
def get_normed_sigs(chunksize=None):
    # get sigs
    sigs, sample_rate = get_dataset(name='olfactory_bulb')
    #~ sigs = np.tile(sigs, (1, 20)) #for testing large channels num

    if sigs.shape[0] % chunksize > 0:
        sigs = sigs[:-(sigs.shape[0] % chunksize), :]

    nb_channel = sigs.shape[1]
    #~ print('nb_channel', nb_channel)

    geometry = np.zeros((nb_channel, 2))
    geometry[:, 0] = np.arange(nb_channel) * 50  # um spacing

    # normalize sigs
    highpass_freq = 300.
    preprocess_params = dict(highpass_freq=highpass_freq,
                             common_ref_removal=True,
                             backward_chunksize=chunksize + chunksize // 4,
                             output_dtype='float32')
    normed_sigs = offline_signal_preprocessor(sigs, sample_rate,
                                              **preprocess_params)

    return sigs, sample_rate, normed_sigs, geometry
예제 #2
0
def test_compare_offline_online_engines():
    #~ HAVE_PYOPENCL = True
    if HAVE_PYOPENCL:
        engines = ['numpy', 'opencl']
        #~ engines = [ 'opencl']
        #~ engines = ['numpy']
    else:
        engines = ['numpy']

    # get sigs
    sigs, sample_rate = get_dataset(name='olfactory_bulb')
    #~ sigs = np.tile(sigs, (1, 20)) #for testing large channels num

    nb_channel = sigs.shape[1]
    print('nb_channel', nb_channel)

    #params
    chunksize = 1024
    peak_sign = '-'
    relative_threshold = 8
    peak_span = 0.0009

    #~ print('n_span', n_span)
    nloop = sigs.shape[0] // chunksize
    sigs = sigs[:chunksize * nloop]

    print('sig duration', sigs.shape[0] / sample_rate)

    # normalize sigs
    highpass_freq = 300.
    preprocess_params = dict(highpass_freq=highpass_freq,
                             common_ref_removal=True,
                             backward_chunksize=chunksize + chunksize // 4,
                             output_dtype='float32')
    normed_sigs = offline_signal_preprocessor(sigs, sample_rate,
                                              **preprocess_params)

    for peak_sign in [
            '-',
            '+',
    ]:
        #~ for peak_sign in ['+', ]:
        #~ for peak_sign in ['-', ]:
        print()
        print('peak_sign', peak_sign)
        if peak_sign == '-':
            sigs = normed_sigs
        elif peak_sign == '+':
            sigs = -normed_sigs

        #~ print(sigs.shape)
        #~ print('nloop', nloop)

        t1 = time.perf_counter()
        offline_peaks, rectified_sum = offline_peak_detect(
            sigs,
            sample_rate,
            peak_sign=peak_sign,
            relative_threshold=relative_threshold,
            peak_span=peak_span)
        t2 = time.perf_counter()
        print('offline', 'process time', t2 - t1)
        #~ print(offline_peaks)

        online_peaks = {}
        for engine in engines:
            print(engine)
            EngineClass = peakdetector_engines[engine]
            #~ buffer_size = chunksize*4
            peakdetector_engine = EngineClass(sample_rate, nb_channel,
                                              chunksize, 'float32')

            peakdetector_engine.change_params(
                peak_sign=peak_sign,
                relative_threshold=relative_threshold,
                peak_span=peak_span)

            all_online_peaks = []
            t1 = time.perf_counter()
            for i in range(nloop):
                #~ print(i)
                pos = (i + 1) * chunksize
                chunk = sigs[pos - chunksize:pos, :]
                n_peaks, chunk_peaks = peakdetector_engine.process_data(
                    pos, chunk)
                if chunk_peaks is not None:
                    #~ all_online_peaks.append(chunk_peaks['index'])
                    all_online_peaks.append(chunk_peaks)
            online_peaks[engine] = np.concatenate(all_online_peaks)
            t2 = time.perf_counter()
            print(engine, 'process time', t2 - t1)

        # remove peaks on border for comparison
        offline_peaks = offline_peaks[(offline_peaks > chunksize) & (
            offline_peaks < sigs.shape[0] - chunksize)]
        for engine in engines:
            onlinepeaks = online_peaks[engine]
            onlinepeaks = onlinepeaks[(onlinepeaks > chunksize) & (
                onlinepeaks < sigs.shape[0] - chunksize)]
            online_peaks[engine] = onlinepeaks

        # compare
        for engine in engines:
            onlinepeaks = online_peaks[engine]
            assert offline_peaks.size == onlinepeaks.size, '{} nb_peak{} instead {}'.format(
                engine, offline_peaks.size, onlinepeaks.size)
            assert np.array_equal(offline_peaks, onlinepeaks)
예제 #3
0
def test_compare_offline_online_engines():
    HAVE_PYOPENCL = True
    if HAVE_PYOPENCL:
        engines = ['numpy', 'opencl']
        #~ engines = [ 'opencl']
        #~ engines = ['numpy']
    else:
        engines = ['numpy']

    # get sigs
    sigs, sample_rate = get_dataset(name='olfactory_bulb')
    #~ sigs = np.tile(sigs, (1, 20)) #for testing large channels num
    
    nb_channel = sigs.shape[1]
    print('nb_channel', nb_channel)

    
    
    #params
    chunksize = 1024
    peak_sign = '-'
    relative_threshold = 8
    peak_span = 0.0009
    
    #~ print('n_span', n_span)
    nloop = sigs.shape[0]//chunksize
    sigs = sigs[:chunksize*nloop]
    
    print('sig duration', sigs.shape[0]/sample_rate)
    
    # normalize sigs
    highpass_freq = 300.
    preprocess_params = dict(
                highpass_freq=highpass_freq,
                common_ref_removal=True,
                backward_chunksize=chunksize+chunksize//4,
                output_dtype='float32')
    normed_sigs = offline_signal_preprocessor(sigs, sample_rate, **preprocess_params)
    
    
    
    for peak_sign in ['-', '+', ]:
    #~ for peak_sign in ['+', ]:
    #~ for peak_sign in ['-', ]:
        print()
        print('peak_sign', peak_sign)
        if peak_sign=='-':
            sigs = normed_sigs
        elif peak_sign=='+':
            sigs = -normed_sigs
        
        #~ print(sigs.shape)
        #~ print('nloop', nloop)
        
        
        t1 = time.perf_counter()
        offline_peaks, rectified_sum = offline_peak_detect(sigs, sample_rate, peak_sign=peak_sign, relative_threshold=relative_threshold, peak_span=peak_span)
        t2 = time.perf_counter()
        print('offline', 'process time', t2-t1)
        #~ print(offline_peaks)
        
        online_peaks = {}
        for engine in engines:
            print(engine)
            EngineClass = peakdetector_engines[engine]
            #~ buffer_size = chunksize*4
            peakdetector_engine = EngineClass(sample_rate, nb_channel, chunksize, 'float32')
            
            peakdetector_engine.change_params(peak_sign=peak_sign, relative_threshold=relative_threshold,
                            peak_span=peak_span)
            
            all_online_peaks = []
            t1 = time.perf_counter()
            for i in range(nloop):
                #~ print(i)
                pos = (i+1)*chunksize
                chunk = sigs[pos-chunksize:pos,:]
                n_peaks, chunk_peaks = peakdetector_engine.process_data(pos, chunk)
                if chunk_peaks is not None:
                    #~ all_online_peaks.append(chunk_peaks['index'])
                    all_online_peaks.append(chunk_peaks)
            online_peaks[engine] = np.concatenate(all_online_peaks)
            t2 = time.perf_counter()
            print(engine, 'process time', t2-t1)
        
        # remove peaks on border for comparison
        offline_peaks = offline_peaks[(offline_peaks>chunksize) & (offline_peaks<sigs.shape[0]-chunksize)]
        for engine in engines:
            onlinepeaks = online_peaks[engine]
            onlinepeaks = onlinepeaks[(onlinepeaks>chunksize) & (onlinepeaks<sigs.shape[0]-chunksize)]
            online_peaks[engine] = onlinepeaks

        # compare
        for engine in engines:
            onlinepeaks = online_peaks[engine]
            assert offline_peaks.size==onlinepeaks.size, '{} nb_peak{} instead {}'.format(engine,  offline_peaks.size, onlinepeaks.size)
            assert np.array_equal(offline_peaks, onlinepeaks)
def test_compare_offline_online_engines():

    sigs, sample_rate = get_dataset()
    #~ sigs = sigs[:, [0]]
    nb_channel = sigs.shape[1]
    print('nb_channel', nb_channel)

    #params
    chunksize = 1024
    nloop = sigs.shape[0]//chunksize
    sigs = sigs[:chunksize*nloop]
    highpass_freq = 300.
    preprocess_params = dict(
                highpass_freq=highpass_freq,
                common_ref_removal=True,
                backward_chunksize=chunksize+chunksize//4,
                output_dtype='float32')
    
    peak_params = dict(peak_sign='-',
                                    relative_threshold=8,
                                    peak_span = 0.0009)
    
    waveforms_params = dict(n_left=-20, n_right=30)
    
    n_left = -20
    n_right = 30
    
    t1 = time.perf_counter()
    offline_sig = offline_signal_preprocessor(sigs, sample_rate, **preprocess_params)
    offline_peaks, rectified_sum = offline_peak_detect(offline_sig, sample_rate, **peak_params)
    keep = (offline_peaks>chunksize) & (offline_peaks<sigs.shape[0]-chunksize)
    offline_peaks = offline_peaks[keep]
    offline_waveforms = cut_full(offline_sig, offline_peaks+n_left, n_right-n_left)
    print(offline_waveforms.shape)
    t2 = time.perf_counter()
    print('offline', 'process time', t2-t1)
    
    # precompute medians and mads
    params2 = dict(preprocess_params)
    params2['normalize'] = False
    sigs_for_noise = offline_signal_preprocessor(sigs, sample_rate, **params2)
    medians = np.median(sigs_for_noise, axis=0)
    mads = np.median(np.abs(sigs_for_noise-medians),axis=0)*1.4826
    preprocess_params['signals_medians'] = medians
    preprocess_params['signals_mads'] = mads
    #
    
    
    signalpreprocessor = SignalPreprocessor_Numpy(sample_rate, nb_channel, chunksize, sigs.dtype)
    signalpreprocessor.change_params(**preprocess_params)
    
    peakdetector = PeakDetectorEngine_Numpy(sample_rate, nb_channel, chunksize, 'float32')
    peakdetector.change_params(**peak_params)
            
    waveformextractor = OnlineWaveformExtractor(nb_channel, chunksize)
    waveformextractor.change_params(**waveforms_params)
    

    all_online_peak = []
    all_online_waveforms = []
    
    t1 = time.perf_counter()
    for i in range(nloop):
        #~ print()
        pos = (i+1)*chunksize
        #~ print('loop', i, 'pos', pos-chunksize, pos)
        
        chunk = sigs[pos-chunksize:pos,:]
        
        pos2, preprocessed_chunk = signalpreprocessor.process_data(pos, chunk)
        if preprocessed_chunk is  None:
            continue
        
        #~ print('pos2', pos)
        
        n_peaks, chunk_peaks = peakdetector.process_data(pos2, preprocessed_chunk)
        if chunk_peaks is  None:
            continue
        
        for peak_pos, chunk_waveforms in waveformextractor.new_peaks(pos2, preprocessed_chunk, chunk_peaks):
            #~ print(peak_pos, chunk_waveforms.shape)
            all_online_peak.append(peak_pos)
            all_online_waveforms.append(chunk_waveforms)
            
    t2 = time.perf_counter()
    print('online process time', t2-t1)
    
    online_peaks = np.concatenate(all_online_peak, axis=0)
    online_waveforms = np.concatenate(all_online_waveforms, axis=0)
    
    keep = (online_peaks>chunksize) & (online_peaks<sigs.shape[0]-chunksize)
    online_peaks = online_peaks[keep]
    online_waveforms = online_waveforms[keep]
    
    assert np.array_equal(offline_peaks, online_peaks)
    

    residual = np.abs((online_waveforms.astype('float64')-offline_waveforms.astype('float64'))/np.mean(np.abs(offline_waveforms.astype('float64'))))

    print(np.max(residual))
    #~ print(np.mean(np.abs(offline_sig.astype('float64'))))
    assert np.max(residual)<5e-5, 'online differt from offline'

    
    
    ind_error_max = np.argmax(np.max(residual.reshape(residual.shape[0], -1), axis=1))
    #~ print(ind_error_max)
    
    offline_wf = offline_waveforms[ind_error_max, : , :]
    online_wf = online_waveforms[ind_error_max, : , :]
    #~ print(online_wf.shape)
    
    fig, ax = pyplot.subplots()
    ax.plot(offline_wf.flatten(), color='g')
    ax.plot(online_wf.flatten(), color='r', ls='--')
    
    
    fig, ax = pyplot.subplots()
    wf2 = offline_waveforms.reshape(offline_waveforms.shape[0], -1)
    ax.plot(np.median(wf2, axis=0), color='g')
    wf3 = online_waveforms.reshape(offline_waveforms.shape[0], -1)
    ax.plot(np.median(wf3, axis=0), color='r', ls='--')
    
    
    pyplot.show()