def test_auto_lostfront_chunksize(): sigs, sample_rate = get_dataset(name='olfactory_bulb') sigs, sample_rate = get_dataset(name='olfactory_bulb') chunksize = 1024 nloop = sigs.shape[0] // chunksize sigs = sigs[:chunksize * nloop] params = { 'common_ref_removal': False, 'highpass_freq': 300., 'lowpass_freq': 4000., 'smooth_size': 0, 'output_dtype': 'float32', 'normalize': True, 'lostfront_chunksize': None, } offline_sig = offline_signal_preprocessor(sigs, sample_rate, **params) online_sig = run_online('numpy', sigs, sample_rate, chunksize, **params) min_size = online_sig.shape[0] offline_sig = offline_sig[chunksize:min_size] online_sig = online_sig[chunksize:min_size] residual = np.abs( online_sig.astype('float64') - offline_sig.astype('float64')) residual_normed = residual / np.mean(np.abs(offline_sig.astype('float64'))) print(' max residual_normed', np.max(residual_normed))
def run_online(engine, sigs, sample_rate, chunksize, **params): sigs, sample_rate = get_dataset(name='olfactory_bulb') nb_channel = sigs.shape[1] # precompute medians and mads params2 = dict(params) params2['normalize'] = False sigs_for_noise = offline_signal_preprocessor(sigs, sample_rate, **params2) medians = np.median(sigs_for_noise, axis=0) mads = np.median(np.abs(sigs_for_noise - medians), axis=0) * 1.4826 params['signals_medians'] = medians params['signals_mads'] = mads SignalPreprocessorClass = signalpreprocessor_engines[engine] signalpreprocessor = SignalPreprocessorClass(sample_rate, nb_channel, chunksize, sigs.dtype) signalpreprocessor.change_params(**params) nloop = sigs.shape[0] // chunksize all_online_sigs = [] t1 = time.perf_counter() for i in range(nloop): #~ print(i) pos = (i + 1) * chunksize chunk = sigs[pos - chunksize:pos, :] pos2, preprocessed_chunk = signalpreprocessor.process_data(pos, chunk) if preprocessed_chunk is not None: #~ print(preprocessed_chunk) all_online_sigs.append(preprocessed_chunk) online_sig = np.concatenate(all_online_sigs) t2 = time.perf_counter() print(engine, 'process time', t2 - t1) return online_sig
def explore_lostfront_chunksize(): sigs, sample_rate = get_dataset(name='olfactory_bulb') chunksize = 1024 nloop = sigs.shape[0] // chunksize sigs = sigs[:chunksize * nloop] params = { 'common_ref_removal': False, 'highpass_freq': 300., 'lowpass_freq': 4000., 'smooth_size': 0, 'output_dtype': 'float32', 'normalize': True, #~ 'lostfront_chunksize': 150 } offline_sig = offline_signal_preprocessor(sigs, sample_rate, **params) lostfront_chunksizes = [ int(sample_rate / params['highpass_freq']) * i for i in range(1, 5) ] online_sigs = {} for lostfront_chunksize in lostfront_chunksizes: print('lostfront_chunksize', lostfront_chunksize) params['lostfront_chunksize'] = lostfront_chunksize online_sigs[lostfront_chunksize] = run_online('numpy', sigs, sample_rate, chunksize, **params) # remove border for comparison min_size = min([ online_sigs[lostfront_chunksize].shape[0] for lostfront_chunksize in lostfront_chunksizes ]) offline_sig = offline_sig[chunksize:min_size] for lostfront_chunksize in lostfront_chunksizes: online_sig = online_sigs[lostfront_chunksize] online_sigs[lostfront_chunksize] = online_sig[chunksize:min_size] for lostfront_chunksize in lostfront_chunksizes: print('lostfront_chunksize', lostfront_chunksize) online_sig = online_sigs[lostfront_chunksize] residual = np.abs( online_sig.astype('float64') - offline_sig.astype('float64')) residual_normed = residual / np.mean( np.abs(offline_sig.astype('float64'))) print(' max residual_normed', np.max(residual_normed))
def test_compare_offline_online_engines(): #~ HAVE_PYOPENCL = True if HAVE_PYOPENCL: engines = ['numpy', 'opencl'] #~ engines = [ 'opencl'] #~ engines = ['numpy'] else: engines = ['numpy'] # get sigs sigs, sample_rate = get_dataset(name='olfactory_bulb') print(sample_rate) #~ sigs = np.tile(sigs, (1, 20)) #for testing large channels num nb_channel = sigs.shape[1] print('nb_channel', nb_channel) #params chunksize = 1024 nloop = sigs.shape[0] // chunksize sigs = sigs[:chunksize * nloop] print('sig duration', sigs.shape[0] / sample_rate) highpass_freq = 500. #~ highpass_freq = 300. #~ highpass_freq = 100. #~ highpass_freq = None lowpass_freq = 4000. #~ lowpass_freq = None #~ smooth_kernel = True smooth_size = 0 #~ lostfront_chunksize = int(sample_rate/highpass_freq)*4 #~ print('lostfront_chunksize', lostfront_chunksize) #~ exit() params = { 'common_ref_removal': False, 'highpass_freq': highpass_freq, 'lowpass_freq': lowpass_freq, 'smooth_size': smooth_size, 'output_dtype': 'float32', 'normalize': True, 'lostfront_chunksize': 150 } t1 = time.perf_counter() offline_sig = offline_signal_preprocessor(sigs, sample_rate, **params) t2 = time.perf_counter() print('offline', 'process time', t2 - t1) online_sigs = {} for engine in engines: online_sigs[engine] = run_online(engine, sigs, sample_rate, chunksize, **params) # remove border for comparison min_size = min([online_sigs[engine].shape[0] for engine in engines]) offline_sig = offline_sig[chunksize:min_size] for engine in engines: online_sig = online_sigs[engine] online_sigs[engine] = online_sig[chunksize:min_size] # compare for engine in engines: print(engine) online_sig = online_sigs[engine] residual = np.abs( online_sig.astype('float64') - offline_sig.astype('float64')) print('max residual', np.max(residual)) print('/', np.mean(np.abs(offline_sig.astype('float64')))) residual_normed = residual / np.mean( np.abs(offline_sig.astype('float64'))) print('max residual_normed', np.max(residual_normed)) # plot # fig, axs = pyplot.subplots(nrows=nb_channel, sharex=True) #~ fig, axs = pyplot.subplots(nrows=4, sharex=True) # for i in range(nb_channel): #~ for i in range(4): #~ ax = axs[i] #~ ax.plot(residual_normed[:, i], color = 'k') #~ ax.plot(residual[:, i], color = 'k') #~ ax.plot(offline_sig[:, i], color = 'g') #~ ax.plot(online_sig[:, i], color = 'r', ls='--') #~ for i in range(nloop): #~ ax.axvline(i*chunksize, color='k', alpha=0.4) #~ pyplot.show() print(np.max(residual_normed)) #~ print(np.mean(np.abs(offline_sig.astype('float64')))) assert np.max( residual_normed) < 0.05, 'online differt from offline more than 5%'
def test_compare_offline_online_engines(): HAVE_PYOPENCL = True if HAVE_PYOPENCL: engines = ['numpy', 'opencl'] #~ engines = [ 'opencl'] #~ engines = ['numpy'] else: engines = ['numpy'] # get sigs sigs, sample_rate = get_dataset(name='olfactory_bulb') #~ sigs = np.tile(sigs, (1, 20)) #for testing large channels num nb_channel = sigs.shape[1] print('nb_channel', nb_channel) #params chunksize = 1024 nloop = sigs.shape[0]//chunksize sigs = sigs[:chunksize*nloop] print('sig duration', sigs.shape[0]/sample_rate) highpass_freq = 300. highpass_freq = None lowpass_freq = 4000. #~ lowpass_freq = None #~ smooth_kernel = True smooth_size = 0 params = { 'common_ref_removal' : False, 'highpass_freq': highpass_freq, 'lowpass_freq': lowpass_freq, 'smooth_size':smooth_size, 'output_dtype': 'float32', 'normalize' : True, 'lostfront_chunksize': 128} t1 = time.perf_counter() offline_sig = offline_signal_preprocessor(sigs, sample_rate, **params) t2 = time.perf_counter() print('offline', 'process time', t2-t1) # precompute medians and mads params2 = dict(params) params2['normalize'] = False sigs_for_noise = offline_signal_preprocessor(sigs, sample_rate, **params2) medians = np.median(sigs_for_noise, axis=0) mads = np.median(np.abs(sigs_for_noise-medians),axis=0)*1.4826 params['signals_medians'] = medians params['signals_mads'] = mads online_sigs = {} for engine in engines: print(engine) SignalPreprocessorClass = signalpreprocessor_engines[engine] signalpreprocessor = SignalPreprocessorClass(sample_rate, nb_channel, chunksize, sigs.dtype) signalpreprocessor.change_params(**params) all_online_sigs = [] t1 = time.perf_counter() for i in range(nloop): #~ print(i) pos = (i+1)*chunksize chunk = sigs[pos-chunksize:pos,:] pos2, preprocessed_chunk = signalpreprocessor.process_data(pos, chunk) if preprocessed_chunk is not None: #~ print(preprocessed_chunk) all_online_sigs.append(preprocessed_chunk) online_sigs[engine] = np.concatenate(all_online_sigs) t2 = time.perf_counter() print(engine, 'process time', t2-t1) # remove border for comparison min_size = min([online_sigs[engine].shape[0] for engine in engines]) offline_sig = offline_sig[chunksize:min_size] for engine in engines: online_sig = online_sigs[engine] online_sigs[engine] = online_sig[chunksize:min_size] # compare for engine in engines: print(engine) online_sig = online_sigs[engine] residual = np.abs((online_sig.astype('float64')-offline_sig.astype('float64'))/np.mean(np.abs(offline_sig.astype('float64')))) # plot #~ # fig, axs = pyplot.subplots(nrows=nb_channel, sharex=True) #~ fig, axs = pyplot.subplots(nrows=4, sharex=True) #~ # for i in range(nb_channel): #~ for i in range(4): #~ ax = axs[i] #~ ax.plot(residual[:, i], color = 'k') #~ ax.plot(offline_sig[:, i], color = 'g') #~ ax.plot(online_sig[:, i], color = 'r', ls='--') #~ for i in range(nloop): #~ ax.axvline(i*chunksize, color='k', alpha=0.4) #~ pyplot.show() print(np.max(residual)) #~ print(np.mean(np.abs(offline_sig.astype('float64')))) assert np.max(residual)<7e-5, 'online differt from offline'