def test_catalogue_constructor(): if os.path.exists('test_catalogueconstructor'): shutil.rmtree('test_catalogueconstructor') dataio = DataIO(dirname='test_catalogueconstructor') localdir, filenames, params = download_dataset(name='olfactory_bulb') #~ localdir, filenames, params = download_dataset(name='locust') dataio.set_data_source(type='RawData', filenames=filenames, **params) channels = range(14) #~ channels=list(range(4)) dataio.add_one_channel_group(channels=channels, chan_grp=0) cc = CatalogueConstructor(dataio=dataio) for memory_mode in ['ram', 'memmap']: for mode in ['dense', 'sparse']: print('*' * 5) print('memory_mode', memory_mode, 'mode', mode) if mode == 'dense': peak_engine = 'numpy' peak_method = 'global' adjacency_radius_um = None elif mode == 'sparse': peak_engine = 'numpy' peak_method = 'geometrical' adjacency_radius_um = 450. cc.set_global_params( chunksize=1024, memory_mode=memory_mode, mode=mode, n_jobs=1, #~ adjacency_radius_um=adjacency_radius_um, ) cc.set_preprocessor_params( #signal preprocessor highpass_freq=300, lowpass_freq=5000., common_ref_removal=False, smooth_size=0, lostfront_chunksize=None) cc.set_peak_detector_params( #peak detector method=peak_method, engine=peak_engine, peak_sign='-', relative_threshold=7, peak_span_ms=0.5, adjacency_radius_um=adjacency_radius_um, ) t1 = time.perf_counter() cc.estimate_signals_noise(seg_num=0, duration=10.) t2 = time.perf_counter() print('estimate_signals_noise', t2 - t1) t1 = time.perf_counter() cc.run_signalprocessor(duration=10., detect_peak=True) t2 = time.perf_counter() print('run_signalprocessor_loop', t2 - t1) for seg_num in range(dataio.nb_segment): mask = cc.all_peaks['segment'] == seg_num print('seg_num', seg_num, 'nb peak', np.sum(mask)) # redetect peak cc.re_detect_peak(method=peak_method, engine=peak_engine, peak_sign='-', relative_threshold=5, peak_span_ms=0.7, adjacency_radius_um=adjacency_radius_um) for seg_num in range(dataio.nb_segment): mask = cc.all_peaks['segment'] == seg_num print('seg_num', seg_num, 'nb peak', np.sum(mask)) cc.set_waveform_extractor_params(n_left=-25, n_right=40) t1 = time.perf_counter() cc.clean_peaks(alien_value_threshold=100, mode='extremum_amplitude') t2 = time.perf_counter() print('clean_peaks extremum_amplitude', t2 - t1) t1 = time.perf_counter() cc.clean_peaks(alien_value_threshold=100, mode='full_waveform') t2 = time.perf_counter() print('clean_peaks full_waveforms', t2 - t1) t1 = time.perf_counter() cc.sample_some_peaks(mode='rand', nb_max=5000) t2 = time.perf_counter() print('sample_some_peaks', t2 - t1) print(cc) #extract_some_noise t1 = time.perf_counter() cc.extract_some_noise(nb_snippet=400) t2 = time.perf_counter() print('extract_some_noise', t2 - t1) if mode == 'dense': # PCA t1 = time.perf_counter() cc.extract_some_features(method='global_pca', n_components=12) t2 = time.perf_counter() print('project pca', t2 - t1) # cluster t1 = time.perf_counter() cc.find_clusters(method='kmeans', n_clusters=11) t2 = time.perf_counter() print('find_clusters', t2 - t1) elif mode == 'sparse': # PCA t1 = time.perf_counter() cc.extract_some_features(method='pca_by_channel', n_components_by_channel=3) t2 = time.perf_counter() print('project pca', t2 - t1) # cluster t1 = time.perf_counter() cc.find_clusters(method='pruningshears') t2 = time.perf_counter() print('find_clusters', t2 - t1) print(cc) t1 = time.perf_counter() cc.auto_split_cluster() t2 = time.perf_counter() print('auto_split_cluster', t2 - t1) t1 = time.perf_counter() cc.trash_not_aligned() t2 = time.perf_counter() print('trash_not_aligned', t2 - t1) t1 = time.perf_counter() cc.auto_merge_cluster() t2 = time.perf_counter() print('auto_merge_cluster', t2 - t1) t1 = time.perf_counter() cc.trash_low_extremum() t2 = time.perf_counter() print('trash_low_extremum', t2 - t1) t1 = time.perf_counter() cc.trash_small_cluster() t2 = time.perf_counter() print('trash_small_cluster', t2 - t1)
def test_catalogue_constructor(): if os.path.exists('test_catalogueconstructor'): shutil.rmtree('test_catalogueconstructor') dataio = DataIO(dirname='test_catalogueconstructor') localdir, filenames, params = download_dataset(name='olfactory_bulb') #~ localdir, filenames, params = download_dataset(name='locust') dataio.set_data_source(type='RawData', filenames=filenames, **params) channels = range(14) #~ channels=list(range(4)) dataio.add_one_channel_group(channels=channels, chan_grp=0) catalogueconstructor = CatalogueConstructor(dataio=dataio) for memory_mode in ['ram', 'memmap']: #~ for memory_mode in ['memmap']: print() print(memory_mode) catalogueconstructor.set_preprocessor_params( chunksize=1024, memory_mode=memory_mode, #signal preprocessor highpass_freq=300, lowpass_freq=5000., common_ref_removal=False, smooth_size=0, lostfront_chunksize=128, #peak detector peakdetector_engine='numpy', peak_sign='-', relative_threshold=7, peak_span=0.0005, #waveformextractor #~ n_left=-20, n_right=30, ) t1 = time.perf_counter() catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.) t2 = time.perf_counter() print('estimate_signals_noise', t2 - t1) #~ t1 = time.perf_counter() #~ for seg_num in range(dataio.nb_segment): #~ print('seg_num', seg_num) #~ catalogueconstructor.run_signalprocessor_loop_one_segment(seg_num=seg_num, duration=10.) catalogueconstructor.run_signalprocessor(duration=10., detect_peak=True) t2 = time.perf_counter() print('run_signalprocessor_loop', t2 - t1) for seg_num in range(dataio.nb_segment): mask = catalogueconstructor.all_peaks['segment'] == seg_num print('seg_num', seg_num, 'nb peak', np.sum(mask)) #redetect peak catalogueconstructor.re_detect_peak(peakdetector_engine='numpy', peak_sign='-', relative_threshold=5, peak_span=0.0002) for seg_num in range(dataio.nb_segment): mask = catalogueconstructor.all_peaks['segment'] == seg_num print('seg_num', seg_num, 'nb peak', np.sum(mask)) t1 = time.perf_counter() catalogueconstructor.extract_some_waveforms(n_left=-25, n_right=40, mode='rand', nb_max=5000) t2 = time.perf_counter() print('extract_some_waveforms rand', t2 - t1) print(catalogueconstructor.some_waveforms.shape) t1 = time.perf_counter() catalogueconstructor.find_good_limits() t2 = time.perf_counter() print('find_good_limits', t2 - t1) print(catalogueconstructor.some_waveforms.shape) t1 = time.perf_counter() catalogueconstructor.extract_some_waveforms(n_left=None, n_right=None, mode='rand', nb_max=5000) t2 = time.perf_counter() print('extract_some_waveforms rand', t2 - t1) print(catalogueconstructor.some_waveforms.shape) t1 = time.perf_counter() catalogueconstructor.clean_waveforms(alien_value_threshold=60.) t2 = time.perf_counter() print('clean_waveforms', t2 - t1) print(catalogueconstructor) #extract_some_noise t1 = time.perf_counter() catalogueconstructor.extract_some_noise(nb_snippet=400) t2 = time.perf_counter() print('extract_some_noise', t2 - t1) # PCA t1 = time.perf_counter() catalogueconstructor.project(method='global_pca', n_components=7, batch_size=16384) t2 = time.perf_counter() print('project pca', t2 - t1) # peak_max #~ t1 = time.perf_counter() #~ catalogueconstructor.project(method='peak_max') #~ t2 = time.perf_counter() #~ print('project peak_max', t2-t1) #~ print(catalogueconstructor.some_features.shape) #~ t1 = time.perf_counter() #~ catalogueconstructor.extract_some_waveforms(index=np.arange(1000)) #~ t2 = time.perf_counter() #~ print('extract_some_waveforms others', t2-t1) #~ print(catalogueconstructor.some_waveforms.shape) # cluster t1 = time.perf_counter() catalogueconstructor.find_clusters(method='kmeans', n_clusters=11) t2 = time.perf_counter() print('find_clusters', t2 - t1) print(catalogueconstructor)