def test_pruningshears(): dirname = 'test_cluster' dataio = DataIO(dirname=dirname) print(dataio) cc = CatalogueConstructor(dataio=dataio) #~ print(cc.mode) #~ exit() #~ cc.extract_some_features(method='pca_by_channel') #~ print(dataio) #~ print(cc) if dataset_name == 'olfactory_bulb': kargs = dict(adjacency_radius_um = 420) else: kargs = {} t0 = time.perf_counter() #~ cc.find_clusters(method='pruningshears', print_debug=True) #~ cc.find_clusters(method='pruningshears', print_debug=True, debug_plot=True, **kargs) cc.find_clusters(method='pruningshears', print_debug=False, debug_plot=False, **kargs) t1 = time.perf_counter() print('cluster', t1-t0) if __name__ == '__main__': app = mkQApp() win = CatalogueWindow(cc) win.show() app.exec_()
def test_make_catalogue(): dataio = DataIO(dirname='test_catalogueconstructor') cc = CatalogueConstructor(dataio=dataio) #~ cc.make_catalogue() cc.make_catalogue_for_peeler()
def debug_interp_centers0(): dataio = DataIO(dirname='test_catalogueconstructor') catalogueconstructor = CatalogueConstructor(dataio=dataio) catalogue = catalogueconstructor.make_catalogue() centers = catalogue['centers0'] interp_centers = catalogue['interp_centers0'] subsample_ratio = catalogue['subsample_ratio']
def test_sawchaincut(): #~ dirname = 'test_catalogueconstructor' #~ dirname = '/home/samuel/Documents/projet/tridesclous/example/tridesclous_locust/' #~ dirname = '/home/samuel/Documents/projet/DataSpikeSorting/GT 252/tdc_20170623_patch1/' #~ dirname = '/home/samuel/Documents/projet/tridesclous/example/tridesclous_locust/' #~ dirname = '/home/samuel/Documents/projet/tridesclous/example/tridesclous_olfactory_bulb/' #~ dirname = '/home/samuel/Documents/projet/tridesclous/example/tridesclous_olfactory_bulb/' #~ dirname = '/home/samuel/Documents/projet/DataSpikeSorting/kampff/tdc_2015_09_03_Cell9.0/' #~ dirname = '/home/samuel/Documents/projet/DataSpikeSorting/spikesortingtest/tdc_silico_0/' #~ dirname = '/home/samuel/Documents/projet/tridesclous/example/tridesclous_purkinje/' dirname = 'test_cluster' dataio = DataIO(dirname=dirname) cc = CatalogueConstructor(dataio=dataio) #~ print(dataio) #~ print(cc) t0 = time.perf_counter() cc.find_clusters(method='sawchaincut', print_debug=True) t1 = time.perf_counter() print('cluster', t1 - t0) #~ exit() #~ print(cc) if __name__ == '__main__': app = mkQApp() win = CatalogueWindow(cc) win.show() app.exec_()
def open_PeelerWindow(): dataio = DataIO(dirname='test_peeler') catalogueconstructor = CatalogueConstructor(dataio=dataio) initial_catalogue = catalogueconstructor.load_catalogue() app = pg.mkQApp() win = PeelerWindow(dataio=dataio, catalogue=initial_catalogue) win.show() app.exec_()
def test_feature_with_lda_selection(): dataio = DataIO(dirname='test_catalogueconstructor') cc = CatalogueConstructor(dataio=dataio) print(cc) selection = np.in1d(cc.all_peaks['cluster_label'], [1, 2, 3, 4]) print(np.sum(selection), '/', cc.all_peaks.size) cc.extract_some_features(method='global_lda', selection=selection) print(cc.some_features.shape) print(cc.some_features)
def test_trash_small_cluster(): dirname = 'test_cleancluster' restore_savepoint(dirname, savepoint='after_trash_low_extremum') dataio = DataIO(dirname=dirname) cc = CatalogueConstructor(dataio=dataio) t1 = time.perf_counter() cc.trash_small_cluster() t2 = time.perf_counter() print('trash_small_cluster', t2 - t1)
def test_peeler(): dataio = DataIO(dirname='test_peeler') print(dataio) catalogueconstructor = CatalogueConstructor(dataio=dataio) initial_catalogue = catalogueconstructor.load_catalogue() peeler = Peeler(dataio) peeler.change_params(catalogue=initial_catalogue, n_peel_level=2, chunksize=1024) t1 = time.perf_counter() peeler.run() t2 = time.perf_counter() print('peeler.run_loop', t2-t1)
def open_catalogue_window(): dataio = DataIO(dirname='test_peeler') catalogueconstructor = CatalogueConstructor(dataio=dataio) app = pg.mkQApp() win = CatalogueWindow(catalogueconstructor) win.show() app.exec_()
def setup_catalogue(dirname, dataset_name='olfactory_bulb'): if os.path.exists(dirname): shutil.rmtree(dirname) dataio = DataIO(dirname=dirname) localdir, filenames, params = download_dataset(name=dataset_name) dataio.set_data_source(type='RawData', filenames=filenames, **params) if dataset_name=='olfactory_bulb': channels = [5, 6, 7, 8, 9] else: channels = [0,1,2,3] dataio.add_one_channel_group(channels=channels) catalogueconstructor = CatalogueConstructor(dataio=dataio) params = { 'duration' : 60., 'preprocessor' : { 'highpass_freq' : 300., 'chunksize' : 1024, 'lostfront_chunksize' : 100, }, 'peak_detector' : { 'peak_sign' : '-', 'relative_threshold' : 7., 'peak_span_ms' : 0.5, }, 'extract_waveforms' : { 'wf_left_ms' : -2.5, 'wf_right_ms' : 4.0, 'nb_max' : 10000, }, 'clean_waveforms' : { 'alien_value_threshold' : 60., }, 'noise_snippet' : { 'nb_snippet' : 300, }, 'feature_method': 'global_pca', 'feature_kargs':{'n_components': 5}, 'cluster_method' : 'kmeans', 'cluster_kargs' : {'n_clusters': 12}, 'clean_cluster' : False, 'clean_cluster_kargs' : {}, } apply_all_catalogue_steps(catalogueconstructor, params, verbose=True) catalogueconstructor.trash_small_cluster() catalogueconstructor.order_clusters(by='waveforms_rms') catalogueconstructor.make_catalogue_for_peeler()
def compare_nb_waveforms(): if os.path.exists('test_catalogueconstructor'): shutil.rmtree('test_catalogueconstructor') dataio = DataIO(dirname='test_catalogueconstructor') localdir, filenames, params = download_dataset(name='olfactory_bulb') dataio.set_data_source(type='RawData', filenames=filenames, **params) dataio.set_manual_channel_group(range(14)) catalogueconstructor = CatalogueConstructor(dataio=dataio) catalogueconstructor.set_preprocessor_params(chunksize=1024, #signal preprocessor highpass_freq=300, backward_chunksize=1280, #peak detector peak_sign='-', relative_threshold=7, peak_span=0.0005, ) t1 = time.perf_counter() catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.) t2 = time.perf_counter() print('estimate_signals_noise', t2-t1) t1 = time.perf_counter() catalogueconstructor.run_signalprocessor() t2 = time.perf_counter() print('run_signalprocessor', t2-t1) print(catalogueconstructor) fig, axs = pyplot.subplots(nrows=2) colors = ['r', 'g', 'b'] for i, nb_max in enumerate([100, 1000, 10000]): t1 = time.perf_counter() catalogueconstructor.extract_some_waveforms(n_left=-20, n_right=30, nb_max=nb_max) t2 = time.perf_counter() print('extract_some_waveforms', nb_max, t2-t1) print(catalogueconstructor.some_waveforms.shape) wf = catalogueconstructor.some_waveforms wf = wf.swapaxes(1,2).reshape(wf.shape[0], -1) axs[0].plot(np.median(wf, axis=0), color=colors[i], label='nb_max {}'.format(nb_max)) axs[1].plot(np.mean(wf, axis=0), color=colors[i], label='nb_max {}'.format(nb_max)) axs[0].legend() axs[0].set_title('median') axs[1].set_title('mean') pyplot.show()
def test_sawchaincut(): dirname = 'test_cluster' dataio = DataIO(dirname=dirname) cc = CatalogueConstructor(dataio=dataio) #~ print(dataio) #~ print(cc) t0 = time.perf_counter() cc.find_clusters(method='sawchaincut', print_debug=True) t1 = time.perf_counter() print('cluster', t1 - t0) #~ exit() #~ print(cc) if __name__ == '__main__': app = mkQApp() win = CatalogueWindow(cc) win.show() app.exec_()
def test_apply_all_catalogue_steps(): if os.path.exists('test_cataloguetools'): shutil.rmtree('test_cataloguetools') dataio = DataIO(dirname='test_cataloguetools') #~ localdir, filenames, params = download_dataset(name='olfactory_bulb') localdir, filenames, params = download_dataset(name='locust') dataio.set_data_source(type='RawData', filenames=filenames, **params) params = get_auto_params_for_catalogue(dataio) cc = CatalogueConstructor(dataio, chan_grp=0) apply_all_catalogue_steps(cc, params, verbose=True)
def test_pruningshears(): dirname = 'test_cluster' dataio = DataIO(dirname=dirname) print(dataio) cc = CatalogueConstructor(dataio=dataio) #~ cc.extract_some_features(method='pca_by_channel') #~ print(dataio) #~ print(cc) t0 = time.perf_counter() cc.find_clusters(method='pruningshears', print_debug=True) t1 = time.perf_counter() print('cluster', t1 - t0) if __name__ == '__main__': app = mkQApp() win = CatalogueWindow(cc) win.show() app.exec_()
def test_all_decomposition(): dirname = 'test_catalogueconstructor' dataio = DataIO(dirname=dirname) cc = catalogueconstructor = CatalogueConstructor(dataio=dataio) print(dataio) print(cc) methods = ['global_pca', 'pca_by_channel', 'peak_max','neighborhood_pca' ] #'neighborhood_pca', 'tsne', 'pca_by_channel_then_tsne' for method in methods: t0 = time.perf_counter() cc.extract_some_features(method=method) t1 = time.perf_counter() print('extract_some_features', method, t1-t0)
def test_plot_signals(): dataio = DataIO('test_matplotlibplot') catalogueconstructor = CatalogueConstructor(dataio=dataio, chan_grp=0) plot_signals(dataio, signal_type='initial') plot_signals(dataio, signal_type='processed') plot_signals(catalogueconstructor, signal_type='processed', with_peaks=True, time_slice=(2., 3)) plot_signals(catalogueconstructor, signal_type='processed', with_span=True, time_slice=(2., 3))
def test_auto_split(): dirname = 'test_cleancluster' restore_savepoint(dirname, savepoint='after_find_clusters') dataio = DataIO(dirname=dirname) cc = CatalogueConstructor(dataio=dataio) cc.find_clusters(method='pruningshears') print(cc) print(cc.n_jobs) t1 = time.perf_counter() cc.auto_split_cluster() t2 = time.perf_counter() print('auto_split_cluster', t2 - t1) print(cc) cc.create_savepoint(name='after_auto_split')
def test_auto_merge(): dirname = 'test_cleancluster' restore_savepoint(dirname, savepoint='after_trash_not_aligned') dataio = DataIO(dirname=dirname) cc = CatalogueConstructor(dataio=dataio) t1 = time.perf_counter() cc.auto_merge_cluster() t2 = time.perf_counter() print('auto_merge_cluster', t2 - t1) cc.create_savepoint(name='after_auto_merge_cluster')
def debug_one_decomposition(): dirname = 'test_catalogueconstructor' dataio = DataIO(dirname=dirname) cc = catalogueconstructor = CatalogueConstructor(dataio=dataio) print(dataio) print(cc) t0 = time.perf_counter() #~ cc.extract_some_features(method='global_pca', n_components=7) #~ cc.extract_some_features(method='peak_max') #~ cc.extract_some_features(method='pca_by_channel', n_components_by_channel=3) cc.extract_some_features(method='neighborhood_pca', n_components_by_neighborhood=3, radius_um=500) print(cc.channel_to_features) print(cc.channel_to_features.shape) t1 = time.perf_counter() print('extract_some_features', t1 - t0)
def test_pruningshears(): dirname = 'test_cluster' dataio = DataIO(dirname=dirname) print(dataio) cc = CatalogueConstructor(dataio=dataio) cc.extract_some_features(method='pca_by_channel') #~ print(dataio) #~ print(cc) t0 = time.perf_counter() cc.find_clusters(method='pruningshears', print_debug=True) t1 = time.perf_counter() print('cluster', t1 - t0)
def setup_catalogue(): if os.path.exists('test_peeler'): shutil.rmtree('test_peeler') dataio = DataIO(dirname='test_peeler') localdir, filenames, params = download_dataset(name='olfactory_bulb') dataio.set_data_source(type='RawData', filenames=filenames, **params) dataio.add_one_channel_group(channels=[5, 6, 7, 8, 9]) catalogueconstructor = CatalogueConstructor(dataio=dataio) fullchain_kargs = { 'duration': 60., 'preprocessor': { 'highpass_freq': 300., 'chunksize': 1024, 'lostfront_chunksize': 100, }, 'peak_detector': { 'peak_sign': '-', 'relative_threshold': 7., 'peak_span': 0.0005, #~ 'peak_span' : 0.000, }, 'extract_waveforms': { 'n_left': -25, 'n_right': 40, 'nb_max': 10000, }, 'clean_waveforms': { 'alien_value_threshold': 60., }, 'noise_snippet': { 'nb_snippet': 300, }, } apply_all_catalogue_steps(catalogueconstructor, fullchain_kargs, 'global_pca', {'n_components': 12}, 'kmeans', {'n_clusters': 12}, verbose=True) catalogueconstructor.trash_small_cluster() catalogueconstructor.make_catalogue_for_peeler()
def setup_catalogue(dirname, dataset_name='olfactory_bulb'): if os.path.exists(dirname): shutil.rmtree(dirname) dataio = DataIO(dirname=dirname) localdir, filenames, params = download_dataset(name=dataset_name) dataio.set_data_source(type='RawData', filenames=filenames, **params) if dataset_name=='olfactory_bulb': channels = [4, 5, 6, 7, 8, 9] mode = 'sparse' adjacency_radius_um = 350 peak_method = 'geometrical' peak_engine = 'numpy' feature_method = 'pca_by_channel' feature_kargs = {'n_components_by_channel': 3} else: channels = [0,1,2,3] mode = 'dense' adjacency_radius_um = None peak_method = 'global' peak_engine = 'numpy' feature_method = 'global_pca' feature_kargs = {'n_components': 5} dataio.add_one_channel_group(channels=channels) catalogueconstructor = CatalogueConstructor(dataio=dataio) params = { 'duration' : 60., 'chunksize': 1024, 'mode': mode, 'memory_mode': 'memmap', 'preprocessor' : { 'highpass_freq' : 300., 'lostfront_chunksize' : 100, 'engine' : 'numpy', }, 'peak_detector' : { 'peak_sign' : '-', 'relative_threshold' : 7., 'peak_span_ms' : 0.5, 'method' : peak_method, 'engine' : peak_engine, 'adjacency_radius_um':adjacency_radius_um, }, 'extract_waveforms' : { 'wf_left_ms' : -2.5, 'wf_right_ms' : 4.0, #~ 'nb_max' : 10000, }, 'clean_peaks' : { 'alien_value_threshold' : 60., 'mode': 'full_waveform', }, 'peak_sampler':{ 'mode': 'rand', 'nb_max' : 10000, }, 'noise_snippet' : { 'nb_snippet' : 300, }, 'feature_method': feature_method, 'feature_kargs':feature_kargs, #~ 'cluster_method' : 'kmeans', #~ 'cluster_kargs' : {'n_clusters': 12}, 'cluster_method' : 'pruningshears', 'cluster_kargs' : {}, 'clean_cluster' : False, 'clean_cluster_kargs' : {}, } #~ pprint(params) apply_all_catalogue_steps(catalogueconstructor, params, verbose=True) catalogueconstructor.make_catalogue_for_peeler()
def test_plot_waveforms(): dataio = DataIO('test_matplotlibplot') catalogueconstructor = CatalogueConstructor(dataio=dataio, chan_grp=0) plot_waveforms(catalogueconstructor)
def test_plot_features_scatter_2d(): dataio = DataIO('test_matplotlibplot') catalogueconstructor = CatalogueConstructor(dataio=dataio, chan_grp=0) #~ plot_features_scatter_2d(catalogueconstructor) plot_features_scatter_2d(catalogueconstructor, labels=[0])
def test_catalogue_constructor(): if os.path.exists('test_catalogueconstructor'): shutil.rmtree('test_catalogueconstructor') dataio = DataIO(dirname='test_catalogueconstructor') localdir, filenames, params = download_dataset(name='olfactory_bulb') dataio.set_data_source(type='RawData', filenames=filenames, **params) channels=range(14) dataio.set_manual_channel_group(channels, chan_grp=0) catalogueconstructor = CatalogueConstructor(dataio=dataio) for memory_mode in ['ram', 'memmap']: #~ for memory_mode in ['memmap']: print() print(memory_mode) catalogueconstructor.set_preprocessor_params(chunksize=1024, memory_mode=memory_mode, #signal preprocessor highpass_freq=300, backward_chunksize=1280, #~ backward_chunksize=1024*2, #peak detector peakdetector_engine='numpy', peak_sign='-', relative_threshold=7, peak_span=0.0005, #waveformextractor #~ n_left=-20, n_right=30, ) t1 = time.perf_counter() catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.) t2 = time.perf_counter() print('estimate_signals_noise', t2-t1) t1 = time.perf_counter() for seg_num in range(dataio.nb_segment): #~ print('seg_num', seg_num) catalogueconstructor.run_signalprocessor_loop_one_segment(seg_num=seg_num, duration=10.) t2 = time.perf_counter() print('run_signalprocessor_loop', t2-t1) t1 = time.perf_counter() catalogueconstructor.finalize_signalprocessor_loop() t2 = time.perf_counter() print('finalize_signalprocessor_loop', t2-t1) for seg_num in range(dataio.nb_segment): mask = catalogueconstructor.all_peaks['segment']==seg_num print('seg_num', seg_num, np.sum(mask)) t1 = time.perf_counter() catalogueconstructor.extract_some_waveforms(n_left=-25, n_right=40, mode='rand', nb_max=5000) t2 = time.perf_counter() print('extract_some_waveforms rand', t2-t1) print(catalogueconstructor.some_waveforms.shape) t1 = time.perf_counter() catalogueconstructor.find_good_limits() t2 = time.perf_counter() print('find_good_limits', t2-t1) print(catalogueconstructor.some_waveforms.shape) t1 = time.perf_counter() catalogueconstructor.extract_some_waveforms(n_left=None, n_right=None, mode='rand', nb_max=2000) t2 = time.perf_counter() print('extract_some_waveforms rand', t2-t1) print(catalogueconstructor.some_waveforms.shape) #~ break # PCA t1 = time.perf_counter() catalogueconstructor.project(method='pca', n_components=7, batch_size=16384) t2 = time.perf_counter() print('project pca', t2-t1) # peak_max #~ t1 = time.perf_counter() #~ catalogueconstructor.project(method='peak_max') #~ t2 = time.perf_counter() #~ print('project peak_max', t2-t1) #~ print(catalogueconstructor.some_features.shape) t1 = time.perf_counter() catalogueconstructor.extract_some_waveforms(index=np.arange(1000)) t2 = time.perf_counter() print('extract_some_waveforms others', t2-t1) print(catalogueconstructor.some_waveforms.shape) # cluster t1 = time.perf_counter() catalogueconstructor.find_clusters(method='kmeans', n_clusters=11) t2 = time.perf_counter() print('find_clusters', t2-t1)
def setup_catalogue(): if os.path.exists('test_peeler'): shutil.rmtree('test_peeler') dataio = DataIO(dirname='test_peeler') localdir, filenames, params = download_dataset(name='olfactory_bulb') dataio.set_data_source(type='RawData', filenames=filenames, **params) dataio.set_manual_channel_group([5, 6, 7, 8, 9]) catalogueconstructor = CatalogueConstructor(dataio=dataio) #~ print(catalogueconstructor) catalogueconstructor.set_preprocessor_params(chunksize=1024, memory_mode='memmap', #signal preprocessor highpass_freq=300, backward_chunksize=1280, #peak detector peak_sign='-', relative_threshold=7, peak_span=0.0005, ) t1 = time.perf_counter() catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.) t2 = time.perf_counter() print('estimate_signals_noise', t2-t1) t1 = time.perf_counter() catalogueconstructor.run_signalprocessor() t2 = time.perf_counter() print('run_signalprocessor', t2-t1) print(catalogueconstructor) t1 = time.perf_counter() catalogueconstructor.extract_some_waveforms(n_left=-25, n_right=40, nb_max=10000) t2 = time.perf_counter() print('extract_some_waveforms', t2-t1) t1 = time.perf_counter() n_left, n_right = catalogueconstructor.find_good_limits() t2 = time.perf_counter() print('find_good_limits', t2-t1) print(n_left, n_right) print(catalogueconstructor.some_waveforms.shape) # PCA t1 = time.perf_counter() catalogueconstructor.project(method='pca', n_components=12, batch_size=16384) t2 = time.perf_counter() print('project', t2-t1) # cluster t1 = time.perf_counter() catalogueconstructor.find_clusters(method='kmeans', n_clusters=12) t2 = time.perf_counter() print('find_clusters', t2-t1) # trash_small_cluster catalogueconstructor.trash_small_cluster()
def test_ratio_amplitude(): dataio = DataIO(dirname='test_catalogueconstructor') catalogueconstructor = CatalogueConstructor(dataio=dataio) pairs = catalogueconstructor.detect_similar_waveform_ratio(0.5) print(pairs)
def test_create_savepoint_catalogue_constructor(): dataio = DataIO(dirname='test_catalogueconstructor') catalogueconstructor = CatalogueConstructor(dataio=dataio) copy_path = catalogueconstructor.create_savepoint() print(copy_path)
def setup_catalogue(dirname, dataset_name='olfactory_bulb', duration=None): if os.path.exists(dirname): shutil.rmtree(dirname) dataio = DataIO(dirname=dirname) localdir, filenames, params = download_dataset(name=dataset_name) dataio.set_data_source(type='RawData', filenames=filenames, **params) if dataset_name=='olfactory_bulb': channels = [4, 5, 6, 7, 8, 9] mode = 'sparse' adjacency_radius_um = 350 peak_method = 'geometrical' peak_engine = 'numpy' feature_method = 'pca_by_channel' feature_kargs = {'n_components_by_channel': 3} cluster_method = 'pruningshears' cluster_kargs = {'adjacency_radius_um' : 350 } else: channels = [0,1,2,3] mode = 'dense' adjacency_radius_um = None peak_method = 'global' peak_engine = 'numpy' feature_method = 'global_pca' feature_kargs = {'n_components': 6} cluster_method = 'pruningshears' cluster_kargs = {'adjacency_radius_um' : 150 } dataio.add_one_channel_group(channels=channels) cc = CatalogueConstructor(dataio=dataio) params = get_auto_params_for_catalogue(dataio) params['mode'] = mode params['n_jobs'] = 1 params['peak_detector']['method'] = peak_method params['peak_detector']['engine'] = peak_engine params['peak_detector']['adjacency_radius_um'] = adjacency_radius_um params['feature_method'] = feature_method params['feature_kargs'] = feature_kargs params['cluster_method'] = cluster_method params['cluster_kargs'] = cluster_kargs if duration is not None: params['duration'] = duration #~ pprint(params) cc.apply_all_steps(params, verbose=True) # already done in apply_all_catalogue_steps: # cc.make_catalogue_for_peeler(inter_sample_oversampling=False, catalogue_name='initial') cc.make_catalogue_for_peeler(inter_sample_oversampling=True, catalogue_name='with_oversampling') return cc, params
def compare_nb_waveforms(): if os.path.exists('test_catalogueconstructor'): shutil.rmtree('test_catalogueconstructor') dataio = DataIO(dirname='test_catalogueconstructor') localdir, filenames, params = download_dataset(name='olfactory_bulb') dataio.set_data_source(type='RawData', filenames=filenames, **params) dataio.add_one_channel_group(channels=range(14), chan_grp=0) cc = CatalogueConstructor(dataio=dataio) cc.set_global_params( chunksize=1024, memory_mode='ram', mode='dense', n_jobs=1, #~ adjacency_radius_um=None, ) cc.set_preprocessor_params( #signal preprocessor highpass_freq=300, lowpass_freq=5000., common_ref_removal=False, smooth_size=0, lostfront_chunksize=None) cc.set_peak_detector_params( #peak detector method='global', engine='numpy', peak_sign='-', relative_threshold=7, peak_span_ms=0.5, #~ adjacency_radius_um=None, ) t1 = time.perf_counter() cc.estimate_signals_noise(seg_num=0, duration=10.) t2 = time.perf_counter() print('estimate_signals_noise', t2 - t1) t1 = time.perf_counter() cc.run_signalprocessor() t2 = time.perf_counter() print('run_signalprocessor', t2 - t1) print(cc) fig, axs = plt.subplots(nrows=2) cc.set_waveform_extractor_params(wf_left_ms=-2.0, wf_right_ms=3.0) t1 = time.perf_counter() cc.sample_some_peaks(mode='rand', nb_max=5000) t2 = time.perf_counter() print('sample_some_peaks', t2 - t1) colors = ['r', 'g', 'b', 'y'] for i, nb_max in enumerate([100, 500, 1000, 2000]): cc.sample_some_peaks(mode='rand', nb_max=nb_max) #~ catalogueconstructor.extract_some_waveforms(wf_left_ms=-2.0, wf_right_ms=3.0, nb_max=nb_max) #~ print(catalogueconstructor.some_waveforms.shape) t1 = time.perf_counter() wf = cc.get_some_waveforms() t2 = time.perf_counter() print('get_some_waveforms', nb_max, t2 - t1) #~ wf = catalogueconstructor.some_waveforms wf = wf.swapaxes(1, 2).reshape(wf.shape[0], -1) axs[0].plot(np.median(wf, axis=0), color=colors[i], label='nb_max {}'.format(nb_max)) axs[1].plot(np.mean(wf, axis=0), color=colors[i], label='nb_max {}'.format(nb_max)) axs[0].legend() axs[0].set_title('median') axs[1].set_title('mean') plt.show()
def test_catalogue_constructor(): if os.path.exists('test_catalogueconstructor'): shutil.rmtree('test_catalogueconstructor') dataio = DataIO(dirname='test_catalogueconstructor') localdir, filenames, params = download_dataset(name='olfactory_bulb') #~ localdir, filenames, params = download_dataset(name='locust') dataio.set_data_source(type='RawData', filenames=filenames, **params) channels = range(14) #~ channels=list(range(4)) dataio.add_one_channel_group(channels=channels, chan_grp=0) cc = CatalogueConstructor(dataio=dataio) for memory_mode in ['ram', 'memmap']: for mode in ['dense', 'sparse']: print('*' * 5) print('memory_mode', memory_mode, 'mode', mode) if mode == 'dense': peak_engine = 'numpy' peak_method = 'global' adjacency_radius_um = None elif mode == 'sparse': peak_engine = 'numpy' peak_method = 'geometrical' adjacency_radius_um = 450. cc.set_global_params( chunksize=1024, memory_mode=memory_mode, mode=mode, n_jobs=1, #~ adjacency_radius_um=adjacency_radius_um, ) cc.set_preprocessor_params( #signal preprocessor highpass_freq=300, lowpass_freq=5000., common_ref_removal=False, smooth_size=0, lostfront_chunksize=None) cc.set_peak_detector_params( #peak detector method=peak_method, engine=peak_engine, peak_sign='-', relative_threshold=7, peak_span_ms=0.5, adjacency_radius_um=adjacency_radius_um, ) t1 = time.perf_counter() cc.estimate_signals_noise(seg_num=0, duration=10.) t2 = time.perf_counter() print('estimate_signals_noise', t2 - t1) t1 = time.perf_counter() cc.run_signalprocessor(duration=10., detect_peak=True) t2 = time.perf_counter() print('run_signalprocessor_loop', t2 - t1) for seg_num in range(dataio.nb_segment): mask = cc.all_peaks['segment'] == seg_num print('seg_num', seg_num, 'nb peak', np.sum(mask)) # redetect peak cc.re_detect_peak(method=peak_method, engine=peak_engine, peak_sign='-', relative_threshold=5, peak_span_ms=0.7, adjacency_radius_um=adjacency_radius_um) for seg_num in range(dataio.nb_segment): mask = cc.all_peaks['segment'] == seg_num print('seg_num', seg_num, 'nb peak', np.sum(mask)) cc.set_waveform_extractor_params(n_left=-25, n_right=40) t1 = time.perf_counter() cc.clean_peaks(alien_value_threshold=100, mode='extremum_amplitude') t2 = time.perf_counter() print('clean_peaks extremum_amplitude', t2 - t1) t1 = time.perf_counter() cc.clean_peaks(alien_value_threshold=100, mode='full_waveform') t2 = time.perf_counter() print('clean_peaks full_waveforms', t2 - t1) t1 = time.perf_counter() cc.sample_some_peaks(mode='rand', nb_max=5000) t2 = time.perf_counter() print('sample_some_peaks', t2 - t1) print(cc) #extract_some_noise t1 = time.perf_counter() cc.extract_some_noise(nb_snippet=400) t2 = time.perf_counter() print('extract_some_noise', t2 - t1) if mode == 'dense': # PCA t1 = time.perf_counter() cc.extract_some_features(method='global_pca', n_components=12) t2 = time.perf_counter() print('project pca', t2 - t1) # cluster t1 = time.perf_counter() cc.find_clusters(method='kmeans', n_clusters=11) t2 = time.perf_counter() print('find_clusters', t2 - t1) elif mode == 'sparse': # PCA t1 = time.perf_counter() cc.extract_some_features(method='pca_by_channel', n_components_by_channel=3) t2 = time.perf_counter() print('project pca', t2 - t1) # cluster t1 = time.perf_counter() cc.find_clusters(method='pruningshears') t2 = time.perf_counter() print('find_clusters', t2 - t1) print(cc) t1 = time.perf_counter() cc.auto_split_cluster() t2 = time.perf_counter() print('auto_split_cluster', t2 - t1) t1 = time.perf_counter() cc.trash_not_aligned() t2 = time.perf_counter() print('trash_not_aligned', t2 - t1) t1 = time.perf_counter() cc.auto_merge_cluster() t2 = time.perf_counter() print('auto_merge_cluster', t2 - t1) t1 = time.perf_counter() cc.trash_low_extremum() t2 = time.perf_counter() print('trash_low_extremum', t2 - t1) t1 = time.perf_counter() cc.trash_small_cluster() t2 = time.perf_counter() print('trash_small_cluster', t2 - t1)
def test_catalogue_constructor(): if os.path.exists('test_catalogueconstructor'): shutil.rmtree('test_catalogueconstructor') dataio = DataIO(dirname='test_catalogueconstructor') localdir, filenames, params = download_dataset(name='olfactory_bulb') #~ localdir, filenames, params = download_dataset(name='locust') dataio.set_data_source(type='RawData', filenames=filenames, **params) channels = range(14) #~ channels=list(range(4)) dataio.add_one_channel_group(channels=channels, chan_grp=0) catalogueconstructor = CatalogueConstructor(dataio=dataio) for memory_mode in ['ram', 'memmap']: #~ for memory_mode in ['memmap']: print() print(memory_mode) catalogueconstructor.set_preprocessor_params( chunksize=1024, memory_mode=memory_mode, #signal preprocessor highpass_freq=300, lowpass_freq=5000., common_ref_removal=False, smooth_size=0, lostfront_chunksize=128, #peak detector peakdetector_engine='numpy', peak_sign='-', relative_threshold=7, peak_span=0.0005, #waveformextractor #~ n_left=-20, n_right=30, ) t1 = time.perf_counter() catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.) t2 = time.perf_counter() print('estimate_signals_noise', t2 - t1) #~ t1 = time.perf_counter() #~ for seg_num in range(dataio.nb_segment): #~ print('seg_num', seg_num) #~ catalogueconstructor.run_signalprocessor_loop_one_segment(seg_num=seg_num, duration=10.) catalogueconstructor.run_signalprocessor(duration=10., detect_peak=True) t2 = time.perf_counter() print('run_signalprocessor_loop', t2 - t1) for seg_num in range(dataio.nb_segment): mask = catalogueconstructor.all_peaks['segment'] == seg_num print('seg_num', seg_num, 'nb peak', np.sum(mask)) #redetect peak catalogueconstructor.re_detect_peak(peakdetector_engine='numpy', peak_sign='-', relative_threshold=5, peak_span=0.0002) for seg_num in range(dataio.nb_segment): mask = catalogueconstructor.all_peaks['segment'] == seg_num print('seg_num', seg_num, 'nb peak', np.sum(mask)) t1 = time.perf_counter() catalogueconstructor.extract_some_waveforms(n_left=-25, n_right=40, mode='rand', nb_max=5000) t2 = time.perf_counter() print('extract_some_waveforms rand', t2 - t1) print(catalogueconstructor.some_waveforms.shape) t1 = time.perf_counter() catalogueconstructor.find_good_limits() t2 = time.perf_counter() print('find_good_limits', t2 - t1) print(catalogueconstructor.some_waveforms.shape) t1 = time.perf_counter() catalogueconstructor.extract_some_waveforms(n_left=None, n_right=None, mode='rand', nb_max=5000) t2 = time.perf_counter() print('extract_some_waveforms rand', t2 - t1) print(catalogueconstructor.some_waveforms.shape) t1 = time.perf_counter() catalogueconstructor.clean_waveforms(alien_value_threshold=60.) t2 = time.perf_counter() print('clean_waveforms', t2 - t1) print(catalogueconstructor) #extract_some_noise t1 = time.perf_counter() catalogueconstructor.extract_some_noise(nb_snippet=400) t2 = time.perf_counter() print('extract_some_noise', t2 - t1) # PCA t1 = time.perf_counter() catalogueconstructor.project(method='global_pca', n_components=7, batch_size=16384) t2 = time.perf_counter() print('project pca', t2 - t1) # peak_max #~ t1 = time.perf_counter() #~ catalogueconstructor.project(method='peak_max') #~ t2 = time.perf_counter() #~ print('project peak_max', t2-t1) #~ print(catalogueconstructor.some_features.shape) #~ t1 = time.perf_counter() #~ catalogueconstructor.extract_some_waveforms(index=np.arange(1000)) #~ t2 = time.perf_counter() #~ print('extract_some_waveforms others', t2-t1) #~ print(catalogueconstructor.some_waveforms.shape) # cluster t1 = time.perf_counter() catalogueconstructor.find_clusters(method='kmeans', n_clusters=11) t2 = time.perf_counter() print('find_clusters', t2 - t1) print(catalogueconstructor)
def compare_nb_waveforms(): if os.path.exists('test_catalogueconstructor'): shutil.rmtree('test_catalogueconstructor') dataio = DataIO(dirname='test_catalogueconstructor') localdir, filenames, params = download_dataset(name='olfactory_bulb') dataio.set_data_source(type='RawData', filenames=filenames, **params) dataio.add_one_channel_group(channels=range(14), chan_grp=0) catalogueconstructor = CatalogueConstructor(dataio=dataio) catalogueconstructor.set_preprocessor_params( chunksize=1024, #signal preprocessor highpass_freq=300., lowpass_freq=5000., lostfront_chunksize=128, #peak detector peak_sign='-', relative_threshold=7, peak_span=0.0005, ) t1 = time.perf_counter() catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.) t2 = time.perf_counter() print('estimate_signals_noise', t2 - t1) t1 = time.perf_counter() catalogueconstructor.run_signalprocessor() t2 = time.perf_counter() print('run_signalprocessor', t2 - t1) print(catalogueconstructor) fig, axs = plt.subplots(nrows=2) colors = ['r', 'g', 'b'] for i, nb_max in enumerate([100, 1000, 10000]): t1 = time.perf_counter() catalogueconstructor.extract_some_waveforms(n_left=-20, n_right=30, nb_max=nb_max) t2 = time.perf_counter() print('extract_some_waveforms', nb_max, t2 - t1) print(catalogueconstructor.some_waveforms.shape) wf = catalogueconstructor.some_waveforms wf = wf.swapaxes(1, 2).reshape(wf.shape[0], -1) axs[0].plot(np.median(wf, axis=0), color=colors[i], label='nb_max {}'.format(nb_max)) axs[1].plot(np.mean(wf, axis=0), color=colors[i], label='nb_max {}'.format(nb_max)) axs[0].legend() axs[0].set_title('median') axs[1].set_title('mean') plt.show()