def test_make_catalogue(): dataio = DataIO(dirname='test_catalogueconstructor') cc = CatalogueConstructor(dataio=dataio) #~ cc.make_catalogue() cc.make_catalogue_for_peeler()
def setup_catalogue(dirname, dataset_name='olfactory_bulb'): if os.path.exists(dirname): shutil.rmtree(dirname) dataio = DataIO(dirname=dirname) localdir, filenames, params = download_dataset(name=dataset_name) dataio.set_data_source(type='RawData', filenames=filenames, **params) if dataset_name=='olfactory_bulb': channels = [5, 6, 7, 8, 9] else: channels = [0,1,2,3] dataio.add_one_channel_group(channels=channels) catalogueconstructor = CatalogueConstructor(dataio=dataio) params = { 'duration' : 60., 'preprocessor' : { 'highpass_freq' : 300., 'chunksize' : 1024, 'lostfront_chunksize' : 100, }, 'peak_detector' : { 'peak_sign' : '-', 'relative_threshold' : 7., 'peak_span_ms' : 0.5, }, 'extract_waveforms' : { 'wf_left_ms' : -2.5, 'wf_right_ms' : 4.0, 'nb_max' : 10000, }, 'clean_waveforms' : { 'alien_value_threshold' : 60., }, 'noise_snippet' : { 'nb_snippet' : 300, }, 'feature_method': 'global_pca', 'feature_kargs':{'n_components': 5}, 'cluster_method' : 'kmeans', 'cluster_kargs' : {'n_clusters': 12}, 'clean_cluster' : False, 'clean_cluster_kargs' : {}, } apply_all_catalogue_steps(catalogueconstructor, params, verbose=True) catalogueconstructor.trash_small_cluster() catalogueconstructor.order_clusters(by='waveforms_rms') catalogueconstructor.make_catalogue_for_peeler()
def setup_catalogue(): if os.path.exists('test_peeler'): shutil.rmtree('test_peeler') dataio = DataIO(dirname='test_peeler') localdir, filenames, params = download_dataset(name='olfactory_bulb') dataio.set_data_source(type='RawData', filenames=filenames, **params) dataio.add_one_channel_group(channels=[5, 6, 7, 8, 9]) catalogueconstructor = CatalogueConstructor(dataio=dataio) fullchain_kargs = { 'duration': 60., 'preprocessor': { 'highpass_freq': 300., 'chunksize': 1024, 'lostfront_chunksize': 100, }, 'peak_detector': { 'peak_sign': '-', 'relative_threshold': 7., 'peak_span': 0.0005, #~ 'peak_span' : 0.000, }, 'extract_waveforms': { 'n_left': -25, 'n_right': 40, 'nb_max': 10000, }, 'clean_waveforms': { 'alien_value_threshold': 60., }, 'noise_snippet': { 'nb_snippet': 300, }, } apply_all_catalogue_steps(catalogueconstructor, fullchain_kargs, 'global_pca', {'n_components': 12}, 'kmeans', {'n_clusters': 12}, verbose=True) catalogueconstructor.trash_small_cluster() catalogueconstructor.make_catalogue_for_peeler()
def setup_catalogue(dirname, dataset_name='olfactory_bulb'): if os.path.exists(dirname): shutil.rmtree(dirname) dataio = DataIO(dirname=dirname) localdir, filenames, params = download_dataset(name=dataset_name) dataio.set_data_source(type='RawData', filenames=filenames, **params) if dataset_name=='olfactory_bulb': channels = [4, 5, 6, 7, 8, 9] mode = 'sparse' adjacency_radius_um = 350 peak_method = 'geometrical' peak_engine = 'numpy' feature_method = 'pca_by_channel' feature_kargs = {'n_components_by_channel': 3} else: channels = [0,1,2,3] mode = 'dense' adjacency_radius_um = None peak_method = 'global' peak_engine = 'numpy' feature_method = 'global_pca' feature_kargs = {'n_components': 5} dataio.add_one_channel_group(channels=channels) catalogueconstructor = CatalogueConstructor(dataio=dataio) params = { 'duration' : 60., 'chunksize': 1024, 'mode': mode, 'memory_mode': 'memmap', 'preprocessor' : { 'highpass_freq' : 300., 'lostfront_chunksize' : 100, 'engine' : 'numpy', }, 'peak_detector' : { 'peak_sign' : '-', 'relative_threshold' : 7., 'peak_span_ms' : 0.5, 'method' : peak_method, 'engine' : peak_engine, 'adjacency_radius_um':adjacency_radius_um, }, 'extract_waveforms' : { 'wf_left_ms' : -2.5, 'wf_right_ms' : 4.0, #~ 'nb_max' : 10000, }, 'clean_peaks' : { 'alien_value_threshold' : 60., 'mode': 'full_waveform', }, 'peak_sampler':{ 'mode': 'rand', 'nb_max' : 10000, }, 'noise_snippet' : { 'nb_snippet' : 300, }, 'feature_method': feature_method, 'feature_kargs':feature_kargs, #~ 'cluster_method' : 'kmeans', #~ 'cluster_kargs' : {'n_clusters': 12}, 'cluster_method' : 'pruningshears', 'cluster_kargs' : {}, 'clean_cluster' : False, 'clean_cluster_kargs' : {}, } #~ pprint(params) apply_all_catalogue_steps(catalogueconstructor, params, verbose=True) catalogueconstructor.make_catalogue_for_peeler()
def setup_catalogue(dirname, dataset_name='olfactory_bulb', duration=None): if os.path.exists(dirname): shutil.rmtree(dirname) dataio = DataIO(dirname=dirname) localdir, filenames, params = download_dataset(name=dataset_name) dataio.set_data_source(type='RawData', filenames=filenames, **params) if dataset_name=='olfactory_bulb': channels = [4, 5, 6, 7, 8, 9] mode = 'sparse' adjacency_radius_um = 350 peak_method = 'geometrical' peak_engine = 'numpy' feature_method = 'pca_by_channel' feature_kargs = {'n_components_by_channel': 3} cluster_method = 'pruningshears' cluster_kargs = {'adjacency_radius_um' : 350 } else: channels = [0,1,2,3] mode = 'dense' adjacency_radius_um = None peak_method = 'global' peak_engine = 'numpy' feature_method = 'global_pca' feature_kargs = {'n_components': 6} cluster_method = 'pruningshears' cluster_kargs = {'adjacency_radius_um' : 150 } dataio.add_one_channel_group(channels=channels) cc = CatalogueConstructor(dataio=dataio) params = get_auto_params_for_catalogue(dataio) params['mode'] = mode params['n_jobs'] = 1 params['peak_detector']['method'] = peak_method params['peak_detector']['engine'] = peak_engine params['peak_detector']['adjacency_radius_um'] = adjacency_radius_um params['feature_method'] = feature_method params['feature_kargs'] = feature_kargs params['cluster_method'] = cluster_method params['cluster_kargs'] = cluster_kargs if duration is not None: params['duration'] = duration #~ pprint(params) cc.apply_all_steps(params, verbose=True) # already done in apply_all_catalogue_steps: # cc.make_catalogue_for_peeler(inter_sample_oversampling=False, catalogue_name='initial') cc.make_catalogue_for_peeler(inter_sample_oversampling=True, catalogue_name='with_oversampling') return cc, params