def test_make_catalogue():
dataio = DataIO(dirname='test_catalogueconstructor')
cc = CatalogueConstructor(dataio=dataio)
#~ cc.make_catalogue()
cc.make_catalogue_for_peeler()
def setup_catalogue(dirname, dataset_name='olfactory_bulb'):
if os.path.exists(dirname):
shutil.rmtree(dirname)
dataio = DataIO(dirname=dirname)
localdir, filenames, params = download_dataset(name=dataset_name)
dataio.set_data_source(type='RawData', filenames=filenames, **params)
if dataset_name=='olfactory_bulb':
channels = [5, 6, 7, 8, 9]
else:
channels = [0,1,2,3]
dataio.add_one_channel_group(channels=channels)
catalogueconstructor = CatalogueConstructor(dataio=dataio)
params = {
'duration' : 60.,
'preprocessor' : {
'highpass_freq' : 300.,
'chunksize' : 1024,
'lostfront_chunksize' : 100,
},
'peak_detector' : {
'peak_sign' : '-',
'relative_threshold' : 7.,
'peak_span_ms' : 0.5,
},
'extract_waveforms' : {
'wf_left_ms' : -2.5,
'wf_right_ms' : 4.0,
'nb_max' : 10000,
},
'clean_waveforms' : {
'alien_value_threshold' : 60.,
},
'noise_snippet' : {
'nb_snippet' : 300,
},
'feature_method': 'global_pca',
'feature_kargs':{'n_components': 5},
'cluster_method' : 'kmeans',
'cluster_kargs' : {'n_clusters': 12},
'clean_cluster' : False,
'clean_cluster_kargs' : {},
}
apply_all_catalogue_steps(catalogueconstructor, params, verbose=True)
catalogueconstructor.trash_small_cluster()
catalogueconstructor.order_clusters(by='waveforms_rms')
catalogueconstructor.make_catalogue_for_peeler()
def setup_catalogue():
if os.path.exists('test_peeler'):
shutil.rmtree('test_peeler')
dataio = DataIO(dirname='test_peeler')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
dataio.add_one_channel_group(channels=[5, 6, 7, 8, 9])
catalogueconstructor = CatalogueConstructor(dataio=dataio)
fullchain_kargs = {
'duration': 60.,
'preprocessor': {
'highpass_freq': 300.,
'chunksize': 1024,
'lostfront_chunksize': 100,
},
'peak_detector': {
'peak_sign': '-',
'relative_threshold': 7.,
'peak_span': 0.0005,
#~ 'peak_span' : 0.000,
},
'extract_waveforms': {
'n_left': -25,
'n_right': 40,
'nb_max': 10000,
},
'clean_waveforms': {
'alien_value_threshold': 60.,
},
'noise_snippet': {
'nb_snippet': 300,
},
}
apply_all_catalogue_steps(catalogueconstructor,
fullchain_kargs,
'global_pca', {'n_components': 12},
'kmeans', {'n_clusters': 12},
verbose=True)
catalogueconstructor.trash_small_cluster()
catalogueconstructor.make_catalogue_for_peeler()
def setup_catalogue(dirname, dataset_name='olfactory_bulb'):
if os.path.exists(dirname):
shutil.rmtree(dirname)
dataio = DataIO(dirname=dirname)
localdir, filenames, params = download_dataset(name=dataset_name)
dataio.set_data_source(type='RawData', filenames=filenames, **params)
if dataset_name=='olfactory_bulb':
channels = [4, 5, 6, 7, 8, 9]
mode = 'sparse'
adjacency_radius_um = 350
peak_method = 'geometrical'
peak_engine = 'numpy'
feature_method = 'pca_by_channel'
feature_kargs = {'n_components_by_channel': 3}
else:
channels = [0,1,2,3]
mode = 'dense'
adjacency_radius_um = None
peak_method = 'global'
peak_engine = 'numpy'
feature_method = 'global_pca'
feature_kargs = {'n_components': 5}
dataio.add_one_channel_group(channels=channels)
catalogueconstructor = CatalogueConstructor(dataio=dataio)
params = {
'duration' : 60.,
'chunksize': 1024,
'mode': mode,
'memory_mode': 'memmap',
'preprocessor' : {
'highpass_freq' : 300.,
'lostfront_chunksize' : 100,
'engine' : 'numpy',
},
'peak_detector' : {
'peak_sign' : '-',
'relative_threshold' : 7.,
'peak_span_ms' : 0.5,
'method' : peak_method,
'engine' : peak_engine,
'adjacency_radius_um':adjacency_radius_um,
},
'extract_waveforms' : {
'wf_left_ms' : -2.5,
'wf_right_ms' : 4.0,
#~ 'nb_max' : 10000,
},
'clean_peaks' : {
'alien_value_threshold' : 60.,
'mode': 'full_waveform',
},
'peak_sampler':{
'mode': 'rand',
'nb_max' : 10000,
},
'noise_snippet' : {
'nb_snippet' : 300,
},
'feature_method': feature_method,
'feature_kargs':feature_kargs,
#~ 'cluster_method' : 'kmeans',
#~ 'cluster_kargs' : {'n_clusters': 12},
'cluster_method' : 'pruningshears',
'cluster_kargs' : {},
'clean_cluster' : False,
'clean_cluster_kargs' : {},
}
#~ pprint(params)
apply_all_catalogue_steps(catalogueconstructor, params, verbose=True)
catalogueconstructor.make_catalogue_for_peeler()
def setup_catalogue(dirname, dataset_name='olfactory_bulb', duration=None):
if os.path.exists(dirname):
shutil.rmtree(dirname)
dataio = DataIO(dirname=dirname)
localdir, filenames, params = download_dataset(name=dataset_name)
dataio.set_data_source(type='RawData', filenames=filenames, **params)
if dataset_name=='olfactory_bulb':
channels = [4, 5, 6, 7, 8, 9]
mode = 'sparse'
adjacency_radius_um = 350
peak_method = 'geometrical'
peak_engine = 'numpy'
feature_method = 'pca_by_channel'
feature_kargs = {'n_components_by_channel': 3}
cluster_method = 'pruningshears'
cluster_kargs = {'adjacency_radius_um' : 350 }
else:
channels = [0,1,2,3]
mode = 'dense'
adjacency_radius_um = None
peak_method = 'global'
peak_engine = 'numpy'
feature_method = 'global_pca'
feature_kargs = {'n_components': 6}
cluster_method = 'pruningshears'
cluster_kargs = {'adjacency_radius_um' : 150 }
dataio.add_one_channel_group(channels=channels)
cc = CatalogueConstructor(dataio=dataio)
params = get_auto_params_for_catalogue(dataio)
params['mode'] = mode
params['n_jobs'] = 1
params['peak_detector']['method'] = peak_method
params['peak_detector']['engine'] = peak_engine
params['peak_detector']['adjacency_radius_um'] = adjacency_radius_um
params['feature_method'] = feature_method
params['feature_kargs'] = feature_kargs
params['cluster_method'] = cluster_method
params['cluster_kargs'] = cluster_kargs
if duration is not None:
params['duration'] = duration
#~ pprint(params)
cc.apply_all_steps(params, verbose=True)
# already done in apply_all_catalogue_steps:
# cc.make_catalogue_for_peeler(inter_sample_oversampling=False, catalogue_name='initial')
cc.make_catalogue_for_peeler(inter_sample_oversampling=True, catalogue_name='with_oversampling')
return cc, params
