def test_pruningshears():
dirname = 'test_cluster'
dataio = DataIO(dirname=dirname)
print(dataio)
cc = CatalogueConstructor(dataio=dataio)
#~ print(cc.mode)
#~ exit()
#~ cc.extract_some_features(method='pca_by_channel')
#~ print(dataio)
#~ print(cc)
if dataset_name == 'olfactory_bulb':
kargs = dict(adjacency_radius_um = 420)
else:
kargs = {}
t0 = time.perf_counter()
#~ cc.find_clusters(method='pruningshears', print_debug=True)
#~ cc.find_clusters(method='pruningshears', print_debug=True, debug_plot=True, **kargs)
cc.find_clusters(method='pruningshears', print_debug=False, debug_plot=False, **kargs)
t1 = time.perf_counter()
print('cluster', t1-t0)
if __name__ == '__main__':
app = mkQApp()
win = CatalogueWindow(cc)
win.show()
app.exec_()
def test_make_catalogue():
dataio = DataIO(dirname='test_catalogueconstructor')
cc = CatalogueConstructor(dataio=dataio)
#~ cc.make_catalogue()
cc.make_catalogue_for_peeler()
def debug_interp_centers0():
dataio = DataIO(dirname='test_catalogueconstructor')
catalogueconstructor = CatalogueConstructor(dataio=dataio)
catalogue = catalogueconstructor.make_catalogue()
centers = catalogue['centers0']
interp_centers = catalogue['interp_centers0']
subsample_ratio = catalogue['subsample_ratio']
def test_sawchaincut():
#~ dirname = 'test_catalogueconstructor'
#~ dirname = '/home/samuel/Documents/projet/tridesclous/example/tridesclous_locust/'
#~ dirname = '/home/samuel/Documents/projet/DataSpikeSorting/GT 252/tdc_20170623_patch1/'
#~ dirname = '/home/samuel/Documents/projet/tridesclous/example/tridesclous_locust/'
#~ dirname = '/home/samuel/Documents/projet/tridesclous/example/tridesclous_olfactory_bulb/'
#~ dirname = '/home/samuel/Documents/projet/tridesclous/example/tridesclous_olfactory_bulb/'
#~ dirname = '/home/samuel/Documents/projet/DataSpikeSorting/kampff/tdc_2015_09_03_Cell9.0/'
#~ dirname = '/home/samuel/Documents/projet/DataSpikeSorting/spikesortingtest/tdc_silico_0/'
#~ dirname = '/home/samuel/Documents/projet/tridesclous/example/tridesclous_purkinje/'
dirname = 'test_cluster'
dataio = DataIO(dirname=dirname)
cc = CatalogueConstructor(dataio=dataio)
#~ print(dataio)
#~ print(cc)
t0 = time.perf_counter()
cc.find_clusters(method='sawchaincut', print_debug=True)
t1 = time.perf_counter()
print('cluster', t1 - t0)
#~ exit()
#~ print(cc)
if __name__ == '__main__':
app = mkQApp()
win = CatalogueWindow(cc)
win.show()
app.exec_()
def open_PeelerWindow():
dataio = DataIO(dirname='test_peeler')
catalogueconstructor = CatalogueConstructor(dataio=dataio)
initial_catalogue = catalogueconstructor.load_catalogue()
app = pg.mkQApp()
win = PeelerWindow(dataio=dataio, catalogue=initial_catalogue)
win.show()
app.exec_()
def test_feature_with_lda_selection():
dataio = DataIO(dirname='test_catalogueconstructor')
cc = CatalogueConstructor(dataio=dataio)
print(cc)
selection = np.in1d(cc.all_peaks['cluster_label'], [1, 2, 3, 4])
print(np.sum(selection), '/', cc.all_peaks.size)
cc.extract_some_features(method='global_lda', selection=selection)
print(cc.some_features.shape)
print(cc.some_features)
def test_trash_small_cluster():
dirname = 'test_cleancluster'
restore_savepoint(dirname, savepoint='after_trash_low_extremum')
dataio = DataIO(dirname=dirname)
cc = CatalogueConstructor(dataio=dataio)
t1 = time.perf_counter()
cc.trash_small_cluster()
t2 = time.perf_counter()
print('trash_small_cluster', t2 - t1)
def test_peeler():
dataio = DataIO(dirname='test_peeler')
print(dataio)
catalogueconstructor = CatalogueConstructor(dataio=dataio)
initial_catalogue = catalogueconstructor.load_catalogue()
peeler = Peeler(dataio)
peeler.change_params(catalogue=initial_catalogue, n_peel_level=2, chunksize=1024)
t1 = time.perf_counter()
peeler.run()
t2 = time.perf_counter()
print('peeler.run_loop', t2-t1)
def open_catalogue_window():
dataio = DataIO(dirname='test_peeler')
catalogueconstructor = CatalogueConstructor(dataio=dataio)
app = pg.mkQApp()
win = CatalogueWindow(catalogueconstructor)
win.show()
app.exec_()
def setup_catalogue(dirname, dataset_name='olfactory_bulb'):
if os.path.exists(dirname):
shutil.rmtree(dirname)
dataio = DataIO(dirname=dirname)
localdir, filenames, params = download_dataset(name=dataset_name)
dataio.set_data_source(type='RawData', filenames=filenames, **params)
if dataset_name=='olfactory_bulb':
channels = [5, 6, 7, 8, 9]
else:
channels = [0,1,2,3]
dataio.add_one_channel_group(channels=channels)
catalogueconstructor = CatalogueConstructor(dataio=dataio)
params = {
'duration' : 60.,
'preprocessor' : {
'highpass_freq' : 300.,
'chunksize' : 1024,
'lostfront_chunksize' : 100,
},
'peak_detector' : {
'peak_sign' : '-',
'relative_threshold' : 7.,
'peak_span_ms' : 0.5,
},
'extract_waveforms' : {
'wf_left_ms' : -2.5,
'wf_right_ms' : 4.0,
'nb_max' : 10000,
},
'clean_waveforms' : {
'alien_value_threshold' : 60.,
},
'noise_snippet' : {
'nb_snippet' : 300,
},
'feature_method': 'global_pca',
'feature_kargs':{'n_components': 5},
'cluster_method' : 'kmeans',
'cluster_kargs' : {'n_clusters': 12},
'clean_cluster' : False,
'clean_cluster_kargs' : {},
}
apply_all_catalogue_steps(catalogueconstructor, params, verbose=True)
catalogueconstructor.trash_small_cluster()
catalogueconstructor.order_clusters(by='waveforms_rms')
catalogueconstructor.make_catalogue_for_peeler()
def compare_nb_waveforms():
if os.path.exists('test_catalogueconstructor'):
shutil.rmtree('test_catalogueconstructor')
dataio = DataIO(dirname='test_catalogueconstructor')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
dataio.set_manual_channel_group(range(14))
catalogueconstructor = CatalogueConstructor(dataio=dataio)
catalogueconstructor.set_preprocessor_params(chunksize=1024,
#signal preprocessor
highpass_freq=300,
backward_chunksize=1280,
#peak detector
peak_sign='-', relative_threshold=7, peak_span=0.0005,
)
t1 = time.perf_counter()
catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.)
t2 = time.perf_counter()
print('estimate_signals_noise', t2-t1)
t1 = time.perf_counter()
catalogueconstructor.run_signalprocessor()
t2 = time.perf_counter()
print('run_signalprocessor', t2-t1)
print(catalogueconstructor)
fig, axs = pyplot.subplots(nrows=2)
colors = ['r', 'g', 'b']
for i, nb_max in enumerate([100, 1000, 10000]):
t1 = time.perf_counter()
catalogueconstructor.extract_some_waveforms(n_left=-20, n_right=30, nb_max=nb_max)
t2 = time.perf_counter()
print('extract_some_waveforms', nb_max, t2-t1)
print(catalogueconstructor.some_waveforms.shape)
wf = catalogueconstructor.some_waveforms
wf = wf.swapaxes(1,2).reshape(wf.shape[0], -1)
axs[0].plot(np.median(wf, axis=0), color=colors[i], label='nb_max {}'.format(nb_max))
axs[1].plot(np.mean(wf, axis=0), color=colors[i], label='nb_max {}'.format(nb_max))
axs[0].legend()
axs[0].set_title('median')
axs[1].set_title('mean')
pyplot.show()
def test_sawchaincut():
dirname = 'test_cluster'
dataio = DataIO(dirname=dirname)
cc = CatalogueConstructor(dataio=dataio)
#~ print(dataio)
#~ print(cc)
t0 = time.perf_counter()
cc.find_clusters(method='sawchaincut', print_debug=True)
t1 = time.perf_counter()
print('cluster', t1 - t0)
#~ exit()
#~ print(cc)
if __name__ == '__main__':
app = mkQApp()
win = CatalogueWindow(cc)
win.show()
app.exec_()
def test_apply_all_catalogue_steps():
if os.path.exists('test_cataloguetools'):
shutil.rmtree('test_cataloguetools')
dataio = DataIO(dirname='test_cataloguetools')
#~ localdir, filenames, params = download_dataset(name='olfactory_bulb')
localdir, filenames, params = download_dataset(name='locust')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
params = get_auto_params_for_catalogue(dataio)
cc = CatalogueConstructor(dataio, chan_grp=0)
apply_all_catalogue_steps(cc, params, verbose=True)
def test_pruningshears():
dirname = 'test_cluster'
dataio = DataIO(dirname=dirname)
print(dataio)
cc = CatalogueConstructor(dataio=dataio)
#~ cc.extract_some_features(method='pca_by_channel')
#~ print(dataio)
#~ print(cc)
t0 = time.perf_counter()
cc.find_clusters(method='pruningshears', print_debug=True)
t1 = time.perf_counter()
print('cluster', t1 - t0)
if __name__ == '__main__':
app = mkQApp()
win = CatalogueWindow(cc)
win.show()
app.exec_()
def test_all_decomposition():
dirname = 'test_catalogueconstructor'
dataio = DataIO(dirname=dirname)
cc = catalogueconstructor = CatalogueConstructor(dataio=dataio)
print(dataio)
print(cc)
methods = ['global_pca', 'pca_by_channel', 'peak_max','neighborhood_pca' ] #'neighborhood_pca', 'tsne', 'pca_by_channel_then_tsne'
for method in methods:
t0 = time.perf_counter()
cc.extract_some_features(method=method)
t1 = time.perf_counter()
print('extract_some_features', method, t1-t0)
def test_plot_signals():
dataio = DataIO('test_matplotlibplot')
catalogueconstructor = CatalogueConstructor(dataio=dataio, chan_grp=0)
plot_signals(dataio, signal_type='initial')
plot_signals(dataio, signal_type='processed')
plot_signals(catalogueconstructor,
signal_type='processed',
with_peaks=True,
time_slice=(2., 3))
plot_signals(catalogueconstructor,
signal_type='processed',
with_span=True,
time_slice=(2., 3))
def test_auto_split():
dirname = 'test_cleancluster'
restore_savepoint(dirname, savepoint='after_find_clusters')
dataio = DataIO(dirname=dirname)
cc = CatalogueConstructor(dataio=dataio)
cc.find_clusters(method='pruningshears')
print(cc)
print(cc.n_jobs)
t1 = time.perf_counter()
cc.auto_split_cluster()
t2 = time.perf_counter()
print('auto_split_cluster', t2 - t1)
print(cc)
cc.create_savepoint(name='after_auto_split')
def test_auto_merge():
dirname = 'test_cleancluster'
restore_savepoint(dirname, savepoint='after_trash_not_aligned')
dataio = DataIO(dirname=dirname)
cc = CatalogueConstructor(dataio=dataio)
t1 = time.perf_counter()
cc.auto_merge_cluster()
t2 = time.perf_counter()
print('auto_merge_cluster', t2 - t1)
cc.create_savepoint(name='after_auto_merge_cluster')
def debug_one_decomposition():
dirname = 'test_catalogueconstructor'
dataio = DataIO(dirname=dirname)
cc = catalogueconstructor = CatalogueConstructor(dataio=dataio)
print(dataio)
print(cc)
t0 = time.perf_counter()
#~ cc.extract_some_features(method='global_pca', n_components=7)
#~ cc.extract_some_features(method='peak_max')
#~ cc.extract_some_features(method='pca_by_channel', n_components_by_channel=3)
cc.extract_some_features(method='neighborhood_pca',
n_components_by_neighborhood=3,
radius_um=500)
print(cc.channel_to_features)
print(cc.channel_to_features.shape)
t1 = time.perf_counter()
print('extract_some_features', t1 - t0)
def test_pruningshears():
dirname = 'test_cluster'
dataio = DataIO(dirname=dirname)
print(dataio)
cc = CatalogueConstructor(dataio=dataio)
cc.extract_some_features(method='pca_by_channel')
#~ print(dataio)
#~ print(cc)
t0 = time.perf_counter()
cc.find_clusters(method='pruningshears', print_debug=True)
t1 = time.perf_counter()
print('cluster', t1 - t0)
def setup_catalogue():
if os.path.exists('test_peeler'):
shutil.rmtree('test_peeler')
dataio = DataIO(dirname='test_peeler')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
dataio.add_one_channel_group(channels=[5, 6, 7, 8, 9])
catalogueconstructor = CatalogueConstructor(dataio=dataio)
fullchain_kargs = {
'duration': 60.,
'preprocessor': {
'highpass_freq': 300.,
'chunksize': 1024,
'lostfront_chunksize': 100,
},
'peak_detector': {
'peak_sign': '-',
'relative_threshold': 7.,
'peak_span': 0.0005,
#~ 'peak_span' : 0.000,
},
'extract_waveforms': {
'n_left': -25,
'n_right': 40,
'nb_max': 10000,
},
'clean_waveforms': {
'alien_value_threshold': 60.,
},
'noise_snippet': {
'nb_snippet': 300,
},
}
apply_all_catalogue_steps(catalogueconstructor,
fullchain_kargs,
'global_pca', {'n_components': 12},
'kmeans', {'n_clusters': 12},
verbose=True)
catalogueconstructor.trash_small_cluster()
catalogueconstructor.make_catalogue_for_peeler()
def setup_catalogue(dirname, dataset_name='olfactory_bulb'):
if os.path.exists(dirname):
shutil.rmtree(dirname)
dataio = DataIO(dirname=dirname)
localdir, filenames, params = download_dataset(name=dataset_name)
dataio.set_data_source(type='RawData', filenames=filenames, **params)
if dataset_name=='olfactory_bulb':
channels = [4, 5, 6, 7, 8, 9]
mode = 'sparse'
adjacency_radius_um = 350
peak_method = 'geometrical'
peak_engine = 'numpy'
feature_method = 'pca_by_channel'
feature_kargs = {'n_components_by_channel': 3}
else:
channels = [0,1,2,3]
mode = 'dense'
adjacency_radius_um = None
peak_method = 'global'
peak_engine = 'numpy'
feature_method = 'global_pca'
feature_kargs = {'n_components': 5}
dataio.add_one_channel_group(channels=channels)
catalogueconstructor = CatalogueConstructor(dataio=dataio)
params = {
'duration' : 60.,
'chunksize': 1024,
'mode': mode,
'memory_mode': 'memmap',
'preprocessor' : {
'highpass_freq' : 300.,
'lostfront_chunksize' : 100,
'engine' : 'numpy',
},
'peak_detector' : {
'peak_sign' : '-',
'relative_threshold' : 7.,
'peak_span_ms' : 0.5,
'method' : peak_method,
'engine' : peak_engine,
'adjacency_radius_um':adjacency_radius_um,
},
'extract_waveforms' : {
'wf_left_ms' : -2.5,
'wf_right_ms' : 4.0,
#~ 'nb_max' : 10000,
},
'clean_peaks' : {
'alien_value_threshold' : 60.,
'mode': 'full_waveform',
},
'peak_sampler':{
'mode': 'rand',
'nb_max' : 10000,
},
'noise_snippet' : {
'nb_snippet' : 300,
},
'feature_method': feature_method,
'feature_kargs':feature_kargs,
#~ 'cluster_method' : 'kmeans',
#~ 'cluster_kargs' : {'n_clusters': 12},
'cluster_method' : 'pruningshears',
'cluster_kargs' : {},
'clean_cluster' : False,
'clean_cluster_kargs' : {},
}
#~ pprint(params)
apply_all_catalogue_steps(catalogueconstructor, params, verbose=True)
catalogueconstructor.make_catalogue_for_peeler()
def test_plot_waveforms():
dataio = DataIO('test_matplotlibplot')
catalogueconstructor = CatalogueConstructor(dataio=dataio, chan_grp=0)
plot_waveforms(catalogueconstructor)
def test_plot_features_scatter_2d():
dataio = DataIO('test_matplotlibplot')
catalogueconstructor = CatalogueConstructor(dataio=dataio, chan_grp=0)
#~ plot_features_scatter_2d(catalogueconstructor)
plot_features_scatter_2d(catalogueconstructor, labels=[0])
def test_catalogue_constructor():
if os.path.exists('test_catalogueconstructor'):
shutil.rmtree('test_catalogueconstructor')
dataio = DataIO(dirname='test_catalogueconstructor')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
channels=range(14)
dataio.set_manual_channel_group(channels, chan_grp=0)
catalogueconstructor = CatalogueConstructor(dataio=dataio)
for memory_mode in ['ram', 'memmap']:
#~ for memory_mode in ['memmap']:
print()
print(memory_mode)
catalogueconstructor.set_preprocessor_params(chunksize=1024,
memory_mode=memory_mode,
#signal preprocessor
highpass_freq=300,
backward_chunksize=1280,
#~ backward_chunksize=1024*2,
#peak detector
peakdetector_engine='numpy',
peak_sign='-', relative_threshold=7, peak_span=0.0005,
#waveformextractor
#~ n_left=-20, n_right=30,
)
t1 = time.perf_counter()
catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.)
t2 = time.perf_counter()
print('estimate_signals_noise', t2-t1)
t1 = time.perf_counter()
for seg_num in range(dataio.nb_segment):
#~ print('seg_num', seg_num)
catalogueconstructor.run_signalprocessor_loop_one_segment(seg_num=seg_num, duration=10.)
t2 = time.perf_counter()
print('run_signalprocessor_loop', t2-t1)
t1 = time.perf_counter()
catalogueconstructor.finalize_signalprocessor_loop()
t2 = time.perf_counter()
print('finalize_signalprocessor_loop', t2-t1)
for seg_num in range(dataio.nb_segment):
mask = catalogueconstructor.all_peaks['segment']==seg_num
print('seg_num', seg_num, np.sum(mask))
t1 = time.perf_counter()
catalogueconstructor.extract_some_waveforms(n_left=-25, n_right=40, mode='rand', nb_max=5000)
t2 = time.perf_counter()
print('extract_some_waveforms rand', t2-t1)
print(catalogueconstructor.some_waveforms.shape)
t1 = time.perf_counter()
catalogueconstructor.find_good_limits()
t2 = time.perf_counter()
print('find_good_limits', t2-t1)
print(catalogueconstructor.some_waveforms.shape)
t1 = time.perf_counter()
catalogueconstructor.extract_some_waveforms(n_left=None, n_right=None, mode='rand', nb_max=2000)
t2 = time.perf_counter()
print('extract_some_waveforms rand', t2-t1)
print(catalogueconstructor.some_waveforms.shape)
#~ break
# PCA
t1 = time.perf_counter()
catalogueconstructor.project(method='pca', n_components=7, batch_size=16384)
t2 = time.perf_counter()
print('project pca', t2-t1)
# peak_max
#~ t1 = time.perf_counter()
#~ catalogueconstructor.project(method='peak_max')
#~ t2 = time.perf_counter()
#~ print('project peak_max', t2-t1)
#~ print(catalogueconstructor.some_features.shape)
t1 = time.perf_counter()
catalogueconstructor.extract_some_waveforms(index=np.arange(1000))
t2 = time.perf_counter()
print('extract_some_waveforms others', t2-t1)
print(catalogueconstructor.some_waveforms.shape)
# cluster
t1 = time.perf_counter()
catalogueconstructor.find_clusters(method='kmeans', n_clusters=11)
t2 = time.perf_counter()
print('find_clusters', t2-t1)
def setup_catalogue():
if os.path.exists('test_peeler'):
shutil.rmtree('test_peeler')
dataio = DataIO(dirname='test_peeler')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
dataio.set_manual_channel_group([5, 6, 7, 8, 9])
catalogueconstructor = CatalogueConstructor(dataio=dataio)
#~ print(catalogueconstructor)
catalogueconstructor.set_preprocessor_params(chunksize=1024,
memory_mode='memmap',
#signal preprocessor
highpass_freq=300,
backward_chunksize=1280,
#peak detector
peak_sign='-', relative_threshold=7, peak_span=0.0005,
)
t1 = time.perf_counter()
catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.)
t2 = time.perf_counter()
print('estimate_signals_noise', t2-t1)
t1 = time.perf_counter()
catalogueconstructor.run_signalprocessor()
t2 = time.perf_counter()
print('run_signalprocessor', t2-t1)
print(catalogueconstructor)
t1 = time.perf_counter()
catalogueconstructor.extract_some_waveforms(n_left=-25, n_right=40, nb_max=10000)
t2 = time.perf_counter()
print('extract_some_waveforms', t2-t1)
t1 = time.perf_counter()
n_left, n_right = catalogueconstructor.find_good_limits()
t2 = time.perf_counter()
print('find_good_limits', t2-t1)
print(n_left, n_right)
print(catalogueconstructor.some_waveforms.shape)
# PCA
t1 = time.perf_counter()
catalogueconstructor.project(method='pca', n_components=12, batch_size=16384)
t2 = time.perf_counter()
print('project', t2-t1)
# cluster
t1 = time.perf_counter()
catalogueconstructor.find_clusters(method='kmeans', n_clusters=12)
t2 = time.perf_counter()
print('find_clusters', t2-t1)
# trash_small_cluster
catalogueconstructor.trash_small_cluster()
def test_ratio_amplitude():
dataio = DataIO(dirname='test_catalogueconstructor')
catalogueconstructor = CatalogueConstructor(dataio=dataio)
pairs = catalogueconstructor.detect_similar_waveform_ratio(0.5)
print(pairs)
def test_create_savepoint_catalogue_constructor():
dataio = DataIO(dirname='test_catalogueconstructor')
catalogueconstructor = CatalogueConstructor(dataio=dataio)
copy_path = catalogueconstructor.create_savepoint()
print(copy_path)
def setup_catalogue(dirname, dataset_name='olfactory_bulb', duration=None):
if os.path.exists(dirname):
shutil.rmtree(dirname)
dataio = DataIO(dirname=dirname)
localdir, filenames, params = download_dataset(name=dataset_name)
dataio.set_data_source(type='RawData', filenames=filenames, **params)
if dataset_name=='olfactory_bulb':
channels = [4, 5, 6, 7, 8, 9]
mode = 'sparse'
adjacency_radius_um = 350
peak_method = 'geometrical'
peak_engine = 'numpy'
feature_method = 'pca_by_channel'
feature_kargs = {'n_components_by_channel': 3}
cluster_method = 'pruningshears'
cluster_kargs = {'adjacency_radius_um' : 350 }
else:
channels = [0,1,2,3]
mode = 'dense'
adjacency_radius_um = None
peak_method = 'global'
peak_engine = 'numpy'
feature_method = 'global_pca'
feature_kargs = {'n_components': 6}
cluster_method = 'pruningshears'
cluster_kargs = {'adjacency_radius_um' : 150 }
dataio.add_one_channel_group(channels=channels)
cc = CatalogueConstructor(dataio=dataio)
params = get_auto_params_for_catalogue(dataio)
params['mode'] = mode
params['n_jobs'] = 1
params['peak_detector']['method'] = peak_method
params['peak_detector']['engine'] = peak_engine
params['peak_detector']['adjacency_radius_um'] = adjacency_radius_um
params['feature_method'] = feature_method
params['feature_kargs'] = feature_kargs
params['cluster_method'] = cluster_method
params['cluster_kargs'] = cluster_kargs
if duration is not None:
params['duration'] = duration
#~ pprint(params)
cc.apply_all_steps(params, verbose=True)
# already done in apply_all_catalogue_steps:
# cc.make_catalogue_for_peeler(inter_sample_oversampling=False, catalogue_name='initial')
cc.make_catalogue_for_peeler(inter_sample_oversampling=True, catalogue_name='with_oversampling')
return cc, params
def compare_nb_waveforms():
if os.path.exists('test_catalogueconstructor'):
shutil.rmtree('test_catalogueconstructor')
dataio = DataIO(dirname='test_catalogueconstructor')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
dataio.add_one_channel_group(channels=range(14), chan_grp=0)
cc = CatalogueConstructor(dataio=dataio)
cc.set_global_params(
chunksize=1024,
memory_mode='ram',
mode='dense',
n_jobs=1,
#~ adjacency_radius_um=None,
)
cc.set_preprocessor_params(
#signal preprocessor
highpass_freq=300,
lowpass_freq=5000.,
common_ref_removal=False,
smooth_size=0,
lostfront_chunksize=None)
cc.set_peak_detector_params(
#peak detector
method='global',
engine='numpy',
peak_sign='-',
relative_threshold=7,
peak_span_ms=0.5,
#~ adjacency_radius_um=None,
)
t1 = time.perf_counter()
cc.estimate_signals_noise(seg_num=0, duration=10.)
t2 = time.perf_counter()
print('estimate_signals_noise', t2 - t1)
t1 = time.perf_counter()
cc.run_signalprocessor()
t2 = time.perf_counter()
print('run_signalprocessor', t2 - t1)
print(cc)
fig, axs = plt.subplots(nrows=2)
cc.set_waveform_extractor_params(wf_left_ms=-2.0, wf_right_ms=3.0)
t1 = time.perf_counter()
cc.sample_some_peaks(mode='rand', nb_max=5000)
t2 = time.perf_counter()
print('sample_some_peaks', t2 - t1)
colors = ['r', 'g', 'b', 'y']
for i, nb_max in enumerate([100, 500, 1000, 2000]):
cc.sample_some_peaks(mode='rand', nb_max=nb_max)
#~ catalogueconstructor.extract_some_waveforms(wf_left_ms=-2.0, wf_right_ms=3.0, nb_max=nb_max)
#~ print(catalogueconstructor.some_waveforms.shape)
t1 = time.perf_counter()
wf = cc.get_some_waveforms()
t2 = time.perf_counter()
print('get_some_waveforms', nb_max, t2 - t1)
#~ wf = catalogueconstructor.some_waveforms
wf = wf.swapaxes(1, 2).reshape(wf.shape[0], -1)
axs[0].plot(np.median(wf, axis=0),
color=colors[i],
label='nb_max {}'.format(nb_max))
axs[1].plot(np.mean(wf, axis=0),
color=colors[i],
label='nb_max {}'.format(nb_max))
axs[0].legend()
axs[0].set_title('median')
axs[1].set_title('mean')
plt.show()
def test_catalogue_constructor():
if os.path.exists('test_catalogueconstructor'):
shutil.rmtree('test_catalogueconstructor')
dataio = DataIO(dirname='test_catalogueconstructor')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
#~ localdir, filenames, params = download_dataset(name='locust')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
channels = range(14)
#~ channels=list(range(4))
dataio.add_one_channel_group(channels=channels, chan_grp=0)
cc = CatalogueConstructor(dataio=dataio)
for memory_mode in ['ram', 'memmap']:
for mode in ['dense', 'sparse']:
print('*' * 5)
print('memory_mode', memory_mode, 'mode', mode)
if mode == 'dense':
peak_engine = 'numpy'
peak_method = 'global'
adjacency_radius_um = None
elif mode == 'sparse':
peak_engine = 'numpy'
peak_method = 'geometrical'
adjacency_radius_um = 450.
cc.set_global_params(
chunksize=1024,
memory_mode=memory_mode,
mode=mode,
n_jobs=1,
#~ adjacency_radius_um=adjacency_radius_um,
)
cc.set_preprocessor_params(
#signal preprocessor
highpass_freq=300,
lowpass_freq=5000.,
common_ref_removal=False,
smooth_size=0,
lostfront_chunksize=None)
cc.set_peak_detector_params(
#peak detector
method=peak_method,
engine=peak_engine,
peak_sign='-',
relative_threshold=7,
peak_span_ms=0.5,
adjacency_radius_um=adjacency_radius_um,
)
t1 = time.perf_counter()
cc.estimate_signals_noise(seg_num=0, duration=10.)
t2 = time.perf_counter()
print('estimate_signals_noise', t2 - t1)
t1 = time.perf_counter()
cc.run_signalprocessor(duration=10., detect_peak=True)
t2 = time.perf_counter()
print('run_signalprocessor_loop', t2 - t1)
for seg_num in range(dataio.nb_segment):
mask = cc.all_peaks['segment'] == seg_num
print('seg_num', seg_num, 'nb peak', np.sum(mask))
# redetect peak
cc.re_detect_peak(method=peak_method,
engine=peak_engine,
peak_sign='-',
relative_threshold=5,
peak_span_ms=0.7,
adjacency_radius_um=adjacency_radius_um)
for seg_num in range(dataio.nb_segment):
mask = cc.all_peaks['segment'] == seg_num
print('seg_num', seg_num, 'nb peak', np.sum(mask))
cc.set_waveform_extractor_params(n_left=-25, n_right=40)
t1 = time.perf_counter()
cc.clean_peaks(alien_value_threshold=100,
mode='extremum_amplitude')
t2 = time.perf_counter()
print('clean_peaks extremum_amplitude', t2 - t1)
t1 = time.perf_counter()
cc.clean_peaks(alien_value_threshold=100, mode='full_waveform')
t2 = time.perf_counter()
print('clean_peaks full_waveforms', t2 - t1)
t1 = time.perf_counter()
cc.sample_some_peaks(mode='rand', nb_max=5000)
t2 = time.perf_counter()
print('sample_some_peaks', t2 - t1)
print(cc)
#extract_some_noise
t1 = time.perf_counter()
cc.extract_some_noise(nb_snippet=400)
t2 = time.perf_counter()
print('extract_some_noise', t2 - t1)
if mode == 'dense':
# PCA
t1 = time.perf_counter()
cc.extract_some_features(method='global_pca', n_components=12)
t2 = time.perf_counter()
print('project pca', t2 - t1)
# cluster
t1 = time.perf_counter()
cc.find_clusters(method='kmeans', n_clusters=11)
t2 = time.perf_counter()
print('find_clusters', t2 - t1)
elif mode == 'sparse':
# PCA
t1 = time.perf_counter()
cc.extract_some_features(method='pca_by_channel',
n_components_by_channel=3)
t2 = time.perf_counter()
print('project pca', t2 - t1)
# cluster
t1 = time.perf_counter()
cc.find_clusters(method='pruningshears')
t2 = time.perf_counter()
print('find_clusters', t2 - t1)
print(cc)
t1 = time.perf_counter()
cc.auto_split_cluster()
t2 = time.perf_counter()
print('auto_split_cluster', t2 - t1)
t1 = time.perf_counter()
cc.trash_not_aligned()
t2 = time.perf_counter()
print('trash_not_aligned', t2 - t1)
t1 = time.perf_counter()
cc.auto_merge_cluster()
t2 = time.perf_counter()
print('auto_merge_cluster', t2 - t1)
t1 = time.perf_counter()
cc.trash_low_extremum()
t2 = time.perf_counter()
print('trash_low_extremum', t2 - t1)
t1 = time.perf_counter()
cc.trash_small_cluster()
t2 = time.perf_counter()
print('trash_small_cluster', t2 - t1)
def test_catalogue_constructor():
if os.path.exists('test_catalogueconstructor'):
shutil.rmtree('test_catalogueconstructor')
dataio = DataIO(dirname='test_catalogueconstructor')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
#~ localdir, filenames, params = download_dataset(name='locust')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
channels = range(14)
#~ channels=list(range(4))
dataio.add_one_channel_group(channels=channels, chan_grp=0)
catalogueconstructor = CatalogueConstructor(dataio=dataio)
for memory_mode in ['ram', 'memmap']:
#~ for memory_mode in ['memmap']:
print()
print(memory_mode)
catalogueconstructor.set_preprocessor_params(
chunksize=1024,
memory_mode=memory_mode,
#signal preprocessor
highpass_freq=300,
lowpass_freq=5000.,
common_ref_removal=False,
smooth_size=0,
lostfront_chunksize=128,
#peak detector
peakdetector_engine='numpy',
peak_sign='-',
relative_threshold=7,
peak_span=0.0005,
#waveformextractor
#~ n_left=-20, n_right=30,
)
t1 = time.perf_counter()
catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.)
t2 = time.perf_counter()
print('estimate_signals_noise', t2 - t1)
#~ t1 = time.perf_counter()
#~ for seg_num in range(dataio.nb_segment):
#~ print('seg_num', seg_num)
#~ catalogueconstructor.run_signalprocessor_loop_one_segment(seg_num=seg_num, duration=10.)
catalogueconstructor.run_signalprocessor(duration=10.,
detect_peak=True)
t2 = time.perf_counter()
print('run_signalprocessor_loop', t2 - t1)
for seg_num in range(dataio.nb_segment):
mask = catalogueconstructor.all_peaks['segment'] == seg_num
print('seg_num', seg_num, 'nb peak', np.sum(mask))
#redetect peak
catalogueconstructor.re_detect_peak(peakdetector_engine='numpy',
peak_sign='-',
relative_threshold=5,
peak_span=0.0002)
for seg_num in range(dataio.nb_segment):
mask = catalogueconstructor.all_peaks['segment'] == seg_num
print('seg_num', seg_num, 'nb peak', np.sum(mask))
t1 = time.perf_counter()
catalogueconstructor.extract_some_waveforms(n_left=-25,
n_right=40,
mode='rand',
nb_max=5000)
t2 = time.perf_counter()
print('extract_some_waveforms rand', t2 - t1)
print(catalogueconstructor.some_waveforms.shape)
t1 = time.perf_counter()
catalogueconstructor.find_good_limits()
t2 = time.perf_counter()
print('find_good_limits', t2 - t1)
print(catalogueconstructor.some_waveforms.shape)
t1 = time.perf_counter()
catalogueconstructor.extract_some_waveforms(n_left=None,
n_right=None,
mode='rand',
nb_max=5000)
t2 = time.perf_counter()
print('extract_some_waveforms rand', t2 - t1)
print(catalogueconstructor.some_waveforms.shape)
t1 = time.perf_counter()
catalogueconstructor.clean_waveforms(alien_value_threshold=60.)
t2 = time.perf_counter()
print('clean_waveforms', t2 - t1)
print(catalogueconstructor)
#extract_some_noise
t1 = time.perf_counter()
catalogueconstructor.extract_some_noise(nb_snippet=400)
t2 = time.perf_counter()
print('extract_some_noise', t2 - t1)
# PCA
t1 = time.perf_counter()
catalogueconstructor.project(method='global_pca',
n_components=7,
batch_size=16384)
t2 = time.perf_counter()
print('project pca', t2 - t1)
# peak_max
#~ t1 = time.perf_counter()
#~ catalogueconstructor.project(method='peak_max')
#~ t2 = time.perf_counter()
#~ print('project peak_max', t2-t1)
#~ print(catalogueconstructor.some_features.shape)
#~ t1 = time.perf_counter()
#~ catalogueconstructor.extract_some_waveforms(index=np.arange(1000))
#~ t2 = time.perf_counter()
#~ print('extract_some_waveforms others', t2-t1)
#~ print(catalogueconstructor.some_waveforms.shape)
# cluster
t1 = time.perf_counter()
catalogueconstructor.find_clusters(method='kmeans', n_clusters=11)
t2 = time.perf_counter()
print('find_clusters', t2 - t1)
print(catalogueconstructor)
def compare_nb_waveforms():
if os.path.exists('test_catalogueconstructor'):
shutil.rmtree('test_catalogueconstructor')
dataio = DataIO(dirname='test_catalogueconstructor')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
dataio.add_one_channel_group(channels=range(14), chan_grp=0)
catalogueconstructor = CatalogueConstructor(dataio=dataio)
catalogueconstructor.set_preprocessor_params(
chunksize=1024,
#signal preprocessor
highpass_freq=300.,
lowpass_freq=5000.,
lostfront_chunksize=128,
#peak detector
peak_sign='-',
relative_threshold=7,
peak_span=0.0005,
)
t1 = time.perf_counter()
catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.)
t2 = time.perf_counter()
print('estimate_signals_noise', t2 - t1)
t1 = time.perf_counter()
catalogueconstructor.run_signalprocessor()
t2 = time.perf_counter()
print('run_signalprocessor', t2 - t1)
print(catalogueconstructor)
fig, axs = plt.subplots(nrows=2)
colors = ['r', 'g', 'b']
for i, nb_max in enumerate([100, 1000, 10000]):
t1 = time.perf_counter()
catalogueconstructor.extract_some_waveforms(n_left=-20,
n_right=30,
nb_max=nb_max)
t2 = time.perf_counter()
print('extract_some_waveforms', nb_max, t2 - t1)
print(catalogueconstructor.some_waveforms.shape)
wf = catalogueconstructor.some_waveforms
wf = wf.swapaxes(1, 2).reshape(wf.shape[0], -1)
axs[0].plot(np.median(wf, axis=0),
color=colors[i],
label='nb_max {}'.format(nb_max))
axs[1].plot(np.mean(wf, axis=0),
color=colors[i],
label='nb_max {}'.format(nb_max))
axs[0].legend()
axs[0].set_title('median')
axs[1].set_title('mean')
plt.show()
