def test_catalogue_constructor():
if os.path.exists('test_catalogueconstructor'):
shutil.rmtree('test_catalogueconstructor')
dataio = DataIO(dirname='test_catalogueconstructor')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
#~ localdir, filenames, params = download_dataset(name='locust')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
channels = range(14)
#~ channels=list(range(4))
dataio.add_one_channel_group(channels=channels, chan_grp=0)
cc = CatalogueConstructor(dataio=dataio)
for memory_mode in ['ram', 'memmap']:
for mode in ['dense', 'sparse']:
print('*' * 5)
print('memory_mode', memory_mode, 'mode', mode)
if mode == 'dense':
peak_engine = 'numpy'
peak_method = 'global'
adjacency_radius_um = None
elif mode == 'sparse':
peak_engine = 'numpy'
peak_method = 'geometrical'
adjacency_radius_um = 450.
cc.set_global_params(
chunksize=1024,
memory_mode=memory_mode,
mode=mode,
n_jobs=1,
#~ adjacency_radius_um=adjacency_radius_um,
)
cc.set_preprocessor_params(
#signal preprocessor
highpass_freq=300,
lowpass_freq=5000.,
common_ref_removal=False,
smooth_size=0,
lostfront_chunksize=None)
cc.set_peak_detector_params(
#peak detector
method=peak_method,
engine=peak_engine,
peak_sign='-',
relative_threshold=7,
peak_span_ms=0.5,
adjacency_radius_um=adjacency_radius_um,
)
t1 = time.perf_counter()
cc.estimate_signals_noise(seg_num=0, duration=10.)
t2 = time.perf_counter()
print('estimate_signals_noise', t2 - t1)
t1 = time.perf_counter()
cc.run_signalprocessor(duration=10., detect_peak=True)
t2 = time.perf_counter()
print('run_signalprocessor_loop', t2 - t1)
for seg_num in range(dataio.nb_segment):
mask = cc.all_peaks['segment'] == seg_num
print('seg_num', seg_num, 'nb peak', np.sum(mask))
# redetect peak
cc.re_detect_peak(method=peak_method,
engine=peak_engine,
peak_sign='-',
relative_threshold=5,
peak_span_ms=0.7,
adjacency_radius_um=adjacency_radius_um)
for seg_num in range(dataio.nb_segment):
mask = cc.all_peaks['segment'] == seg_num
print('seg_num', seg_num, 'nb peak', np.sum(mask))
cc.set_waveform_extractor_params(n_left=-25, n_right=40)
t1 = time.perf_counter()
cc.clean_peaks(alien_value_threshold=100,
mode='extremum_amplitude')
t2 = time.perf_counter()
print('clean_peaks extremum_amplitude', t2 - t1)
t1 = time.perf_counter()
cc.clean_peaks(alien_value_threshold=100, mode='full_waveform')
t2 = time.perf_counter()
print('clean_peaks full_waveforms', t2 - t1)
t1 = time.perf_counter()
cc.sample_some_peaks(mode='rand', nb_max=5000)
t2 = time.perf_counter()
print('sample_some_peaks', t2 - t1)
print(cc)
#extract_some_noise
t1 = time.perf_counter()
cc.extract_some_noise(nb_snippet=400)
t2 = time.perf_counter()
print('extract_some_noise', t2 - t1)
if mode == 'dense':
# PCA
t1 = time.perf_counter()
cc.extract_some_features(method='global_pca', n_components=12)
t2 = time.perf_counter()
print('project pca', t2 - t1)
# cluster
t1 = time.perf_counter()
cc.find_clusters(method='kmeans', n_clusters=11)
t2 = time.perf_counter()
print('find_clusters', t2 - t1)
elif mode == 'sparse':
# PCA
t1 = time.perf_counter()
cc.extract_some_features(method='pca_by_channel',
n_components_by_channel=3)
t2 = time.perf_counter()
print('project pca', t2 - t1)
# cluster
t1 = time.perf_counter()
cc.find_clusters(method='pruningshears')
t2 = time.perf_counter()
print('find_clusters', t2 - t1)
print(cc)
t1 = time.perf_counter()
cc.auto_split_cluster()
t2 = time.perf_counter()
print('auto_split_cluster', t2 - t1)
t1 = time.perf_counter()
cc.trash_not_aligned()
t2 = time.perf_counter()
print('trash_not_aligned', t2 - t1)
t1 = time.perf_counter()
cc.auto_merge_cluster()
t2 = time.perf_counter()
print('auto_merge_cluster', t2 - t1)
t1 = time.perf_counter()
cc.trash_low_extremum()
t2 = time.perf_counter()
print('trash_low_extremum', t2 - t1)
t1 = time.perf_counter()
cc.trash_small_cluster()
t2 = time.perf_counter()
print('trash_small_cluster', t2 - t1)
def test_catalogue_constructor():
if os.path.exists('test_catalogueconstructor'):
shutil.rmtree('test_catalogueconstructor')
dataio = DataIO(dirname='test_catalogueconstructor')
localdir, filenames, params = download_dataset(name='olfactory_bulb')
#~ localdir, filenames, params = download_dataset(name='locust')
dataio.set_data_source(type='RawData', filenames=filenames, **params)
channels = range(14)
#~ channels=list(range(4))
dataio.add_one_channel_group(channels=channels, chan_grp=0)
catalogueconstructor = CatalogueConstructor(dataio=dataio)
for memory_mode in ['ram', 'memmap']:
#~ for memory_mode in ['memmap']:
print()
print(memory_mode)
catalogueconstructor.set_preprocessor_params(
chunksize=1024,
memory_mode=memory_mode,
#signal preprocessor
highpass_freq=300,
lowpass_freq=5000.,
common_ref_removal=False,
smooth_size=0,
lostfront_chunksize=128,
#peak detector
peakdetector_engine='numpy',
peak_sign='-',
relative_threshold=7,
peak_span=0.0005,
#waveformextractor
#~ n_left=-20, n_right=30,
)
t1 = time.perf_counter()
catalogueconstructor.estimate_signals_noise(seg_num=0, duration=10.)
t2 = time.perf_counter()
print('estimate_signals_noise', t2 - t1)
#~ t1 = time.perf_counter()
#~ for seg_num in range(dataio.nb_segment):
#~ print('seg_num', seg_num)
#~ catalogueconstructor.run_signalprocessor_loop_one_segment(seg_num=seg_num, duration=10.)
catalogueconstructor.run_signalprocessor(duration=10.,
detect_peak=True)
t2 = time.perf_counter()
print('run_signalprocessor_loop', t2 - t1)
for seg_num in range(dataio.nb_segment):
mask = catalogueconstructor.all_peaks['segment'] == seg_num
print('seg_num', seg_num, 'nb peak', np.sum(mask))
#redetect peak
catalogueconstructor.re_detect_peak(peakdetector_engine='numpy',
peak_sign='-',
relative_threshold=5,
peak_span=0.0002)
for seg_num in range(dataio.nb_segment):
mask = catalogueconstructor.all_peaks['segment'] == seg_num
print('seg_num', seg_num, 'nb peak', np.sum(mask))
t1 = time.perf_counter()
catalogueconstructor.extract_some_waveforms(n_left=-25,
n_right=40,
mode='rand',
nb_max=5000)
t2 = time.perf_counter()
print('extract_some_waveforms rand', t2 - t1)
print(catalogueconstructor.some_waveforms.shape)
t1 = time.perf_counter()
catalogueconstructor.find_good_limits()
t2 = time.perf_counter()
print('find_good_limits', t2 - t1)
print(catalogueconstructor.some_waveforms.shape)
t1 = time.perf_counter()
catalogueconstructor.extract_some_waveforms(n_left=None,
n_right=None,
mode='rand',
nb_max=5000)
t2 = time.perf_counter()
print('extract_some_waveforms rand', t2 - t1)
print(catalogueconstructor.some_waveforms.shape)
t1 = time.perf_counter()
catalogueconstructor.clean_waveforms(alien_value_threshold=60.)
t2 = time.perf_counter()
print('clean_waveforms', t2 - t1)
print(catalogueconstructor)
#extract_some_noise
t1 = time.perf_counter()
catalogueconstructor.extract_some_noise(nb_snippet=400)
t2 = time.perf_counter()
print('extract_some_noise', t2 - t1)
# PCA
t1 = time.perf_counter()
catalogueconstructor.project(method='global_pca',
n_components=7,
batch_size=16384)
t2 = time.perf_counter()
print('project pca', t2 - t1)
# peak_max
#~ t1 = time.perf_counter()
#~ catalogueconstructor.project(method='peak_max')
#~ t2 = time.perf_counter()
#~ print('project peak_max', t2-t1)
#~ print(catalogueconstructor.some_features.shape)
#~ t1 = time.perf_counter()
#~ catalogueconstructor.extract_some_waveforms(index=np.arange(1000))
#~ t2 = time.perf_counter()
#~ print('extract_some_waveforms others', t2-t1)
#~ print(catalogueconstructor.some_waveforms.shape)
# cluster
t1 = time.perf_counter()
catalogueconstructor.find_clusters(method='kmeans', n_clusters=11)
t2 = time.perf_counter()
print('find_clusters', t2 - t1)
print(catalogueconstructor)
