def test_peeler():
dataio = DataIO(dirname = 'datatest')
#~ dataio = DataIO(dirname = 'datatest_neo')
sigs = dataio.get_signals(seg_num=0)
#peak
peakdetector = PeakDetector(sigs)
peak_pos = peakdetector.detect_peaks(threshold=-4, peak_sign = '-', n_span = 5)
#waveforms
waveformextractor = WaveformExtractor(peakdetector, n_left=-30, n_right=50)
limit_left, limit_right = waveformextractor.find_good_limits(mad_threshold = 1.1)
#~ print(limit_left, limit_right)
short_wf = waveformextractor.get_ajusted_waveforms()
#~ print(short_wf.shape)
#clustering
clustering = Clustering(short_wf)
features = clustering.project(method = 'pca', n_components = 4)
clustering.find_clusters(8, order_clusters = True)
catalogue = clustering.construct_catalogue()
#~ clustering.plot_catalogue(sameax = False)
#~ clustering.plot_catalogue(sameax = True)
#~ clustering.merge_cluster(1, 2)
catalogue = clustering.construct_catalogue()
clustering.plot_catalogue(sameax = False)
#~ clustering.plot_catalogue(sameax = True)
#peeler
signals = peakdetector.normed_sigs
peeler = Peeler(signals, catalogue, limit_left, limit_right,
threshold=-5., peak_sign = '-', n_span = 5)
prediction0, residuals0 = peeler.peel()
prediction1, residuals1 = peeler.peel()
spiketrains = peeler.get_spiketrains()
print(spiketrains)
fig, axs = pyplot.subplots(nrows = 6, sharex = True)#, sharey = True)
axs[0].plot(signals)
axs[1].plot(prediction0)
axs[2].plot(residuals0)
axs[3].plot(prediction1)
axs[4].plot(residuals1)
for i in range(5):
axs[i].set_ylim(-25, 10)
peeler.plot_spiketrains(ax = axs[5])
def test_peeler():
dataio = DataIO(dirname = 'datatest')
sigs = dataio.get_signals(seg_num=0)
#peak
peakdetector = PeakDetector(sigs)
peak_pos = peakdetector.detect_peaks(threshold=-4, peak_sign = '-', n_span = 5)
#waveforms
waveformextractor = WaveformExtractor(peakdetector, n_left=-30, n_right=50)
limit_left, limit_right = waveformextractor.find_good_limits(mad_threshold = 1.1)
#~ print(limit_left, limit_right)
short_wf = waveformextractor.get_ajusted_waveforms(margin=2)
#~ print(short_wf.shape)
#clustering
clustering = Clustering(short_wf)
features = clustering.project(method = 'pca', n_components = 5)
clustering.find_clusters(7)
catalogue = clustering.construct_catalogue()
clustering.plot_catalogue()
#peeler
signals = peakdetector.normed_sigs
peeler = Peeler(signals, catalogue, limit_left, limit_right,
threshold=-4, peak_sign = '-', n_span = 5)
prediction0, residuals0 = peeler.peel()
prediction1, residuals1 = peeler.peel()
fig, axs = pyplot.subplots(nrows = 6, sharex = True)#, sharey = True)
axs[0].plot(signals)
axs[1].plot(prediction0)
axs[2].plot(residuals0)
axs[3].plot(prediction1)
axs[4].plot(residuals1)
colors = sns.color_palette('husl', len(catalogue))
spiketrains = peeler.get_spiketrains()
i = 0
for k , pos in spiketrains.items():
axs[5].plot(pos, np.ones(pos.size)*k, ls = 'None', marker = '|', markeredgecolor = colors[i], markersize = 10, markeredgewidth = 2)
i += 1
axs[5].set_ylim(0, len(catalogue))
def apply_peeler():
dataio = DataIO(dirname=dirname)
catalogue = dataio.load_catalogue(chan_grp=0)
peeler = Peeler(dataio)
peeler.change_params(catalogue=catalogue, chunksize=1024)
peeler.run(progressbar=True)
def test_peeler_geometry():
dataio = DataIO(dirname='test_peeler')
catalogue0 = dataio.load_catalogue(chan_grp=0)
catalogue1 = dataio.load_catalogue(chan_grp=0, name='with_oversampling')
for catalogue in (catalogue0, catalogue1):
print()
print('engine=geometrical')
print('inter_sample_oversampling',
catalogue['inter_sample_oversampling'])
peeler = Peeler(dataio)
peeler.change_params(engine='geometrical',
catalogue=catalogue,
chunksize=1024,
argmin_method='numba')
#~ argmin_method='opencl')
t1 = time.perf_counter()
peeler.run(progressbar=False)
t2 = time.perf_counter()
print('peeler.run_loop', t2 - t1)
spikes = dataio.get_spikes(chan_grp=0).copy()
labels = catalogue['clusters']['cluster_label']
count_by_label = [
np.sum(spikes['cluster_label'] == label) for label in labels
]
print(labels)
print(count_by_label)
def test_peeler_several_chunksize():
dataio = DataIO(dirname='test_peeler')
print(dataio)
catalogue = dataio.load_catalogue(chan_grp=0)
all_spikes = []
sig_length = dataio.get_segment_length(0)
chunksizes = [174, 512, 1024, 1023, 10000, 150000]
#~ chunksizes = [512, 1024,]
for chunksize in chunksizes:
print('**', chunksize, '**')
peeler = Peeler(dataio)
peeler.change_params(engine='geometrical',
catalogue=catalogue,
chunksize=chunksize,
argmin_method='numba')
t1 = time.perf_counter()
peeler.run(progressbar=False)
t2 = time.perf_counter()
print('extra_size', peeler.peeler_engine.extra_size, 'n_span',
peeler.peeler_engine.n_span, 'peak_width',
peeler.peeler_engine.peak_width)
print('peeler.run_loop', t2 - t1)
# copy is need because the memmap is reset at each loop
spikes = dataio.get_spikes(seg_num=0, chan_grp=0).copy()
all_spikes.append(spikes)
print(spikes.size)
# clip to last spike
last = min([spikes[-1]['index'] for spikes in all_spikes])
for i, chunksize in enumerate(chunksizes):
spikes = all_spikes[i]
all_spikes[i] = spikes[spikes['index'] <= last]
previsous_spikes = None
for i, chunksize in enumerate(chunksizes):
print('**', chunksize, '**')
spikes = all_spikes[i]
is_sorted = np.all(np.diff(spikes['index']) >= 0)
assert is_sorted
labeled_spike = spikes[spikes['cluster_label'] >= 0]
unlabeled_spike = spikes[spikes['cluster_label'] < 0]
print('labeled_spike.size', labeled_spike.size, 'unlabeled_spike.size',
unlabeled_spike.size)
print(spikes)
# TODO: Peeler chunksize influence the number of spikes
if previsous_spikes is not None:
assert previsous_spikes.size == spikes.size
np.testing.assert_array_equal(previsous_spikes['index'],
spikes['index'])
np.testing.assert_array_equal(previsous_spikes['cluster_label'],
spikes['cluster_label'])
previsous_spikes = spikes
def setup_module():
setup_catalogue('test_report', dataset_name='striatum_rat')
dataio = DataIO(dirname='test_report')
initial_catalogue = dataio.load_catalogue(chan_grp=0)
peeler = Peeler(dataio)
peeler.change_params(catalogue=initial_catalogue, chunksize=1024)
peeler.run(progressbar=False)
def test_peeler():
dataio = DataIO(dirname='test_peeler')
initial_catalogue = dataio.load_catalogue(chan_grp=0)
peeler = Peeler(dataio)
peeler.change_params(engine='classic', catalogue=initial_catalogue,chunksize=1024)
t1 = time.perf_counter()
peeler.run(progressbar=False)
t2 = time.perf_counter()
print('peeler.run_loop', t2-t1)
def test_peeler_several_chunksize():
dataio = DataIO(dirname='test_peeler')
print(dataio)
catalogue = dataio.load_catalogue(chan_grp=0)
all_spikes = []
sig_length = dataio.get_segment_length(0)
chunksizes = [174, 512, 1024, 1023, 10000, 150000]
#~ chunksizes = [512, 1024,]
for chunksize in chunksizes:
print('**', chunksize, '**')
peeler = Peeler(dataio)
peeler.change_params(catalogue=catalogue, chunksize=chunksize)
t1 = time.perf_counter()
peeler.run_offline_loop_one_segment(seg_num=0, progressbar=False)
t2 = time.perf_counter()
print('n_side', peeler.n_side, 'n_span', peeler.n_span, 'peak_width',
peeler.peak_width)
print('peeler.run_loop', t2 - t1)
spikes = dataio.get_spikes(seg_num=0, chan_grp=0)
all_spikes.append(spikes)
# clip to last spike
last = min([spikes[-1]['index'] for spikes in all_spikes])
for i, chunksize in enumerate(chunksizes):
spikes = all_spikes[i]
all_spikes[i] = spikes[spikes['index'] <= last]
previsous_spikes = None
for i, chunksize in enumerate(chunksizes):
print('**', chunksize, '**')
spikes = all_spikes[i]
is_sorted = np.all(np.diff(spikes['index']) >= 0)
assert is_sorted
labeled_spike = spikes[spikes['cluster_label'] >= 0]
unlabeled_spike = spikes[spikes['cluster_label'] < 0]
print('labeled_spike.size', labeled_spike.size, 'unlabeled_spike.size',
unlabeled_spike.size)
if previsous_spikes is not None:
assert previsous_spikes.size == spikes.size
np.testing.assert_array_equal(previsous_spikes['index'],
spikes['index'])
np.testing.assert_array_equal(previsous_spikes['cluster_label'],
spikes['cluster_label'])
previsous_spikes = spikes
def run_peeler(dirname, chan_grp):
dataio = DataIO(dirname=dirname, ch_grp=chan_grp)
initial_catalogue = dataio.load_catalogue(chan_grp=chan_grp)
peeler = Peeler(dataio)
peeler.change_params(catalogue=initial_catalogue,
chunksize=32768,
use_sparse_template=False,
sparse_threshold_mad=1.5,
use_opencl_with_sparse=False)
t1 = time.perf_counter()
peeler.run()
t2 = time.perf_counter()
print('peeler.run', t2 - t1)
def run_peeler(dirname):
dataio = DataIO(dirname=dirname)
initial_catalogue = dataio.load_catalogue(chan_grp=0)
peeler = Peeler(dataio)
peeler.change_params(catalogue=initial_catalogue)
t1 = time.perf_counter()
peeler.run()
t2 = time.perf_counter()
print('peeler.run', t2 - t1)
print()
for seg_num in range(dataio.nb_segment):
spikes = dataio.get_spikes(seg_num)
print('seg_num', seg_num, 'nb_spikes', spikes.size)
def test_peeler_sparse_opencl():
dataio = DataIO(dirname='test_peeler')
initial_catalogue = dataio.load_catalogue(chan_grp=0)
peeler = Peeler(dataio)
peeler.change_params(
catalogue=initial_catalogue,
chunksize=1024,
use_sparse_template=True,
sparse_threshold_mad=1.5,
use_opencl_with_sparse=True,
)
t1 = time.perf_counter()
peeler.run(progressbar=False)
t2 = time.perf_counter()
print('peeler.run_loop', t2 - t1)
def test_peeler_with_and_without_preprocessor():
if ON_CI_CLOUD:
engines = ['geometrical']
else:
engines = ['geometrical', 'geometrical_opencl']
#~ engines = ['geometrical_opencl']
for engine in engines:
for i in range(2):
#~ for i in [1]:
print()
if i == 0:
print(engine, 'without processing')
dataio = DataIO(dirname='test_peeler')
else:
print(engine, 'with processing')
dataio = DataIO(dirname='test_peeler2')
catalogue = dataio.load_catalogue(chan_grp=0)
peeler = Peeler(dataio)
peeler.change_params(engine=engine,
catalogue=catalogue,
chunksize=1024)
t1 = time.perf_counter()
peeler.run(progressbar=False)
t2 = time.perf_counter()
print('peeler run_time', t2 - t1)
spikes = dataio.get_spikes(chan_grp=0).copy()
labels = catalogue['clusters']['cluster_label']
count_by_label = [
np.sum(spikes['cluster_label'] == label) for label in labels
]
print(labels)
print(count_by_label)
def test_peeler_argmin_methods():
dataio = DataIO(dirname='test_peeler')
catalogue = dataio.load_catalogue(chan_grp=0, name='with_oversampling')
argmin_methods = ['opencl', 'numba', 'pythran']
#~ argmin_methods = ['opencl', 'pythran']
for argmin_method in argmin_methods:
peeler = Peeler(dataio)
peeler.change_params(
engine='classic',
catalogue=catalogue,
chunksize=1024,
argmin_method=argmin_method,
cl_platform_index=0,
cl_device_index=0,
)
t1 = time.perf_counter()
peeler.run(progressbar=False)
t2 = time.perf_counter()
print(argmin_method, 'peeler.run_loop', t2 - t1)
def test_peeler_empty_catalogue():
"""
This test peeler with empty catalogue.
This is like a peak detector.
Check several chunksize and compare to offline-one-buffer.
"""
dataio = DataIO(dirname='test_peeler')
#~ print(dataio)
catalogue = dataio.load_catalogue(chan_grp=0)
# empty catalogue for debug peak detection
s = catalogue['centers0'].shape
empty_centers = np.zeros((0, s[1], s[2]), dtype='float32')
catalogue['centers0'] = empty_centers
catalogue['centers1'] = empty_centers
catalogue['centers2'] = empty_centers
catalogue['cluster_labels'] = np.zeros(0, dtype=catalogue['cluster_labels'].dtype)
sig_length = dataio.get_segment_length(0)
chunksizes = [ 101, 174, 512, 1024, 1023, 10000, 150000]
#~ chunksizes = [1024,]
previous_peak = None
for chunksize in chunksizes:
print('**', chunksize, '**')
peeler = Peeler(dataio)
peeler.change_params(engine='classic', catalogue=catalogue,chunksize=chunksize)
t1 = time.perf_counter()
#~ peeler.run(progressbar=False)
peeler.run_offline_loop_one_segment(seg_num=0, progressbar=False)
t2 = time.perf_counter()
#~ print('n_side', peeler.n_side, 'n_span', peeler.n_span, 'peak_width', peeler.peak_width)
#~ print('peeler.run_loop', t2-t1)
spikes = dataio.get_spikes(seg_num=0, chan_grp=0)
labeled_spike = spikes[spikes['cluster_label']>=0]
unlabeled_spike = spikes[spikes['cluster_label']<0]
assert labeled_spike.size == 0
is_sorted = np.all(np.diff(unlabeled_spike['index'])>=0)
assert is_sorted
online_peaks = unlabeled_spike['index']
engine = peeler.peeler_engine
i_stop = sig_length-catalogue['signal_preprocessor_params']['lostfront_chunksize']-engine.n_side+engine.n_span
sigs = dataio.get_signals_chunk(signal_type='processed', i_stop=i_stop)
offline_peaks = detect_peaks_in_chunk(sigs, engine.n_span, engine.relative_threshold, engine.peak_sign)
offline_peaks = offline_peaks[offline_peaks<=online_peaks[-1]]
assert offline_peaks.size == online_peaks.size
np.testing.assert_array_equal(offline_peaks, online_peaks)
if previous_peak is not None:
last = min(previous_peak[-1], online_peaks[-1])
previous_peak = previous_peak[previous_peak<=last]
online_peaks_cliped = online_peaks[online_peaks<=last]
assert previous_peak.size == online_peaks_cliped.size
np.testing.assert_array_equal(previous_peak, online_peaks_cliped)
previous_peak = online_peaks
def debug_compare_peeler_engines():
dataio = DataIO(dirname='test_peeler')
print(dataio)
engine_list = [
('classic argmin opencl', 'classic', {
'argmin_method': 'opencl'
}),
('classic argmin numba', 'classic', {
'argmin_method': 'numba'
}),
('geometrical argmin opencl', 'geometrical', {
'argmin_method': 'opencl'
}),
('geometrical argmin numba', 'geometrical', {
'argmin_method': 'numba'
}),
('geometrical_opencl', 'geometrical_opencl', {}),
]
all_spikes = []
for name, engine, kargs in engine_list:
#~ print()
#~ print(name)
catalogue = dataio.load_catalogue(chan_grp=0, name='with_oversampling')
peeler = Peeler(dataio)
peeler.change_params(engine=engine,
catalogue=catalogue,
chunksize=1024,
**kargs)
t1 = time.perf_counter()
peeler.run(progressbar=False, duration=None)
t2 = time.perf_counter()
print(name, 'run', t2 - t1)
spikes = dataio.get_spikes(chan_grp=0).copy()
#~ print(spikes.size)
all_spikes.append(spikes)
#~ print(dataio.get_spikes(chan_grp=0).size)
print()
#~ all_spikes[0] = all_spikes[0][88+80:88+81+10]
#~ all_spikes[1] = all_spikes[1][88+80:88+81+10]
#~ all_spikes[0] = all_spikes[0][:88+81]
#~ all_spikes[1] = all_spikes[1][:88+81]
labels = catalogue['clusters']['cluster_label']
for i, spikes in enumerate(all_spikes):
name = engine_list[i][0]
print()
print(name)
print(spikes[:10])
print(spikes.size)
count_by_label = [
np.sum(spikes['cluster_label'] == label) for label in labels
]
print(count_by_label)
