def get_normed_sigs(chunksize=None):
# get sigs
sigs, sample_rate = get_dataset(name='olfactory_bulb')
#~ sigs = np.tile(sigs, (1, 20)) #for testing large channels num
if sigs.shape[0] % chunksize > 0:
sigs = sigs[:-(sigs.shape[0] % chunksize), :]
nb_channel = sigs.shape[1]
#~ print('nb_channel', nb_channel)
geometry = np.zeros((nb_channel, 2))
geometry[:, 0] = np.arange(nb_channel) * 50 # um spacing
# normalize sigs
highpass_freq = 300.
preprocess_params = dict(highpass_freq=highpass_freq,
common_ref_removal=True,
backward_chunksize=chunksize + chunksize // 4,
output_dtype='float32')
normed_sigs = offline_signal_preprocessor(sigs, sample_rate,
**preprocess_params)
return sigs, sample_rate, normed_sigs, geometry
def test_compare_offline_online_engines():
#~ HAVE_PYOPENCL = True
if HAVE_PYOPENCL:
engines = ['numpy', 'opencl']
#~ engines = [ 'opencl']
#~ engines = ['numpy']
else:
engines = ['numpy']
# get sigs
sigs, sample_rate = get_dataset(name='olfactory_bulb')
#~ sigs = np.tile(sigs, (1, 20)) #for testing large channels num
nb_channel = sigs.shape[1]
print('nb_channel', nb_channel)
#params
chunksize = 1024
peak_sign = '-'
relative_threshold = 8
peak_span = 0.0009
#~ print('n_span', n_span)
nloop = sigs.shape[0] // chunksize
sigs = sigs[:chunksize * nloop]
print('sig duration', sigs.shape[0] / sample_rate)
# normalize sigs
highpass_freq = 300.
preprocess_params = dict(highpass_freq=highpass_freq,
common_ref_removal=True,
backward_chunksize=chunksize + chunksize // 4,
output_dtype='float32')
normed_sigs = offline_signal_preprocessor(sigs, sample_rate,
**preprocess_params)
for peak_sign in [
'-',
'+',
]:
#~ for peak_sign in ['+', ]:
#~ for peak_sign in ['-', ]:
print()
print('peak_sign', peak_sign)
if peak_sign == '-':
sigs = normed_sigs
elif peak_sign == '+':
sigs = -normed_sigs
#~ print(sigs.shape)
#~ print('nloop', nloop)
t1 = time.perf_counter()
offline_peaks, rectified_sum = offline_peak_detect(
sigs,
sample_rate,
peak_sign=peak_sign,
relative_threshold=relative_threshold,
peak_span=peak_span)
t2 = time.perf_counter()
print('offline', 'process time', t2 - t1)
#~ print(offline_peaks)
online_peaks = {}
for engine in engines:
print(engine)
EngineClass = peakdetector_engines[engine]
#~ buffer_size = chunksize*4
peakdetector_engine = EngineClass(sample_rate, nb_channel,
chunksize, 'float32')
peakdetector_engine.change_params(
peak_sign=peak_sign,
relative_threshold=relative_threshold,
peak_span=peak_span)
all_online_peaks = []
t1 = time.perf_counter()
for i in range(nloop):
#~ print(i)
pos = (i + 1) * chunksize
chunk = sigs[pos - chunksize:pos, :]
n_peaks, chunk_peaks = peakdetector_engine.process_data(
pos, chunk)
if chunk_peaks is not None:
#~ all_online_peaks.append(chunk_peaks['index'])
all_online_peaks.append(chunk_peaks)
online_peaks[engine] = np.concatenate(all_online_peaks)
t2 = time.perf_counter()
print(engine, 'process time', t2 - t1)
# remove peaks on border for comparison
offline_peaks = offline_peaks[(offline_peaks > chunksize) & (
offline_peaks < sigs.shape[0] - chunksize)]
for engine in engines:
onlinepeaks = online_peaks[engine]
onlinepeaks = onlinepeaks[(onlinepeaks > chunksize) & (
onlinepeaks < sigs.shape[0] - chunksize)]
online_peaks[engine] = onlinepeaks
# compare
for engine in engines:
onlinepeaks = online_peaks[engine]
assert offline_peaks.size == onlinepeaks.size, '{} nb_peak{} instead {}'.format(
engine, offline_peaks.size, onlinepeaks.size)
assert np.array_equal(offline_peaks, onlinepeaks)
def test_compare_offline_online_engines():
HAVE_PYOPENCL = True
if HAVE_PYOPENCL:
engines = ['numpy', 'opencl']
#~ engines = [ 'opencl']
#~ engines = ['numpy']
else:
engines = ['numpy']
# get sigs
sigs, sample_rate = get_dataset(name='olfactory_bulb')
#~ sigs = np.tile(sigs, (1, 20)) #for testing large channels num
nb_channel = sigs.shape[1]
print('nb_channel', nb_channel)
#params
chunksize = 1024
peak_sign = '-'
relative_threshold = 8
peak_span = 0.0009
#~ print('n_span', n_span)
nloop = sigs.shape[0]//chunksize
sigs = sigs[:chunksize*nloop]
print('sig duration', sigs.shape[0]/sample_rate)
# normalize sigs
highpass_freq = 300.
preprocess_params = dict(
highpass_freq=highpass_freq,
common_ref_removal=True,
backward_chunksize=chunksize+chunksize//4,
output_dtype='float32')
normed_sigs = offline_signal_preprocessor(sigs, sample_rate, **preprocess_params)
for peak_sign in ['-', '+', ]:
#~ for peak_sign in ['+', ]:
#~ for peak_sign in ['-', ]:
print()
print('peak_sign', peak_sign)
if peak_sign=='-':
sigs = normed_sigs
elif peak_sign=='+':
sigs = -normed_sigs
#~ print(sigs.shape)
#~ print('nloop', nloop)
t1 = time.perf_counter()
offline_peaks, rectified_sum = offline_peak_detect(sigs, sample_rate, peak_sign=peak_sign, relative_threshold=relative_threshold, peak_span=peak_span)
t2 = time.perf_counter()
print('offline', 'process time', t2-t1)
#~ print(offline_peaks)
online_peaks = {}
for engine in engines:
print(engine)
EngineClass = peakdetector_engines[engine]
#~ buffer_size = chunksize*4
peakdetector_engine = EngineClass(sample_rate, nb_channel, chunksize, 'float32')
peakdetector_engine.change_params(peak_sign=peak_sign, relative_threshold=relative_threshold,
peak_span=peak_span)
all_online_peaks = []
t1 = time.perf_counter()
for i in range(nloop):
#~ print(i)
pos = (i+1)*chunksize
chunk = sigs[pos-chunksize:pos,:]
n_peaks, chunk_peaks = peakdetector_engine.process_data(pos, chunk)
if chunk_peaks is not None:
#~ all_online_peaks.append(chunk_peaks['index'])
all_online_peaks.append(chunk_peaks)
online_peaks[engine] = np.concatenate(all_online_peaks)
t2 = time.perf_counter()
print(engine, 'process time', t2-t1)
# remove peaks on border for comparison
offline_peaks = offline_peaks[(offline_peaks>chunksize) & (offline_peaks<sigs.shape[0]-chunksize)]
for engine in engines:
onlinepeaks = online_peaks[engine]
onlinepeaks = onlinepeaks[(onlinepeaks>chunksize) & (onlinepeaks<sigs.shape[0]-chunksize)]
online_peaks[engine] = onlinepeaks
# compare
for engine in engines:
onlinepeaks = online_peaks[engine]
assert offline_peaks.size==onlinepeaks.size, '{} nb_peak{} instead {}'.format(engine, offline_peaks.size, onlinepeaks.size)
assert np.array_equal(offline_peaks, onlinepeaks)
def test_compare_offline_online_engines():
sigs, sample_rate = get_dataset()
#~ sigs = sigs[:, [0]]
nb_channel = sigs.shape[1]
print('nb_channel', nb_channel)
#params
chunksize = 1024
nloop = sigs.shape[0]//chunksize
sigs = sigs[:chunksize*nloop]
highpass_freq = 300.
preprocess_params = dict(
highpass_freq=highpass_freq,
common_ref_removal=True,
backward_chunksize=chunksize+chunksize//4,
output_dtype='float32')
peak_params = dict(peak_sign='-',
relative_threshold=8,
peak_span = 0.0009)
waveforms_params = dict(n_left=-20, n_right=30)
n_left = -20
n_right = 30
t1 = time.perf_counter()
offline_sig = offline_signal_preprocessor(sigs, sample_rate, **preprocess_params)
offline_peaks, rectified_sum = offline_peak_detect(offline_sig, sample_rate, **peak_params)
keep = (offline_peaks>chunksize) & (offline_peaks<sigs.shape[0]-chunksize)
offline_peaks = offline_peaks[keep]
offline_waveforms = cut_full(offline_sig, offline_peaks+n_left, n_right-n_left)
print(offline_waveforms.shape)
t2 = time.perf_counter()
print('offline', 'process time', t2-t1)
# precompute medians and mads
params2 = dict(preprocess_params)
params2['normalize'] = False
sigs_for_noise = offline_signal_preprocessor(sigs, sample_rate, **params2)
medians = np.median(sigs_for_noise, axis=0)
mads = np.median(np.abs(sigs_for_noise-medians),axis=0)*1.4826
preprocess_params['signals_medians'] = medians
preprocess_params['signals_mads'] = mads
#
signalpreprocessor = SignalPreprocessor_Numpy(sample_rate, nb_channel, chunksize, sigs.dtype)
signalpreprocessor.change_params(**preprocess_params)
peakdetector = PeakDetectorEngine_Numpy(sample_rate, nb_channel, chunksize, 'float32')
peakdetector.change_params(**peak_params)
waveformextractor = OnlineWaveformExtractor(nb_channel, chunksize)
waveformextractor.change_params(**waveforms_params)
all_online_peak = []
all_online_waveforms = []
t1 = time.perf_counter()
for i in range(nloop):
#~ print()
pos = (i+1)*chunksize
#~ print('loop', i, 'pos', pos-chunksize, pos)
chunk = sigs[pos-chunksize:pos,:]
pos2, preprocessed_chunk = signalpreprocessor.process_data(pos, chunk)
if preprocessed_chunk is None:
continue
#~ print('pos2', pos)
n_peaks, chunk_peaks = peakdetector.process_data(pos2, preprocessed_chunk)
if chunk_peaks is None:
continue
for peak_pos, chunk_waveforms in waveformextractor.new_peaks(pos2, preprocessed_chunk, chunk_peaks):
#~ print(peak_pos, chunk_waveforms.shape)
all_online_peak.append(peak_pos)
all_online_waveforms.append(chunk_waveforms)
t2 = time.perf_counter()
print('online process time', t2-t1)
online_peaks = np.concatenate(all_online_peak, axis=0)
online_waveforms = np.concatenate(all_online_waveforms, axis=0)
keep = (online_peaks>chunksize) & (online_peaks<sigs.shape[0]-chunksize)
online_peaks = online_peaks[keep]
online_waveforms = online_waveforms[keep]
assert np.array_equal(offline_peaks, online_peaks)
residual = np.abs((online_waveforms.astype('float64')-offline_waveforms.astype('float64'))/np.mean(np.abs(offline_waveforms.astype('float64'))))
print(np.max(residual))
#~ print(np.mean(np.abs(offline_sig.astype('float64'))))
assert np.max(residual)<5e-5, 'online differt from offline'
ind_error_max = np.argmax(np.max(residual.reshape(residual.shape[0], -1), axis=1))
#~ print(ind_error_max)
offline_wf = offline_waveforms[ind_error_max, : , :]
online_wf = online_waveforms[ind_error_max, : , :]
#~ print(online_wf.shape)
fig, ax = pyplot.subplots()
ax.plot(offline_wf.flatten(), color='g')
ax.plot(online_wf.flatten(), color='r', ls='--')
fig, ax = pyplot.subplots()
wf2 = offline_waveforms.reshape(offline_waveforms.shape[0], -1)
ax.plot(np.median(wf2, axis=0), color='g')
wf3 = online_waveforms.reshape(offline_waveforms.shape[0], -1)
ax.plot(np.median(wf3, axis=0), color='r', ls='--')
pyplot.show()