def deconv_test1(sp1=None, impulse=True):
g = np.array([0.87797883, -0.10934919])
if impulse:
sp1 = impulse_model_test(sp1, show=False)
else:
# simulated slow ramp test
sp1 = get_sample_spikes_slow_ramp() if sp1 is None else sp1
spc = SpikeCalciumizer(fmodel="AR_2",
fluorescence_saturation=0,
std_noise=0.02,
alpha=1,
g=g)
rec = spc.binned_spikes_to_calcium(sp1.reshape((1, -1)))
c2, bl2, c12, g2, sn2, sp2, lam2 = deconvolution.constrained_foopsi(
rec.ravel(), p=2, bl=0, bas_nonneg=False, s_min=0)
c2, bl2, c12, g2, sn2, sp3, lam2 = deconvolution.constrained_foopsi(
rec.ravel(), p=2, bas_nonneg=False, s_min=0)
conv = np.array([1] + list(-g2))
h0 = inverse_kernel(conv, N=rec.shape[1], fft=True)
x_hat, hhat = wiener_deconvolution(rec.ravel(), h0)
fig, axes = plt.subplots(nrows=5, ncols=1, sharex=True)
axes[0].plot(sp1)
axes[1].plot(sp3)
axes[2].plot(sp2)
axes[3].plot(x_hat)
axes[4].plot(rec.ravel())
axes[0].set_ylabel("truth spike")
axes[1].set_ylabel("bl_auto")
axes[2].set_ylabel('bl=0')
axes[3].set_ylabel("wiener")
axes[4].set_ylabel("calcium")
def deconvolve_detrended(trace, scan_fps, detrend_period=600, AR_order=2):
"""Same as the the `deconvolve` method, except that the fluorescence trace is detrended
before autoregressive modeling
:param np.array trace: 1-d array (num_frames) with the fluorescence trace.
:param float scan_fps: fps of the scan
:param float detrend_period: number of seconds over which percentiles are computed
:param int AR_order: Order of the autoregressive process used to model the impulse
response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay.
:returns: Deconvolved spike trace.
:returns: AR coefficients (AR_order) that model the calcium response:
c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ...
"""
detrend_window = int(round(detrend_period * scan_fps))
n_chunks = len(trace) // detrend_window
if detrend_window > 0 and n_chunks > 0:
chunks_len = n_chunks * detrend_window
trace_chunks = trace[:chunks_len].reshape(-1, detrend_window)
data_prct = df_percentile(trace_chunks, axis=1)[0].mean()
trace = trace - percentile_filter(trace, data_prct, detrend_window)
_, _, _, AR_coeffs, _, spike_trace, _ = deconvolution.constrained_foopsi(trace,
p=AR_order, method='cvxpy', bas_nonneg=False, fudge_factor=0.96)
return spike_trace, AR_coeffs
def deconvolve_reconvolve_test(dff,
deconv_p=2,
conv_p=2,
optimize_g=0,
s_min=0,
lags=5,
fudge_factor=1.,
alpha_est=None,
f_saturation=0,
SN=None,
show=True,
tag="",
save=None):
# optimize_g, s_min
# s_min = 0, None
c, bl, c1, g, sn, sp, lam = deconvolution.constrained_foopsi(
dff,
p=deconv_p,
bas_nonneg=False,
optimize_g=optimize_g,
s_min=s_min,
lags=lags,
fudge_factor=fudge_factor)
if alpha_est is None:
alpha = 1
else:
alpha = alpha_est
fluorescence_model = f"AR_{conv_p}"
spc = SpikeCalciumizer(fmodel=fluorescence_model,
fluorescence_saturation=f_saturation,
std_noise=sn if SN is None else SN,
alpha=alpha,
g=g,
bl=bl)
reconv = spc.binned_spikes_to_calcium(sp.reshape((1, len(sp))))
reconv = reconv.ravel()
spc.std_noise = 0
reconv_clean = spc.binned_spikes_to_calcium(sp.reshape((1, len(sp))))
reconv_clean = reconv_clean.ravel()
# PLOTTING
if show or save is not None:
fig, axes = plt.subplots(nrows=2, ncols=1, sharex=True)
xs = np.arange(1, len(reconv) + 1)
axes[0].plot(xs, np.vstack([dff, reconv]).T)
axes[0].legend(['Original', 'Simulated'])
axes[1].plot(xs, sp)
axes[1].set_xlabel('frames')
fig.suptitle(f"Reconvolve Neuron Test {tag}")
if show:
plt.show()
if save is not None:
fname = os.path.join(save, f'reconvolve_{tag}')
fig.savefig(fname + '.png')
fig.savefig(fname + '.eps')
# include bl for better calculation
return c, c1, sp, sn, reconv, reconv_clean
def foo(method, p):
t = time()
g = np.array([[.95], [1.7, -.71]][p - 1])
for i, sn in enumerate([.2, .5]): # high and low SNR
y, c, s = [a[0] for a in gen_data(g, sn)]
res = constrained_foopsi(y, g=g, sn=sn, p=p, method=method)
npt.assert_allclose(np.corrcoef(res[0], c)[0, 1], 1, [.01, .1][i])
npt.assert_allclose(np.corrcoef(res[-2], s)[0, 1], 1, [.03, .3][i])
print(['\n', ''][p - 1] + ' %5s AR%d %.4fs' % (method, p, time() - t))
def foo(method, p):
t = time()
g = np.array([[.95], [1.7, -.71]][p - 1])
for i, sn in enumerate([.2, .5]): # high and low SNR
y, c, s = [a[0] for a in gen_data(g, sn)]
res = constrained_foopsi(y, g=g, sn=sn, p=p, method=method)
npt.assert_allclose(np.corrcoef(res[0], c)[0, 1], 1, [.01, .1][i])
npt.assert_allclose(np.corrcoef(res[-2], s)[0, 1], 1, [.03, .3][i])
print(['\n', ''][p - 1] + ' %5s AR%d %.4fs' % (method, p, time() - t))
def deconvolve_reconvolve(calcium,
c2spike=None,
spike2c=None,
shuffle=False,
conv_p=2,
f_saturation=0,
**kwargs):
""" Takes in 1D calcium data after filtering baseline, deconvolve, shuffle spikes (if shuffle equals
True), and then reconvolve
signal
:param calcium:
:param p:
:param c2spike:
:param kwargs:
:return:
"""
if c2spike is None:
deconv_params = {
"p": 2,
"bas_nonneg": False,
"optimize_g": 0,
"s_min": 0,
"lags": 5,
"fudge_factor": 1.
}
deconv_params.update(kwargs)
try:
c, bl, c1, g, sn, sp, lam = deconvolution.constrained_foopsi(
calcium, **deconv_params)
except:
return np.full(len(calcium), np.nan)
if shuffle:
np.random.shuffle(sp)
fluorescence_model = f"AR_{conv_p}"
spc = SpikeCalciumizer(fmodel=fluorescence_model,
fluorescence_saturation=f_saturation,
std_noise=sn,
alpha=1,
g=g,
bl=bl)
reconv = spc.binned_spikes_to_calcium(sp.reshape((1, len(sp))))
reconv = reconv.ravel()
else:
assert spike2c is not None, "specific c2spike must have corresponding spike to calcium function"
sp = c2spike(calcium)
if shuffle:
np.random.shuffle(sp)
reconv = spike2c(sp)
return reconv
def deconvolve(trace, AR_order=2):
""" Deconvolve traces using noise constrained deconvolution (Pnevmatikakis et al., 2016)
:param np.array trace: 1-d array (num_frames) with the fluorescence trace.
:param int AR_order: Order of the autoregressive process used to model the impulse
response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay.
:returns: Deconvolved spike trace.
:returns: AR coefficients (AR_order) that model the calcium response:
c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ...
"""
_, _, _, AR_coeffs, _, spike_trace, _ = deconvolution.constrained_foopsi(trace,
p=AR_order, method='cvxpy', bas_nonneg=False, fudge_factor=0.96)
# fudge_factor is a regularization term
return spike_trace, AR_coeffs
def deconvolve(trace, AR_order=2):
""" Deconvolve traces using noise constrained deconvolution (Pnevmatikakis et al., 2016)
:param np.array trace: 1-d array (num_frames) with the fluorescence trace.
:param int AR_order: Order of the autoregressive process used to model the impulse
response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay.
:returns: Deconvolved spike trace.
:returns: AR coefficients (AR_order) that model the calcium response:
c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ...
"""
_, _, _, AR_coeffs, _, spike_trace, _ = deconvolution.constrained_foopsi(trace,
p=AR_order, method='cvxpy', bas_nonneg=False, fudge_factor=0.96)
# fudge_factor is a regularization term
return spike_trace, AR_coeffs
def wiener_deconv_test(y, diff=False):
c, bl, c1, g, sn, sp, lam = deconvolution.constrained_foopsi(
y, p=2, bas_nonneg=False, s_min=0)
conv = np.array([1] + list(-g))
h0 = inverse_kernel(conv, N=len(y), fft=True)
x_hat, hhat = wiener_deconvolution(y, h0)
fig, axes = plt.subplots(nrows=3, ncols=1, sharex=True)
if diff:
axes[0].plot(np.vstack((np.diff(y, prepend=0), sp, x_hat)).T)
axes[0].legend(['df/dt', 'deconv', 'wiener'])
else:
axes[0].plot(np.vstack((sp, x_hat)).T)
axes[0].legend(['deconv', 'wiener'])
axes[1].plot(y)
axes[1].set_ylabel('dff')
axes[2].plot(h0)
axes[2].set_ylabel("IRF")
def genDeconv(out_dir, ratio):
c = np.zeros((ratio.shape[0], ratio.shape[1]))
sp = np.zeros((ratio.shape[0], ratio.shape[1]))
#r2 = np.transpose(ratio).flatten()
#t = time.time()
#print('Generating deconvolution')
#(c2, bl, c1, g, sn, sp2, lam) = deconv.constrained_foopsi(r2, p=2, verbosity=True, method_deconvolution='cvxpy')
#print('Took {}'.format(time.time() - t))
#c = np.reshape(c2, (ratio.shape[0], ratio.shape[1]))
#sp = np.reshape(sp2, (ratio.shape[0], ratio.shape[1]))
for i in range(ratio.shape[1]):
print('Generating deconvolution for neuron {}'.format(i+1))
t = time.time()
(c[:, i], bl, c1, g, sn, sp[:, i], lam) = deconv.constrained_foopsi(ratio[:, i], p=2, verbosity=True, method_deconvolution='oasis')
print('Took {}'.format(time.time() - t))
#np.fft.restore_all()
#for i in range(ratio.shape[1]):
# (sp, c, params) = deconv.spike_inference(ratio[:, i], mode='psd')
savemat(os.path.join(out_dir, 'deconv.mat'), {'deconv': sp}, do_compression=True)
savemat(os.path.join(out_dir, 'ratio_model.mat'), {'ratio_model': c}, do_compression=True)
if not os.path.exists(plots):
os.makedirs(plots)
number_planes_total = 6
C = np.array(f['C'])
dff = np.array(f['dff'])
blen = f.attrs['blen']
nerden = f['nerden']
ens_neur = np.array(f['ens_neur'])
online_data = pd.read_csv(csvf)
units = len(ens_neur)
online = online_data.iloc[:, 2:2 + units].values.T
online[np.isnan(online)] = 0
frames = online_data['frameNumber'].values // number_planes_total + blen
tests = [deconvolution.constrained_foopsi(dff[i], p=2) for i in range(dff.shape[0])]
ts = np.vstack([t[0] for t in tests])
nts = ts[nerden]
ndff = dff[nerden]
nC = C[nerden]
allcorrs = [np.corrcoef(ts[i], dff[i])[0, 1] for i in range(ts.shape[0])]
allcorrs = np.array(allcorrs)
nacorrs = allcorrs[nerden]
dff_ens = dff[ens_neur]
dff_ens_p = dff_ens[:, frames]
C_ens = C[ens_neur]
C_ens_p = C_ens[:, frames]
ts_ens = ts[ens_neur]
ts_ens_p = ts_ens[:, frames]
def myfun(x):
from caiman.source_extraction.cnmf.deconvolution import constrained_foopsi
dc = constrained_foopsi(*x)
return (dc[0], dc[5])
def dff_test(root, animal, day, dff_func):
f = h5py.File(
root +
"processed/{}/full_{}_{}__data.hdf5".format(animal, animal, day))
csvf = root + "raw/{}/{}/bmi_IntegrationRois_00001.csv".format(animal, day)
plots = "/Users/albertqu/Documents/7.Research/BMI/plots/caiman_test/corrDFFdoublepass_{}_{}".format(
animal, day)
if not os.path.exists(plots):
os.makedirs(plots)
number_planes_total = 6
C = np.array(f['C'])
dff = dff_func(f)
blen = f.attrs['blen']
nerden = f['nerden']
ens_neur = np.array(f['ens_neur'], dtype=np.int16)
online_data = pd.read_csv(csvf)
units = len(ens_neur)
online = online_data.iloc[:, 2:2 + units].values.T
online[np.isnan(online)] = 0
frames = online_data['frameNumber'].values // number_planes_total + blen
tests = [
deconvolution.constrained_foopsi(dff[i], p=2)
for i in range(dff.shape[0])
]
ts = np.vstack([t[0] for t in tests])
nts = ts[nerden]
ndff = dff[nerden]
nC = C[nerden]
allcorrs = [np.corrcoef(ts[i], dff[i])[0, 1] for i in range(ts.shape[0])]
allcorrs = np.array(allcorrs)
nacorrs = allcorrs[nerden]
dff_ens = dff[ens_neur]
dff_ens_p = dff_ens[:, frames]
C_ens = C[ens_neur]
C_ens_p = C_ens[:, frames]
ts_ens = ts[ens_neur]
ts_ens_p = ts_ens[:, frames]
plt.hist([allcorrs, nacorrs], density=True)
plt.legend(['allcorrs', 'neuron_only'])
plt.title(
'Correlation distribution of dff and the double(on dff) pass inferred C'
)
plt.xlabel('R')
plt.ylabel('Relative Freq')
plt.show()
fname = os.path.join(plots, "doublePassDFF_dff_corr")
plt.savefig(fname + '.png')
plt.savefig(fname + '.eps')
plt.close('all')
"""
animal, day = 'IT2', '181115'
In [80]: %time tests = [deconvolution.constrained_foopsi(dff[i], p=2) for i in range(dff.shape[0])]
CPU times: user 18min 43s, sys: 4min 44s, total: 23min 28s
Wall time: 6min 30s
In [164]: dffR
Out[164]: (array([0.95562897, 0.94901895, 0.80737658, 0.89138115]), 0.017843456512294558)
In [165]: doubleCR
Out[165]: (array([0.91732025, 0.90276186, 0.4199655 , 0.89416271]), 0.021650326709414264)
In [166]: CR
Out[166]: (array([0.95621973, 0.92302353, 0.46681481, 0.98135983]), 0.02671477859889339)
"""
fig, axes = plt.subplots(nrows=2, ncols=1, sharex=True, figsize=(15, 15))
i = 0
axes[0].plot(
np.vstack(
[zscore(dff_ens_p[i]),
zscore(online[i]),
zscore(ts_ens_p[i])]).T)
axes[0].plot(zscore(C_ens_p[i]), c='k')
axes[0].legend(['dff', 'online raw', 'double pass C', 'C'])
axes[0].set_title('zscore')
axes[1].plot(zscore(C_ens_p[i]), c='k')
axes[1].plot(np.vstack([dff_ens_p[i], online[i], ts_ens_p[i]]).T)
axes[1].legend(['C', 'dff', 'online raw', 'double pass C'])
axes[1].set_title('Raw No zscore')
def deconv_fano_spikefinder(dataset,
fano,
p=2,
W=None,
T=100,
binT=1,
sample_deconv=True,
outpath=None):
dataset = os.path.join(dataset, '{}')
if fano == 'raw':
fano_metric = neuron_fano
elif fano == 'norm_pre':
fano_metric = lambda *args: neuron_fano_norm(*args, pre=True)
else:
fano_metric = lambda *args: neuron_fano_norm(*args, pre=False)
measures = {'spike': {}, 'calcium': {}, 'deconv_corr': {}}
for i in range(1, 11):
print(i)
calcium_train = pd.read_csv(dataset.format(i) + '.train.calcium.csv')
spikes_train = pd.read_csv(dataset.format(i) + '.train.spikes.csv')
neurons = spikes_train.columns
measures['deconv_corr'][i] = np.zeros(len(neurons))
for m in measures.keys():
if m != 'deconv_corr':
measures[m][i] = {}
measures[m][i]['neurons'] = neurons
measures[m][i]['fano'] = np.zeros(len(neurons))
for n in neurons:
spike, calcium = spikes_train[n], calcium_train[n]
nonnan = ~np.isnan(spike)
fano_spike = neuron_fano(np.array(spike[nonnan]), W, T)
deconv = deconvolution.constrained_foopsi(np.array(
calcium[nonnan]),
p=p)[5]
corr = np.corrcoef(deconv, spike[nonnan])[0, 1]
if outpath:
fano_record = np.around(fano_spike, 4)
deconv_ptv = deconv[~np.isclose(deconv, 0)]
if binT > 1:
r, c = len(deconv) // binT, binT
r_p, c_p = len(deconv_ptv) // binT, binT
deconv_bin = np.sum(deconv[:r * c].reshape((r, c)),
axis=1).ravel()
deconv_ptv_bin = np.sum(deconv_ptv[:r_p * c_p].reshape(
(r_p, c_p)),
axis=1).ravel()
else:
deconv_bin, deconv_ptv_bin = deconv, deconv_ptv
bsize1 = best_nbins(deconv_bin)
bsize2 = best_nbins(deconv_ptv_bin)
plt.subplots_adjust(bottom=0.1, wspace=0.3, hspace=0.5)
plt.subplot(211)
plt.hist(deconv_bin, bins=bsize1)
plt.title('Deconv All')
plt.subplot(212)
plt.hist(deconv_ptv_bin, bins=bsize2)
plt.title('Deconv Positive')
plt.suptitle('{}_{} #{} Neuron {}, Fano: {}'.format(
fano, p, i, n, fano_record))
savepath = os.path.join(outpath,
'distribution_binT_{}'.format(binT),
"{}_T{}_W{}_p{}".format(fano, T, W, p))
if not os.path.exists(savepath):
os.makedirs(savepath)
plt.savefig(
os.path.join(
savepath,
"spikefinder_{}_neuron{}_fano_{}_corr_{}.png".format(
i, n, fano_record, np.around(corr, 4))))
plt.close('all')
if sample_deconv:
fig, axes = plt.subplots(nrows=3,
ncols=1,
sharex=True,
figsize=(20, 10))
axes[0].plot(spike[:300])
axes[0].legend('spike')
axes[1].plot(calcium[:300])
axes[1].legend('calcium')
axes[2].plot(deconv[:300])
axes[2].legend('deconv')
plt.savefig(savepath +
'/signal_{}_neuron{}_corr_{}.png'.format(
i, n, np.around(corr, 4)))
plt.close('all')
fano_calcium = fano_metric(deconv, W, T)
measures['spike'][i]['fano'][int(n)] = fano_spike
measures['calcium'][i]['fano'][int(n)] = fano_calcium
measures['deconv_corr'][i][int(n)] = corr
print(int(n), fano_spike, fano_calcium)
return measures
