def deconv_test1(sp1=None, impulse=True): g = np.array([0.87797883, -0.10934919]) if impulse: sp1 = impulse_model_test(sp1, show=False) else: # simulated slow ramp test sp1 = get_sample_spikes_slow_ramp() if sp1 is None else sp1 spc = SpikeCalciumizer(fmodel="AR_2", fluorescence_saturation=0, std_noise=0.02, alpha=1, g=g) rec = spc.binned_spikes_to_calcium(sp1.reshape((1, -1))) c2, bl2, c12, g2, sn2, sp2, lam2 = deconvolution.constrained_foopsi( rec.ravel(), p=2, bl=0, bas_nonneg=False, s_min=0) c2, bl2, c12, g2, sn2, sp3, lam2 = deconvolution.constrained_foopsi( rec.ravel(), p=2, bas_nonneg=False, s_min=0) conv = np.array([1] + list(-g2)) h0 = inverse_kernel(conv, N=rec.shape[1], fft=True) x_hat, hhat = wiener_deconvolution(rec.ravel(), h0) fig, axes = plt.subplots(nrows=5, ncols=1, sharex=True) axes[0].plot(sp1) axes[1].plot(sp3) axes[2].plot(sp2) axes[3].plot(x_hat) axes[4].plot(rec.ravel()) axes[0].set_ylabel("truth spike") axes[1].set_ylabel("bl_auto") axes[2].set_ylabel('bl=0') axes[3].set_ylabel("wiener") axes[4].set_ylabel("calcium")
def deconvolve_detrended(trace, scan_fps, detrend_period=600, AR_order=2): """Same as the the `deconvolve` method, except that the fluorescence trace is detrended before autoregressive modeling :param np.array trace: 1-d array (num_frames) with the fluorescence trace. :param float scan_fps: fps of the scan :param float detrend_period: number of seconds over which percentiles are computed :param int AR_order: Order of the autoregressive process used to model the impulse response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay. :returns: Deconvolved spike trace. :returns: AR coefficients (AR_order) that model the calcium response: c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ... """ detrend_window = int(round(detrend_period * scan_fps)) n_chunks = len(trace) // detrend_window if detrend_window > 0 and n_chunks > 0: chunks_len = n_chunks * detrend_window trace_chunks = trace[:chunks_len].reshape(-1, detrend_window) data_prct = df_percentile(trace_chunks, axis=1)[0].mean() trace = trace - percentile_filter(trace, data_prct, detrend_window) _, _, _, AR_coeffs, _, spike_trace, _ = deconvolution.constrained_foopsi(trace, p=AR_order, method='cvxpy', bas_nonneg=False, fudge_factor=0.96) return spike_trace, AR_coeffs
def deconvolve_reconvolve_test(dff, deconv_p=2, conv_p=2, optimize_g=0, s_min=0, lags=5, fudge_factor=1., alpha_est=None, f_saturation=0, SN=None, show=True, tag="", save=None): # optimize_g, s_min # s_min = 0, None c, bl, c1, g, sn, sp, lam = deconvolution.constrained_foopsi( dff, p=deconv_p, bas_nonneg=False, optimize_g=optimize_g, s_min=s_min, lags=lags, fudge_factor=fudge_factor) if alpha_est is None: alpha = 1 else: alpha = alpha_est fluorescence_model = f"AR_{conv_p}" spc = SpikeCalciumizer(fmodel=fluorescence_model, fluorescence_saturation=f_saturation, std_noise=sn if SN is None else SN, alpha=alpha, g=g, bl=bl) reconv = spc.binned_spikes_to_calcium(sp.reshape((1, len(sp)))) reconv = reconv.ravel() spc.std_noise = 0 reconv_clean = spc.binned_spikes_to_calcium(sp.reshape((1, len(sp)))) reconv_clean = reconv_clean.ravel() # PLOTTING if show or save is not None: fig, axes = plt.subplots(nrows=2, ncols=1, sharex=True) xs = np.arange(1, len(reconv) + 1) axes[0].plot(xs, np.vstack([dff, reconv]).T) axes[0].legend(['Original', 'Simulated']) axes[1].plot(xs, sp) axes[1].set_xlabel('frames') fig.suptitle(f"Reconvolve Neuron Test {tag}") if show: plt.show() if save is not None: fname = os.path.join(save, f'reconvolve_{tag}') fig.savefig(fname + '.png') fig.savefig(fname + '.eps') # include bl for better calculation return c, c1, sp, sn, reconv, reconv_clean
def foo(method, p): t = time() g = np.array([[.95], [1.7, -.71]][p - 1]) for i, sn in enumerate([.2, .5]): # high and low SNR y, c, s = [a[0] for a in gen_data(g, sn)] res = constrained_foopsi(y, g=g, sn=sn, p=p, method=method) npt.assert_allclose(np.corrcoef(res[0], c)[0, 1], 1, [.01, .1][i]) npt.assert_allclose(np.corrcoef(res[-2], s)[0, 1], 1, [.03, .3][i]) print(['\n', ''][p - 1] + ' %5s AR%d %.4fs' % (method, p, time() - t))
def foo(method, p): t = time() g = np.array([[.95], [1.7, -.71]][p - 1]) for i, sn in enumerate([.2, .5]): # high and low SNR y, c, s = [a[0] for a in gen_data(g, sn)] res = constrained_foopsi(y, g=g, sn=sn, p=p, method=method) npt.assert_allclose(np.corrcoef(res[0], c)[0, 1], 1, [.01, .1][i]) npt.assert_allclose(np.corrcoef(res[-2], s)[0, 1], 1, [.03, .3][i]) print(['\n', ''][p - 1] + ' %5s AR%d %.4fs' % (method, p, time() - t))
def deconvolve_reconvolve(calcium, c2spike=None, spike2c=None, shuffle=False, conv_p=2, f_saturation=0, **kwargs): """ Takes in 1D calcium data after filtering baseline, deconvolve, shuffle spikes (if shuffle equals True), and then reconvolve signal :param calcium: :param p: :param c2spike: :param kwargs: :return: """ if c2spike is None: deconv_params = { "p": 2, "bas_nonneg": False, "optimize_g": 0, "s_min": 0, "lags": 5, "fudge_factor": 1. } deconv_params.update(kwargs) try: c, bl, c1, g, sn, sp, lam = deconvolution.constrained_foopsi( calcium, **deconv_params) except: return np.full(len(calcium), np.nan) if shuffle: np.random.shuffle(sp) fluorescence_model = f"AR_{conv_p}" spc = SpikeCalciumizer(fmodel=fluorescence_model, fluorescence_saturation=f_saturation, std_noise=sn, alpha=1, g=g, bl=bl) reconv = spc.binned_spikes_to_calcium(sp.reshape((1, len(sp)))) reconv = reconv.ravel() else: assert spike2c is not None, "specific c2spike must have corresponding spike to calcium function" sp = c2spike(calcium) if shuffle: np.random.shuffle(sp) reconv = spike2c(sp) return reconv
def deconvolve(trace, AR_order=2): """ Deconvolve traces using noise constrained deconvolution (Pnevmatikakis et al., 2016) :param np.array trace: 1-d array (num_frames) with the fluorescence trace. :param int AR_order: Order of the autoregressive process used to model the impulse response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay. :returns: Deconvolved spike trace. :returns: AR coefficients (AR_order) that model the calcium response: c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ... """ _, _, _, AR_coeffs, _, spike_trace, _ = deconvolution.constrained_foopsi(trace, p=AR_order, method='cvxpy', bas_nonneg=False, fudge_factor=0.96) # fudge_factor is a regularization term return spike_trace, AR_coeffs
def deconvolve(trace, AR_order=2): """ Deconvolve traces using noise constrained deconvolution (Pnevmatikakis et al., 2016) :param np.array trace: 1-d array (num_frames) with the fluorescence trace. :param int AR_order: Order of the autoregressive process used to model the impulse response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay. :returns: Deconvolved spike trace. :returns: AR coefficients (AR_order) that model the calcium response: c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ... """ _, _, _, AR_coeffs, _, spike_trace, _ = deconvolution.constrained_foopsi(trace, p=AR_order, method='cvxpy', bas_nonneg=False, fudge_factor=0.96) # fudge_factor is a regularization term return spike_trace, AR_coeffs
def wiener_deconv_test(y, diff=False): c, bl, c1, g, sn, sp, lam = deconvolution.constrained_foopsi( y, p=2, bas_nonneg=False, s_min=0) conv = np.array([1] + list(-g)) h0 = inverse_kernel(conv, N=len(y), fft=True) x_hat, hhat = wiener_deconvolution(y, h0) fig, axes = plt.subplots(nrows=3, ncols=1, sharex=True) if diff: axes[0].plot(np.vstack((np.diff(y, prepend=0), sp, x_hat)).T) axes[0].legend(['df/dt', 'deconv', 'wiener']) else: axes[0].plot(np.vstack((sp, x_hat)).T) axes[0].legend(['deconv', 'wiener']) axes[1].plot(y) axes[1].set_ylabel('dff') axes[2].plot(h0) axes[2].set_ylabel("IRF")
def genDeconv(out_dir, ratio): c = np.zeros((ratio.shape[0], ratio.shape[1])) sp = np.zeros((ratio.shape[0], ratio.shape[1])) #r2 = np.transpose(ratio).flatten() #t = time.time() #print('Generating deconvolution') #(c2, bl, c1, g, sn, sp2, lam) = deconv.constrained_foopsi(r2, p=2, verbosity=True, method_deconvolution='cvxpy') #print('Took {}'.format(time.time() - t)) #c = np.reshape(c2, (ratio.shape[0], ratio.shape[1])) #sp = np.reshape(sp2, (ratio.shape[0], ratio.shape[1])) for i in range(ratio.shape[1]): print('Generating deconvolution for neuron {}'.format(i+1)) t = time.time() (c[:, i], bl, c1, g, sn, sp[:, i], lam) = deconv.constrained_foopsi(ratio[:, i], p=2, verbosity=True, method_deconvolution='oasis') print('Took {}'.format(time.time() - t)) #np.fft.restore_all() #for i in range(ratio.shape[1]): # (sp, c, params) = deconv.spike_inference(ratio[:, i], mode='psd') savemat(os.path.join(out_dir, 'deconv.mat'), {'deconv': sp}, do_compression=True) savemat(os.path.join(out_dir, 'ratio_model.mat'), {'ratio_model': c}, do_compression=True)
if not os.path.exists(plots): os.makedirs(plots) number_planes_total = 6 C = np.array(f['C']) dff = np.array(f['dff']) blen = f.attrs['blen'] nerden = f['nerden'] ens_neur = np.array(f['ens_neur']) online_data = pd.read_csv(csvf) units = len(ens_neur) online = online_data.iloc[:, 2:2 + units].values.T online[np.isnan(online)] = 0 frames = online_data['frameNumber'].values // number_planes_total + blen tests = [deconvolution.constrained_foopsi(dff[i], p=2) for i in range(dff.shape[0])] ts = np.vstack([t[0] for t in tests]) nts = ts[nerden] ndff = dff[nerden] nC = C[nerden] allcorrs = [np.corrcoef(ts[i], dff[i])[0, 1] for i in range(ts.shape[0])] allcorrs = np.array(allcorrs) nacorrs = allcorrs[nerden] dff_ens = dff[ens_neur] dff_ens_p = dff_ens[:, frames] C_ens = C[ens_neur] C_ens_p = C_ens[:, frames] ts_ens = ts[ens_neur] ts_ens_p = ts_ens[:, frames]
def myfun(x): from caiman.source_extraction.cnmf.deconvolution import constrained_foopsi dc = constrained_foopsi(*x) return (dc[0], dc[5])
def dff_test(root, animal, day, dff_func): f = h5py.File( root + "processed/{}/full_{}_{}__data.hdf5".format(animal, animal, day)) csvf = root + "raw/{}/{}/bmi_IntegrationRois_00001.csv".format(animal, day) plots = "/Users/albertqu/Documents/7.Research/BMI/plots/caiman_test/corrDFFdoublepass_{}_{}".format( animal, day) if not os.path.exists(plots): os.makedirs(plots) number_planes_total = 6 C = np.array(f['C']) dff = dff_func(f) blen = f.attrs['blen'] nerden = f['nerden'] ens_neur = np.array(f['ens_neur'], dtype=np.int16) online_data = pd.read_csv(csvf) units = len(ens_neur) online = online_data.iloc[:, 2:2 + units].values.T online[np.isnan(online)] = 0 frames = online_data['frameNumber'].values // number_planes_total + blen tests = [ deconvolution.constrained_foopsi(dff[i], p=2) for i in range(dff.shape[0]) ] ts = np.vstack([t[0] for t in tests]) nts = ts[nerden] ndff = dff[nerden] nC = C[nerden] allcorrs = [np.corrcoef(ts[i], dff[i])[0, 1] for i in range(ts.shape[0])] allcorrs = np.array(allcorrs) nacorrs = allcorrs[nerden] dff_ens = dff[ens_neur] dff_ens_p = dff_ens[:, frames] C_ens = C[ens_neur] C_ens_p = C_ens[:, frames] ts_ens = ts[ens_neur] ts_ens_p = ts_ens[:, frames] plt.hist([allcorrs, nacorrs], density=True) plt.legend(['allcorrs', 'neuron_only']) plt.title( 'Correlation distribution of dff and the double(on dff) pass inferred C' ) plt.xlabel('R') plt.ylabel('Relative Freq') plt.show() fname = os.path.join(plots, "doublePassDFF_dff_corr") plt.savefig(fname + '.png') plt.savefig(fname + '.eps') plt.close('all') """ animal, day = 'IT2', '181115' In [80]: %time tests = [deconvolution.constrained_foopsi(dff[i], p=2) for i in range(dff.shape[0])] CPU times: user 18min 43s, sys: 4min 44s, total: 23min 28s Wall time: 6min 30s In [164]: dffR Out[164]: (array([0.95562897, 0.94901895, 0.80737658, 0.89138115]), 0.017843456512294558) In [165]: doubleCR Out[165]: (array([0.91732025, 0.90276186, 0.4199655 , 0.89416271]), 0.021650326709414264) In [166]: CR Out[166]: (array([0.95621973, 0.92302353, 0.46681481, 0.98135983]), 0.02671477859889339) """ fig, axes = plt.subplots(nrows=2, ncols=1, sharex=True, figsize=(15, 15)) i = 0 axes[0].plot( np.vstack( [zscore(dff_ens_p[i]), zscore(online[i]), zscore(ts_ens_p[i])]).T) axes[0].plot(zscore(C_ens_p[i]), c='k') axes[0].legend(['dff', 'online raw', 'double pass C', 'C']) axes[0].set_title('zscore') axes[1].plot(zscore(C_ens_p[i]), c='k') axes[1].plot(np.vstack([dff_ens_p[i], online[i], ts_ens_p[i]]).T) axes[1].legend(['C', 'dff', 'online raw', 'double pass C']) axes[1].set_title('Raw No zscore')
def deconv_fano_spikefinder(dataset, fano, p=2, W=None, T=100, binT=1, sample_deconv=True, outpath=None): dataset = os.path.join(dataset, '{}') if fano == 'raw': fano_metric = neuron_fano elif fano == 'norm_pre': fano_metric = lambda *args: neuron_fano_norm(*args, pre=True) else: fano_metric = lambda *args: neuron_fano_norm(*args, pre=False) measures = {'spike': {}, 'calcium': {}, 'deconv_corr': {}} for i in range(1, 11): print(i) calcium_train = pd.read_csv(dataset.format(i) + '.train.calcium.csv') spikes_train = pd.read_csv(dataset.format(i) + '.train.spikes.csv') neurons = spikes_train.columns measures['deconv_corr'][i] = np.zeros(len(neurons)) for m in measures.keys(): if m != 'deconv_corr': measures[m][i] = {} measures[m][i]['neurons'] = neurons measures[m][i]['fano'] = np.zeros(len(neurons)) for n in neurons: spike, calcium = spikes_train[n], calcium_train[n] nonnan = ~np.isnan(spike) fano_spike = neuron_fano(np.array(spike[nonnan]), W, T) deconv = deconvolution.constrained_foopsi(np.array( calcium[nonnan]), p=p)[5] corr = np.corrcoef(deconv, spike[nonnan])[0, 1] if outpath: fano_record = np.around(fano_spike, 4) deconv_ptv = deconv[~np.isclose(deconv, 0)] if binT > 1: r, c = len(deconv) // binT, binT r_p, c_p = len(deconv_ptv) // binT, binT deconv_bin = np.sum(deconv[:r * c].reshape((r, c)), axis=1).ravel() deconv_ptv_bin = np.sum(deconv_ptv[:r_p * c_p].reshape( (r_p, c_p)), axis=1).ravel() else: deconv_bin, deconv_ptv_bin = deconv, deconv_ptv bsize1 = best_nbins(deconv_bin) bsize2 = best_nbins(deconv_ptv_bin) plt.subplots_adjust(bottom=0.1, wspace=0.3, hspace=0.5) plt.subplot(211) plt.hist(deconv_bin, bins=bsize1) plt.title('Deconv All') plt.subplot(212) plt.hist(deconv_ptv_bin, bins=bsize2) plt.title('Deconv Positive') plt.suptitle('{}_{} #{} Neuron {}, Fano: {}'.format( fano, p, i, n, fano_record)) savepath = os.path.join(outpath, 'distribution_binT_{}'.format(binT), "{}_T{}_W{}_p{}".format(fano, T, W, p)) if not os.path.exists(savepath): os.makedirs(savepath) plt.savefig( os.path.join( savepath, "spikefinder_{}_neuron{}_fano_{}_corr_{}.png".format( i, n, fano_record, np.around(corr, 4)))) plt.close('all') if sample_deconv: fig, axes = plt.subplots(nrows=3, ncols=1, sharex=True, figsize=(20, 10)) axes[0].plot(spike[:300]) axes[0].legend('spike') axes[1].plot(calcium[:300]) axes[1].legend('calcium') axes[2].plot(deconv[:300]) axes[2].legend('deconv') plt.savefig(savepath + '/signal_{}_neuron{}_corr_{}.png'.format( i, n, np.around(corr, 4))) plt.close('all') fano_calcium = fano_metric(deconv, W, T) measures['spike'][i]['fano'][int(n)] = fano_spike measures['calcium'][i]['fano'][int(n)] = fano_calcium measures['deconv_corr'][i][int(n)] = corr print(int(n), fano_spike, fano_calcium) return measures