def test_stationary_mean(self):
for ele_num in range(0,40,10):
for fitfunc_name in ["exponential_with_bias", "complex"]:
name_data = "./data/activity_mat_{}.pickled".format(ele_num)
with self.subTest(fitfunc_name = fitfunc_name, data_file = name_data):
k_arr = np.arange(7, 1500, 1)
activity_mat = pickle.load(open(name_data, "rb"))
corr_arr = calc_corr_arr_stationary(activity_mat, k_arr)
if fitfunc_name == "complex":
popt,_ = calc_popt(fitfunction_complex, k_arr, corr_arr, np.ones_like(k_arr), self.bounds_compl, self.startvalues_compl, maxfev=None,
try_only_once=False)
rk = mre.coefficients(activity_mat, method='stationarymean')
res_mre = mre.correlation_fit(rk, fitfunc='complex')
print('popts:', popt, res_mre.popt)
for i in range(len(popt)):
self.assertTrue(test_similarity(popt[i], res_mre.popt[i], ratio_different=1e-5))
#plt.plot(k_arr, corr_arr)
#plt.plot(k_arr, fitfunction_complex(k_arr, *popt))
#plt.show()
elif fitfunc_name == "exponential_with_bias":
popt,_ = calc_popt(fitfunction_exp_with_offset, k_arr, corr_arr, np.ones_like(k_arr), self.bounds_exp_with_offset,
self.startvalues_exp_with_offset, maxfev=None,
try_only_once=False)
#plt.plot(k_arr, corr_arr)
#plt.plot(k_arr, fitfunction_exp_with_offset(k_arr, *popt))
#plt.show()
rk = mre.coefficients(activity_mat, method='stationarymean')
res_mre = mre.correlation_fit(rk, fitfunc='exponentialoffset')
print('popts:', popt, res_mre.popt)
for i in range(len(popt)):
self.assertTrue(test_similarity(popt[i], res_mre.popt[i], ratio_different=1e-5))
def test_with_spike_rec():
directory = "../data/spikefinder_train/"
dataset = "8"
calcium = np.genfromtxt(directory + dataset + '.train.calcium.csv',
delimiter=",",
skip_header=1).transpose()
spikes = np.genfromtxt(directory + dataset + '.train.spikes.csv',
delimiter=",",
skip_header=1).transpose()
t = np.arange(calcium.shape[1]) / 100.0
#plt.plot(t, calcium[2])
#plt.plot(t, spikes[2])
#plt.show()
spks_deconv = np.empty_like(spikes) * np.nan
for i, arr in enumerate(tqdm(calcium)):
spks_tmp = deconvolve(arr[~np.isnan(arr)][None, :], 100, tau=1.5)[0]
spks_deconv[i, :len(spks_tmp)] = spks_tmp
k_arr = np.arange(1, 100)
coefficients = np.zeros_like(k_arr, dtype=float)
for i, arr in enumerate(tqdm(spikes[:])):
coeff_res = mre.coefficients(arr[~np.isnan(arr)], k_arr)
coefficients += coeff_res.coefficients
coefficients_deconv = np.zeros_like(k_arr, dtype=float)
for i, arr in enumerate(tqdm(spks_deconv[:])):
coeff_res = mre.coefficients(arr[~np.isnan(arr)], k_arr)
coefficients_deconv += coeff_res.coefficients
print(coefficients)
f = plt.figure(figsize=(4.3, 3))
t = np.arange(0, len(coefficients)) / 100
plt.plot(t,
coefficients / len(spikes),
label="from spikes, 'ground truth'")
plt.plot(t,
coefficients_deconv / len(spks_deconv) / 8.5,
label='from deconvolved calcium imaging')
plt.xlabel("time interval $\Delta T$ [s]")
plt.ylabel("autocorrelation (normed)")
plt.tick_params(
axis='y', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
left=False, # ticks along the bottom edge are off
right=False, # ticks along the top edge are off
labelleft=False) # labels along the bottom edge are off
plt.legend()
plt.tight_layout()
plt.savefig(
"../reports/estimating_timescales/figures/comparison3_normed.pdf")
plt.show()
def branching_ratio(spikes, bin_w):
GExc_spks = spikes
#with open(bpath + '/raw/gexc_spks.p', 'rb') as pfile:
# GExc_spks = pickle.load(pfile)
# with open(bpath+'/raw/ginh_spks.p', 'rb') as pfile:
# GInh_spks = pickle.load(pfile)
#ts = GExc_spks['t'] / ms
#ts = ts[ts > (nsp['T1'] + nsp['T2'] + nsp['T3'] + nsp['T4']) / ms]
#ts = ts - (nsp['T1'] + nsp['T2'] + nsp['T3'] + nsp['T4']) / ms
#assert (np.min(ts) >= 0)
bins = np.arange(0, (len(spikes) + bin_w), bin_w)
counts, bins = np.histogram(GExc_spks, bins=bins)
print(np.shape(counts))
# counts = np.reshape(counts, (len(counts),1))
# print(np.shape(counts))
rk = mre.coefficients(counts, dt=bin_w, dtunit='ms') #, desc=''
ft = mre.fit(rk)
return rk, ft, (counts, bins)
def branching_ratio(ax, bpath, nsp, bin_w):
with open(bpath+'/raw/gexc_spks.p', 'rb') as pfile:
GExc_spks = pickle.load(pfile)
# with open(bpath+'/raw/ginh_spks.p', 'rb') as pfile:
# GInh_spks = pickle.load(pfile)
ts = GExc_spks['t']/ms
ts = ts[ts>(nsp['T1']+nsp['T2']+nsp['T3']+nsp['T4'])/ms]
ts = ts - (nsp['T1']+nsp['T2']+nsp['T3']+nsp['T4'])/ms
assert(np.min(ts) >= 0)
bins = np.arange(0, (nsp['T5']+bin_w)/ms, bin_w/ms)
counts, bins = np.histogram(ts, bins=bins, density=False)
print(np.shape(counts))
# counts = np.reshape(counts, (len(counts),1))
# print(np.shape(counts))
rk = mre.coefficients(counts, dt=bin_w/ms, dtunit='ms', desc='')
ft = mre.fit(rk)
return rk, ft, (counts, bins)
def test_separate(self):
print("\nTesting trial separated correlation coefficients: \n")
for ele_num in range(0, 40, 10):
name_data = "./data/activity_mat_{}.pickled".format(ele_num)
activity_mat = pickle.load(open(name_data, "rb"))
activity_mat = activity_mat.astype(dtype="float64")
k_arr = np.arange(7, 1500, 1)
corr_arr = calc_corr_mat_separate(activity_mat, k_arr)
time_beg = time.time()
numboot = 100
mre_res = mre.coefficients(activity_mat,
steps=k_arr,
method='trialseparated',
numboot=numboot)
print('trialseparated, time needed: {:.2f} ms'.format(
(time.time() - time_beg) * 1000))
print('mre: ', mre_res.coefficients[:5])
print('true value: ', corr_arr[:5])
self.assertTrue(
test_similarity(mre_res.coefficients,
corr_arr,
ratio_different=1e-10))
bootstrap_mat = np.array(
[boot.coefficients for boot in mre_res.bootstrapcrs])
mean_bootstrap = np.mean(bootstrap_mat, axis=0)
print("boot mean: ", mean_bootstrap[:5])
self.assertTrue(
test_similarity_abs(mre_res.coefficients,
np.mean(bootstrap_mat, axis=0),
max_difference=0.04 / np.sqrt(numboot)))
def get_auto_correlation_time(spikes, min_steps, max_steps, bin_size_ms):
rk = mre.coefficients(spikes, dt=bin_size_ms, steps=(min_steps, max_steps))
T = (rk.steps - rk.steps[0] + 1) * bin_size_ms
fit = mre.fit(rk, steps=(min_steps, max_steps), fitfunc=mre.f_exponential)
tau_C = fit.tau
C_raw = rk.coefficients
range = int(6 * tau_C)
tau_C_int = plots.get_T_avg(T[:range + 1], C_raw[:range + 1], 0)
print(tau_C, tau_C_int)
return tau_C_int, rk, T, fit
def get_auto_correlation_time(counts_from_spikes, min_steps, max_steps,
bin_size_ms):
rk = mre.coefficients(counts_from_spikes,
dt=bin_size_ms,
steps=(min_steps, max_steps))
fit = mre.fit(rk,
steps=(min_steps, max_steps),
fitfunc=mre.f_exponential_offset)
tau_C = fit.tau
return tau_C, fit
def plot_map_tau():
directory = "../data/Spontaneous/suite2p/plane0/"
# load traces and subtract neuropil
iscell = np.load(directory + "iscell.npy")
F = np.load(directory + "F.npy")
Fneu = np.load(directory + "Fneu.npy")
Fc = F - 1 * Fneu
stat = np.load(directory + 'stat.npy', allow_pickle=True)
x_pos = np.array([np.median(neur["xpix"]) for neur in stat])
y_pos = np.array([np.median(neur["ypix"]) for neur in stat])
mask_iscell = iscell[:, 1] > 0.5
Fc = Fc[mask_iscell, :]
Fc = Fc[:]
x_pos = x_pos[mask_iscell]
y_pos = y_pos[mask_iscell]
x_pos = x_pos[:]
y_pos = y_pos[:]
Fc_conv = Fc
spks = deconvolve(Fc_conv, fs=30, tau=1.5)
t = np.linspace(0, Fc_conv.shape[1] / 30, Fc_conv.shape[1])
k_arr = np.arange(1, 40)
taus = []
for i, arr in enumerate(tqdm(spks[:])):
coeff_res = mre.coefficients(arr, k_arr, dt=1 / 30 * 1000)
plt.plot(coeff_res.coefficients)
plt.show()
fit_result = mre.fit(coeff_res, steps=np.arange(1, 40))
taus.append(fit_result.tau)
cm = plt.cm.get_cmap("inferno")
sc = plt.scatter(x_pos,
y_pos,
c=taus,
vmin=0,
vmax=300,
s=35,
cmap=cm,
norm=colors.PowerNorm(0.7, 10, 300))
plt.xlabel("x position")
plt.ylabel("y position")
cbar = plt.colorbar(sc)
cbar.set_label("Timescale [ms]")
plt.tight_layout()
plt.savefig("../reports/estimating_timescales/figures/map_taus.pdf")
plt.show()
def get_auto_correlation_time(counts_from_spikes, min_steps, max_steps,
bin_size_ms):
rk = mre.coefficients(counts_from_spikes,
dt=bin_size_ms,
steps=(min_steps, max_steps))
fit = mre.fit(rk,
steps=(min_steps, max_steps),
fitfunc=mre.f_exponential_offset)
tau_C = fit.tau
# Computing the generalized timescale on offset corrected coefficients (does not work well because of huge fluctuations in autocorrelation)
# rk_offset = fit.popt[2]
# C_raw = rk.coefficients-rk
# range = int(6*tau_C)
# tau_C_generalized = plots.get_T_avg(T[:range+1], C_raw[:range+1], 0)
return tau_C, fit
def test_deconv():
directory = "../data/Spontaneous/suite2p/plane0/"
# load traces and subtract neuropil
iscell = np.load(directory + "iscell.npy")
F = np.load(directory + "F.npy")
Fneu = np.load(directory + "Fneu.npy")
Fc = F - 1 * Fneu
N = 3
mask_iscell = iscell[:, 1] > 0.5
Fc = Fc[mask_iscell, :]
Fc = Fc[:]
#Fc_conv = np.empty((Fc.shape[0], Fc.shape[1]-N+1))
#for i in range(len(Fc)):
# Fc_conv[i] = np.convolve(Fc[i], np.ones((N,))/N, mode='valid')
Fc_conv = Fc
spks = deconvolve(Fc_conv, fs=30, tau=1.3)
t = np.linspace(0, Fc_conv.shape[1] / 30, Fc_conv.shape[1])
print(iscell[0])
plt.plot(t, spks[0] * 10)
plt.plot(t, Fc_conv[0], alpha=0.8)
plt.show()
k_arr = np.arange(1, 40)
coefficients = np.zeros_like(k_arr, dtype=float)
for i, arr in enumerate(tqdm(spks[:])):
coeff_res = mre.coefficients(arr, k_arr)
coefficients += coeff_res.coefficients
f = plt.figure(figsize=(4, 3))
t = np.arange(0, len(coefficients)) / 30
plt.plot(t, coefficients / len(spks))
#plt.gca().set_yscale('log')
#plt.gca().set_xscale('log')
plt.xlabel("time interval $\Delta T$ [s]")
plt.ylabel("autocorrelation")
plt.tight_layout()
plt.savefig(
"../reports/estimating_timescales/figures/packer_example_lin_pil1.pdf")
plt.show()
def plot_spikes():
directory = "../data/Spontaneous/suite2p/plane0/"
iscell = np.load(directory + "iscell.npy")
spks = np.load(directory + "spks.npy")
F = np.load(directory + "F.npy")
Fneu = np.load(directory + "Fneu.npy")
t = np.linspace(0, F.shape[1] / 30, F.shape[1])
print(iscell[0])
plt.plot(t, spks[0] * 10)
plt.plot(t, (F - 0.7 * Fneu)[0], alpha=0.8)
plt.show()
return
mask_iscell = iscell[:, 1] > 0.5
spks_iscell = spks[mask_iscell, :]
tau_list = []
k_arr = np.arange(1, 50)
coefficients = np.zeros_like(k_arr, dtype=float)
for i, arr in enumerate(tqdm(spks_iscell[:])):
coeff_res = mre.coefficients(arr, k_arr)
coefficients += coeff_res.coefficients
plt.plot(coefficients / len(spks_iscell))
plt.show()
# see https://matplotlib.org/api/_as_gen/matplotlib.pyplot.plot.html
# for some more style options
oful.add_ts(srcsub, alpha=0.25, color='yellow', label='trials (subs.)')
oful.add_ts(avgsub, ls='dashed', color='maroon', label='average (subs.)')
# add the drive
oful.add_ts(srcdrv, color='green', label='drive')
plt.show()
# ------------------------------------------------------------------ #
# analyzing
# ------------------------------------------------------------------ #
# correlation coefficients with default settings, assumes 1ms time bins
rkdefault = mre.coefficients(srcful)
print(rkdefault)
print('this guy has the following attributes: ', rkdefault._fields)
# specify the range of time steps (from, to) for which coefficients are wanted
# also, set the unit and the number of time steps per bin e.g. 4ms per k:
rk = mre.coefficients(srcsub, steps=(1, 5000), dt=4, dtunit='ms', desc='mydat')
# fit with defaults: exponential over the full range of rk
m = mre.fit(rk)
# specify a custom fit range and fitfunction.
m2 = mre.fit(rk, steps=(1, 3000), fitfunc='offset')
# you could also provide an np array containing all the steps you want to
# use, e.g. with strides other than one
# m2 = mre.fit(rk, steps=np.arange(1, 3000, 100), fitfunc='offset')
spiketimes = np.load('{}/V1/spks/spiketimes-{}-{}.npy'.format(
DATA_DIR, neuron[0], neuron[1]))
# Add 5 seconds to make sure that only spikes with sufficient spiking history are considered
t_0 = spiketimes[0] + 5.
spiketimes = spiketimes - t_0
spiketimes = spiketimes[spiketimes > 0]
"""Compute autocorrelation time, median ISI and CV"""
counts_from_spikes = utl.get_binned_neuron_activity(spiketimes, bin_size)
tau_C, fit = get_auto_correlation_time(counts_from_spikes,
min_step_autocorrelation,
max_step_autocorrelation,
bin_size_ms)
print(tau_C)
rk = mre.coefficients(counts_from_spikes,
dt=bin_size_ms,
steps=(min_steps_plotting, max_steps_plotting))
T = rk.steps * bin_size_ms
rate, median_ISI, CV = get_spiking_stats(spiketimes)
print(rate, tau_C)
stats_dict = {
'rate': rate,
'medianISI': median_ISI,
'CV': CV,
'autocorrelation_time': tau_C
}
"""Plotting"""
rc('text', usetex=True)
matplotlib.rcParams['font.size'] = '16.0'
matplotlib.rcParams['xtick.labelsize'] = '16'
matplotlib.rcParams['ytick.labelsize'] = '16'
T_0 = 0.01
T_0_ms = T_0 * 1000
bin_size = 0.004
bin_size_ms = int(bin_size * 1000)
"""Load and preprocess data"""
DATA_DIR = '{}/data/{}'.format(CODE_DIR, recorded_system)
spiketimes = np.load('{}/spiketimes_tau{}ms_{}.npy'.format(
DATA_DIR, int(tau), rec_length))
# same processing as in the HD toolbox
spiketimes = spiketimes - spiketimes[0]
Trec = spiketimes[-1] - spiketimes[0]
counts_from_sptimes = utl.get_binned_neuron_activity(spiketimes, bin_size)
"""Compute measures"""
# Corr
rk = mre.coefficients(counts_from_sptimes,
dt=bin_size_ms,
steps=(min_step_autocorrelation,
max_step_autocorrelation))
T_C_ms = rk.steps * bin_size_ms
fit = mre.fit(rk, steps=(5, 500), fitfunc=mre.f_exponential_offset)
tau_est = fit.tau
rk_offset = fit.popt[2]
# computing integrated timescale on raw data
C_raw = rk.coefficients - rk_offset
tau_C_raw = plots.get_T_avg(T_C_ms, C_raw, T_0_ms)
# computing integrated timescale on fitted curve
C_fit = mre.f_exponential_offset(rk.steps, fit.tau / bin_size_ms, *
fit.popt[1:]) - rk_offset
tau_C_fit = plots.get_T_avg(T_C_ms, C_fit, T_0_ms)
# R and Delta R
ANALYSIS_DIR, analysis_num_str, R_tot, T_D, T_R, R, R_CI_lo, R_CI_hi = plots.load_analysis_results(
subp=0.05,
length=20000,
numtrials=10,
seed=43771)
# make sure the data has the right format
src = mre.input_handler(bp)
#----#
# 2. #
#----#
# calculate autocorrelation coefficients and
# embed information about the time steps
rks = mre.coefficients(src,
steps=(1, 500),
dt=1,
dtunit=' bp steps',
method='trialseparated')
#----#
# 3. #
#----#
# fit an exponential autocorrelation function
fit1 = mre.fit(rks, fitfunc='exponential')
fit2 = mre.fit(rks, fitfunc='exponential_offset')
#----#
# 4. #
#----#
def fit_tau(act, fs, numboot = 0, k_arr = None):
if k_arr is None:
k_arr = np.arange(1, fs * 1)
coeff_res = mre.coefficients(act, k_arr, dt=1/fs* 1000, numboot=numboot, method='ts')
tau_res = mre.fit(coeff_res, fitfunc='exponentialoffset', numboot=numboot)
return tau_res.tau
def correlation_spatial_subsampling():
#cells = {1: [8,0,1,1,8,8],
# 2: [15,4,3,3,15,15],
# 3: [19,23,19,18,19,19],
# 4: [16,40,88,75,16,16],
# 5: [24,25,43,46,24,24],
# 6: [46,58,48,109,46,46],
# 7: [26,41,37,41,26,26],
# 8: [28,45,31,37,28,28]}
cells = {
1: [8, 0, 1, 1, 1, 2, 2],
2: [15, 4, 3, 3, 2, 1, 11],
3: [19, 23, 19, 18, 18, 19, 240],
4: [16, 40, 88, 75, 85, 22, 51],
5: [24, 25, 43, 46, 38, 39, 255],
6: [46, 58, 48, 109, 44, 42, None],
7: [26, 41, 37, 41, 39, 40, 438],
8: [28, 45, 31, 37, 29, 25, 250]
}
labels = [
'60 Hz, 0.54x0.54$\mu$m, 30 min',
'60 Hz, 0.54x1.08$\mu$m, 30 min (subs.)',
'60 Hz, 1.08x1.08$\mu$m, 30 min (subs.)',
'60 Hz, 1.62x1.62$\mu$m, 30 min (subs.)',
'60 Hz, 2.16x2.16$\mu$m, 30 min (subs.)',
'60 Hz, 2.7x2.7$\mu$m, 30 min (subs.)', '60 Hz, 1.37$\mu$m, 15 min'
]
tau = 1.5
fs_list = [60, 60, 60, 60, 60, 60, 60]
k_arr_list = []
Fc_list = []
directory = '/data.nst/jdehning/packer_data/calcium_subsampled/'
subpath_F = 'suite2p/plane0/F.npy'
subpath_Fneu = 'suite2p/plane0/Fneu.npy'
paths = [
'2019-11-08_RL065_t-001_1x1', '2019-11-08_RL065_t-001_1x2',
'2019-11-08_RL065_t-001_2x2', '2019-11-08_RL065_t-001_3x3',
'2019-11-08_RL065_t-001_4x4', '2019-11-08_RL065_t-001_5x5',
'2019-11-08_RL065_t-002'
]
for path in paths:
F = np.load(os.path.join(directory, path, subpath_F))
Fneu = np.load(os.path.join(directory, path, subpath_Fneu))
Fc_list.append(F - 0.7 * Fneu)
del F, Fneu
spks_2dlist = []
for cell in cells.keys():
spks_2dlist.append([])
for Fc, cell_num, fs, i_rec in zip(Fc_list, cells[cell], fs_list,
range(1000)):
if cell_num is not None:
if i_rec == 4:
beg, end = (0, Fc.shape[1] // 2)
elif i_rec == 5:
beg, end = (Fc.shape[1] // 2, -1)
else:
beg, end = (0, -1)
spks_2dlist[-1].append(
deconvolve(Fc[cell_num][None, beg:end], fs, tau=tau)[0])
else:
spks_2dlist[-1].append(None)
mpl.rcParams["axes.spines.right"] = False
mpl.rcParams["axes.spines.top"] = False
f, axes_list = plt.subplots(4, 2, figsize=(16, 20))
axes_list = [ax for axes in axes_list for ax in axes]
colors = [
'tab:blue', 'tab:pink', 'tab:red', 'tab:cyan', 'tab:olive',
'tab:green', 'k'
]
tau_res_list2d = []
def mult_coeff_res(coeff_res, mult):
return coeff_res._replace(
coefficients=coeff_res.coefficients * mult,
stderrs=coeff_res.stderrs *
mult if coeff_res.stderrs is not None else None)
to_plot = [0, 1, 2, 3, 4, 5]
for cell in range(len(cells)):
taus = []
plot = mre.OutputHandler([], ax=axes_list[cell])
tau_res_list2d.append([])
for i in range(len(labels)):
k_arr = np.arange(1, fs_list[i] * 1)
if spks_2dlist[cell][i] is not None:
act = spks_2dlist[cell][i]
print(act.shape)
len_trial = round(40 * fs_list[i])
act = act[:-(len(act) % len_trial)].reshape((-1, len_trial))
coeff_res = mre.coefficients(act,
k_arr,
dt=1 / fs_list[i] * 1000,
numboot=500,
method='ts')
#norm = 1/coeff_res.coefficients[0]
#coeff_res = mult_coeff_res(coeff_res, norm)
#for j, bootstrapcrs in enumerate(coeff_res.bootstrapcrs):
# coeff_res.bootstrapcrs[j] = mult_coeff_res(bootstrapcrs, norm)
#axes_list[cell].plot(k_arr/fs_list[i]*1000 ,coeff_res.coefficients, color=colors[i],
# label = labels[i] if cell==0 else None)
if i in to_plot:
plot.add_coefficients(
coeff_res,
color=colors[i],
label=labels[i] if cell == 0 else None)
tau_res_list2d[-1].append(
mre.fit(coeff_res,
fitfunc='exponentialoffset',
numboot=500))
else:
tau_res_list2d[-1].append(None)
#axes_list[cell].set_ylim(-0.3,1.2)
axes_list[cell].set_title('cell {}'.format(cell))
plt.legend()
plt.tight_layout()
plt.savefig(
'../reports/spatial_subsampling/autocorrelation_plots_spatialSubs3.pdf'
)
#axes_list[0].set_ylabel('Autocorrelation')
plt.show()
x_ticks = np.arange(len(cells))
offsets = np.linspace(-0.25, 0.25, len(labels))
f, axes_list = plt.subplots(figsize=(10, 6))
for i_ana, offset in enumerate(offsets):
res_list = [
tau_res_list2d[i_cell][i_ana] for i_cell in range(len(cells))
]
taus = np.array(
[res.tau if res is not None else None for res in res_list],
dtype=np.float)
yerr_lower = taus - np.array([
res.tauquantiles[0] if res is not None else None
for res in res_list
],
dtype=np.float)
yerr_upper = np.array([
res.tauquantiles[-1] if res is not None else None
for res in res_list
],
dtype=np.float) - taus
plt.errorbar(x_ticks + offset,
y=taus,
yerr=[yerr_lower, yerr_upper],
color=colors[i_ana],
label=labels[i_ana],
marker="x",
elinewidth=1.5,
capsize=2,
markersize=6,
markeredgewidth=1,
linestyle="")
plt.ylim(0, 400)
plt.legend()
plt.xlabel("Cell number")
plt.ylabel('Timescale (ms)')
#fit_result = mre.fit(coeff_res, steps=k_arr)
#taus.append(fit_result.tau)
plt.savefig('../reports/spatial_subsampling/timescales_spatialSubs3.pdf')
plt.show()
# ------------------------------------------------------------------------------ #
# Script that creates Figure 2.
# Demonstrates the subsampling bias.
# ------------------------------------------------------------------------------ #
import os
import mrestimator as mre
import matplotlib.pyplot as plt
bp_full = mre.simulate_branching(a=1000, m=0.98, numtrials=500, length=20000)
bp_subs = mre.simulate_subsampling(bp_full, prob=0.02)
bp_subz = mre.simulate_subsampling(bp_full, prob=0.001)
rk_full = mre.coefficients(bp_full,
steps=(1, 300),
dt=1,
method="trialseparated",
dtunit="steps",
description="fully sampled")
rk_subs = mre.coefficients(bp_subs,
steps=(1, 300),
dt=1,
method="trialseparated",
dtunit="steps",
description="subsampled to 2%")
rk_subz = mre.coefficients(bp_subz,
steps=(1, 300),
dt=1,
method="trialseparated",
dtunit="steps",
description="subsampled to 0.1%")
def compare_injected_vs_gen():
paths = ['/data.nst/share/data/packer_calcium_mice/2019-11-08_RL065/2019-11-08_RL065_t-003/suite2p/plane0',
'/data.nst/share/data/packer_calcium_mice/2019-03-01_R024/Spontaneous/suite2p/plane0',
"/data.nst/share/data/packer_calcium_mice/2019-08-15_RL055_t-003",
'/data.nst/share/data/packer_calcium_mice/2019-08-14_J059_t-002',
'/data.nst/jdehning/packer_data/2019-11-07_J061_t-003/suite2p/plane0']
fs_list = [30,30,30,30,15]
tau_dcnv = 1.5
Fc_list = [get_Fc(path)[get_cell_nums(path)] for path in paths]
dcnv_list = [deconvolve_Fc(Fc, fs, tau=tau_dcnv) for Fc, fs in zip(Fc_list, fs_list)]
tau_2Dlist = []
for act_mat, fs in zip(dcnv_list, fs_list):
tau_2Dlist.append([])
for act in act_mat:
tau = fit_tau(act, fs, k_arr=np.arange(1,70))
tau_2Dlist[-1].append(tau)
nth_largest_snr = 5
n_bins_rolling_sum = 3
##Calculate SNR with Jonas' method
#nth_largest_list=np.round(np.logspace(np.log10(1),np.log10(100),20)).astype('int')
nth_largest_list = [nth_largest_snr]
snr_diff_2D_list = []
snr_two_periods_list = []
for i_exp in range(len(paths)):
snr_diff_2D_list.append([])
for nth_largest in nth_largest_list:
snr_periods = [None, None]
for i_period in range(2):
Fc_all = Fc_list[i_exp]
dcnv_all = dcnv_list[i_exp]
Fc = Fc_all[:, Fc_all.shape[1]//2:] if i_period == 0 else Fc_all[:, :Fc_all.shape[1]//2]
dcnv = dcnv_all[:, dcnv_all.shape[1]//2:] if i_period == 0 else dcnv_all[:, :dcnv_all.shape[1]//2]
snr = calc_signal(dcnv, n_bins_rolling_sum, nth_largest)/np.std(Fc, axis=-1)
print(i_exp, nth_largest, snr[1])
snr_periods[i_period] = np.array(snr)
#if i_exp == 2:
# plt.plot(snr_periods[0], snr_periods[1], '.', alpha=0.4)
#print(np.corrcoef(snr_periods[0], snr_periods[1]))
#median_diff = np.median(np.abs((snr_periods[1]-snr_periods[0])/snr_periods[0]))
median_diff = np.corrcoef(snr_periods[0], snr_periods[1])[0,1]
snr_diff_2D_list[-1].append(median_diff)
snr_two_periods_list.append((snr_periods[0], snr_periods[1]))
if False:
f, axes = plt.subplots(1, 5, figsize = (15,3))
titles = ['transgenic: 30 Hz, 30 min (2019-11-08, RL065)', 'transgenic: 30 Hz, 10 min (2019-03-01, RL024)',
'injected: 30 Hz, 26 min (2019-08-15, RL055)', 'injected: 30 Hz, 25 min (2019-08-14, J059)',
'injected: 15 Hz, 30 min (2019-11-07, J061)']
for i_ax, ax in enumerate(axes):
ax.plot(nth_largest_list, snr_diff_2D_list[i_ax])
#ax.set_ylim(0,400)
ax.set_xlabel("snr")
if i_ax == 0:
ax.set_ylabel('median percentual difference of snr\nbetween begin and end of recording')
#ax.set_xlim(0,12)
ax.set_title(titles[i_ax])
plt.show()
snr_2Dlist = []
range_snr = [[1,1.5],[1.5,2],[2,2.5],[2.5,3], [3,4]]
coefficients_list = []
for Fc_mat, act_mat, tau_mat, i_exp in zip(Fc_list, dcnv_list, tau_2Dlist, range(1000)):
snr_2Dlist.append([])
i_plot = 0
coefficients_list.append([])
for i_hist in range(len(range_snr)):
coefficients_list[-1].append([])
for Fc, act, tau in zip(Fc_mat, act_mat, tau_mat):
#l_norm = 10
#act_for_snr = act
#act_for_snr = np.concatenate([np.sum([act[:-2], act[1:-1], act[2:]], axis=0), [0,0]])
#act_for_snr = np.concatenate([np.sum([act[:-1], act[1:]], axis=0), [0]])
snr = calc_signal(act, n_bins_rolling_sum, nth_largest_snr) / np.std(Fc)
#snr = np.max(act_for_snr)/np.std(Fc)
#snr = np.sum(act_for_snr**l_norm)**(1/l_norm)/np.std(Fc)
if snr < 2 and snr > 1.5 and i_exp==0 and False:
print(snr, tau)
fs = fs_list[i_exp]
#plt.clf()
#f, axes = plt.subplots(2, 1, figsize=(10, 5))
#axes[0].plot(Fc)
#axes[0].plot(act_for_snr)
#amax = np.argmax(act_for_snr)
#axes[0].set_xlim(amax-fs*2, amax+fs*5)
coefficients.append(mre.coefficients(act, k_arr, dt=1/fs* 1000, numboot=0, method='ts').coefficients)
#axes[1].plot(mre.coefficients(act, k_arr, dt=1/fs* 1000, numboot=0, method='ts').coefficients)
#input()
plt.close()
for i_hist, (min_snr, max_snr) in enumerate(range_snr):
if snr > min_snr and snr < max_snr:
fs = fs_list[i_exp]
k_arr = np.arange(1, 2*fs)
coefficients_list[-1][i_hist].append(mre.coefficients(act, k_arr, dt=1/fs* 1000, numboot=0, method='ts').coefficients)
snr_2Dlist[-1].append(snr)
#plt.plot(np.mean(np.array(coefficients), axis=0))
#plt.show()
f, axes = plt.subplots(4, 5, figsize = (18,12))
titles = ['transgenic: 30 Hz, 30 min\n(2019-11-08, RL065)', 'transgenic: 30 Hz, 10 min\n(2019-03-01, RL024)',
'injected: 30 Hz, 26 min\n(2019-08-15, RL055)', 'injected: 30 Hz, 25 min\n(2019-08-14, J059)',
'injected: 15 Hz, 20 min\n(2019-11-07, J061)']
for i_ax, ax in enumerate(axes[0]):
ax.plot(snr_2Dlist[i_ax], tau_2Dlist[i_ax], '.', alpha=0.3)
#ax.hist(snr_2Dlist[i_ax], bins=np.linspace(0,8,30))
ax.set_ylim(0,400)
ax.set_xlabel("Signal-to-noise ratio")
if i_ax == 0:
ax.set_ylabel('timescales (ms)')
ax.set_xlim(0,6)
ax.set_title(titles[i_ax])
for i_ax, ax in enumerate(axes[1]):
#ax.plot(snr_2Dlist[i_ax], tau_2Dlist[i_ax], '.', alpha=0.3)
ax.hist(snr_2Dlist[i_ax], bins=np.linspace(0,8,30))
ax.set_ylim(0,200)
ax.set_xlabel("Signal-to-noise ratio")
if i_ax == 0:
ax.set_ylabel('number of cells')
ax.set_xlim(0,6)
for i_ax, ax in enumerate(axes[2]):
#ax.plot(snr_2Dlist[i_ax], tau_2Dlist[i_ax], '.', alpha=0.3)
for i_hist, (min_snr, max_snr) in enumerate(range_snr):
fs = fs_list[i_ax]
k_arr = np.arange(1, 2 * fs)
time_arr = k_arr/fs*1000
correlation_coeff = np.mean(np.array(coefficients_list[i_ax][i_hist]), axis=0)
num_cells = np.array(coefficients_list[i_ax][i_hist]).shape[0]
try:
ax.plot(time_arr, correlation_coeff,
label = 'cells with SNR between {} and {}\nnumber of cells: {}'.format(min_snr, max_snr, num_cells))
except ValueError:
continue
ax.set_xlabel("Time difference [ms]")
ax.legend(fontsize=7)
if i_ax == 0:
ax.set_ylabel('Average autocorrelation')
#ax.set_xlim(0,10)
for i_ax, ax in enumerate(axes[3]):
snr1 = snr_two_periods_list[i_ax][0]
snr2 = snr_two_periods_list[i_ax][1]
ax.plot(snr1,snr2, '.', alpha =0.4)
ax.set_ylim(0, ax.get_ylim()[1])
ax.set_xlim(ax.get_ylim())
ax.set_xlabel("SNR first half of recording\nCorrelation coefficient: {:.2f}".format(np.corrcoef(snr1,snr2)[0,1]))
ax.plot([0,ax.get_xlim()[1]], [0,ax.get_ylim()[1]], ':' ,color='tab:gray')
if i_ax == 0:
ax.set_ylabel("SNR second half of recording")
plt.tight_layout()
plt.savefig('../reports/snr_of_recordings/snr_overview_figure.png', dpi=200)
plt.show()
def correlation_different_resolutions5():
cells = {
1: [8, 2, 14, 0],
2: [15, 11, 87, 1],
3: [19, 240, 196, 4],
4: [16, 51, 124, 2],
5: [24, 255, 305, 3],
6: [46, None, None, 9],
7: [26, 438, None, 17],
8: [28, 250, 793, 6]
}
labels = [
'60 Hz, 1.37$\mu$m, 15 min', '30 Hz, 1.37$\mu$m, 15 min (downsampled)',
'20 Hz, 1.37$\mu$m, 15 min (downsampled)',
'15 Hz, 1.37$\mu$m, 15 min (downsampled)',
'10 Hz, 1.37$\mu$m, 15 min (downsampled)',
'7.5 Hz, 1.37$\mu$m, 15 min (downsampled)',
'5 Hz 1.37$\mu$m, 15 min (downsampled)'
]
tau = 1.5
fs_list = [60, 30, 20, 15, 10, 7.5, 5]
k_arr_list = []
Fc_list = []
directory = '/data.nst/share/data/packer_calcium_mice/2019-11-08_RL065/'
subpath_F = 'suite2p/plane0/F.npy'
subpath_Fneu = 'suite2p/plane0/Fneu.npy'
paths = ['2019-11-08_RL065_t-002']
for path in paths:
F = np.load(os.path.join(directory, path, subpath_F))
Fneu = np.load(os.path.join(directory, path, subpath_Fneu))
Fc_list.append(F - 0.7 * Fneu)
del F, Fneu
spks_2dlist = []
for cell in cells.keys():
spks_2dlist.append([])
for Fc, cell_num, fs in zip(Fc_list, cells[cell][1:], fs_list):
if cell_num is not None:
spks_2dlist[-1].append(
deconvolve(Fc[cell_num][None, :], fs, tau=tau)[0])
else:
spks_2dlist[-1].append(None)
subsampling_list = [30, 20, 15, 10, 7.5, 5]
for i, subs in enumerate(subsampling_list):
for i_cell, cell in enumerate(cells.keys()):
if cells[cell][1] is not None:
cell_num = cells[cell][1]
Fc = Fc_list[0][cell_num][None, ::round(60 // subs)]
spks1 = deconvolve(Fc, subs, tau=tau)[0]
spks_2dlist[i_cell].append(spks1)
else:
spks_2dlist[i_cell].append(None)
mpl.rcParams["axes.spines.right"] = False
mpl.rcParams["axes.spines.top"] = False
f, axes_list = plt.subplots(4, 2, figsize=(16, 20))
axes_list = [ax for axes in axes_list for ax in axes]
colors = [
'tab:blue', 'tab:pink', 'tab:red', 'tab:cyan', 'tab:olive',
'tab:green', 'tab:purple', 'tab:gray'
]
tau_res_list2d = []
def mult_coeff_res(coeff_res, mult):
return coeff_res._replace(
coefficients=coeff_res.coefficients * mult,
stderrs=coeff_res.stderrs *
mult if coeff_res.stderrs is not None else None)
to_plot = [0, 1, 3, 4]
for cell in range(len(cells)):
taus = []
plot = mre.OutputHandler([], ax=axes_list[cell])
tau_res_list2d.append([])
for i in range(len(labels)):
k_arr = np.arange(1, fs_list[i] * 1)
if spks_2dlist[cell][i] is not None:
act = spks_2dlist[cell][i]
print(act.shape)
len_trial = round(40 * fs_list[i])
act = act[:-(len(act) % len_trial)].reshape((-1, len_trial))
coeff_res = mre.coefficients(act,
k_arr,
dt=1 / fs_list[i] * 1000,
numboot=500,
method='ts')
#norm = 1/coeff_res.coefficients[0]
#coeff_res = mult_coeff_res(coeff_res, norm)
#for j, bootstrapcrs in enumerate(coeff_res.bootstrapcrs):
# coeff_res.bootstrapcrs[j] = mult_coeff_res(bootstrapcrs, norm)
#axes_list[cell].plot(k_arr/fs_list[i]*1000 ,coeff_res.coefficients, color=colors[i],
# label = labels[i] if cell==0 else None)
if i in to_plot:
plot.add_coefficients(
coeff_res,
color=colors[i],
label=labels[i] if cell == 0 else None)
tau_res_list2d[-1].append(
mre.fit(coeff_res,
fitfunc='exponentialoffset',
numboot=500))
else:
tau_res_list2d[-1].append(None)
#axes_list[cell].set_ylim(-0.3,1.2)
axes_list[cell].set_title('cell {}'.format(cell))
plt.legend()
plt.tight_layout()
plt.savefig(
'../reports/different_zoom_levels/autocorrelation_plots_tempSubs2.pdf')
#axes_list[0].set_ylabel('Autocorrelation')
plt.show()
x_ticks = np.arange(len(cells))
offsets = np.linspace(-0.25, 0.25, len(labels))
f, axes_list = plt.subplots(figsize=(10, 6))
for i_ana, offset in enumerate(offsets):
res_list = [
tau_res_list2d[i_cell][i_ana] for i_cell in range(len(cells))
]
taus = np.array(
[res.tau if res is not None else None for res in res_list],
dtype=np.float)
yerr_lower = taus - np.array([
res.tauquantiles[0] if res is not None else None
for res in res_list
],
dtype=np.float)
yerr_upper = np.array([
res.tauquantiles[-1] if res is not None else None
for res in res_list
],
dtype=np.float) - taus
plt.errorbar(x_ticks + offset,
y=taus,
yerr=[yerr_lower, yerr_upper],
color=colors[i_ana],
label=labels[i_ana],
marker="x",
elinewidth=1.5,
capsize=2,
markersize=6,
markeredgewidth=1,
linestyle="")
plt.ylim(0, 400)
plt.legend()
plt.xlabel("Cell number")
plt.ylabel('Timescale (ms)')
#fit_result = mre.fit(coeff_res, steps=k_arr)
#taus.append(fit_result.tau)
plt.savefig('../reports/different_zoom_levels/timescales_tempSubs2.pdf')
plt.show()
