def button_update(self, Network_UI, group):
if self.WP_test_execute:
self.WP_test_execute = False
#if self.WPT_init:
import Exploration.Analysis.WP_testing as WPT
import mrestimator as mre
#self.WPT_init = True
group_A_t = group[self.sum_tag, 0, 'np'].copy() # [-1000:]
net_A_t = np.sum(Network_UI.network[self.sum_tag],
axis=0) # np.swapaxes(, 0, 1)#
rk, ft, (counts, bins) = WPT.branching_ratio(group_A_t, 1)
expoffset_fit = mre.fit(rk, fitfunc=mre.f_exponential_offset)
test1 = WPT.h_offset(ft, expoffset_fit)
test2 = WPT.h_tau(ft, expoffset_fit)
test3 = WPT.h_lin(rk, ft)
print(str(test1) + ' ' + str(test2) + ' ' + str(test3))
rk, ft, (counts, bins) = WPT.branching_ratio(net_A_t, 1)
expoffset_fit = mre.fit(rk, fitfunc=mre.f_exponential_offset)
test1 = WPT.h_offset(ft, expoffset_fit)
test2 = WPT.h_tau(ft, expoffset_fit)
test3 = WPT.h_lin(rk, ft)
print(str(test1) + ' ' + str(test2) + ' ' + str(test3))
def branching_ratio(ax, bpath, nsp, bin_w):
with open(bpath+'/raw/gexc_spks.p', 'rb') as pfile:
GExc_spks = pickle.load(pfile)
# with open(bpath+'/raw/ginh_spks.p', 'rb') as pfile:
# GInh_spks = pickle.load(pfile)
ts = GExc_spks['t']/ms
ts = ts[ts>(nsp['T1']+nsp['T2']+nsp['T3']+nsp['T4'])/ms]
ts = ts - (nsp['T1']+nsp['T2']+nsp['T3']+nsp['T4'])/ms
assert(np.min(ts) >= 0)
bins = np.arange(0, (nsp['T5']+bin_w)/ms, bin_w/ms)
counts, bins = np.histogram(ts, bins=bins, density=False)
print(np.shape(counts))
# counts = np.reshape(counts, (len(counts),1))
# print(np.shape(counts))
rk = mre.coefficients(counts, dt=bin_w/ms, dtunit='ms', desc='')
ft = mre.fit(rk)
return rk, ft, (counts, bins)
def branching_ratio(spikes, bin_w):
GExc_spks = spikes
#with open(bpath + '/raw/gexc_spks.p', 'rb') as pfile:
# GExc_spks = pickle.load(pfile)
# with open(bpath+'/raw/ginh_spks.p', 'rb') as pfile:
# GInh_spks = pickle.load(pfile)
#ts = GExc_spks['t'] / ms
#ts = ts[ts > (nsp['T1'] + nsp['T2'] + nsp['T3'] + nsp['T4']) / ms]
#ts = ts - (nsp['T1'] + nsp['T2'] + nsp['T3'] + nsp['T4']) / ms
#assert (np.min(ts) >= 0)
bins = np.arange(0, (len(spikes) + bin_w), bin_w)
counts, bins = np.histogram(GExc_spks, bins=bins)
print(np.shape(counts))
# counts = np.reshape(counts, (len(counts),1))
# print(np.shape(counts))
rk = mre.coefficients(counts, dt=bin_w, dtunit='ms') #, desc=''
ft = mre.fit(rk)
return rk, ft, (counts, bins)
def h_lin(rk: mre.CoefficientResult, exp_fit: mre.FitResult) -> bool:
"""
Test the H_lin hypothesis as described in (Wilting&Priesemann 2018; Supplemental Note 5)
:return: if True, then the data set should be rejected for m estimation
"""
lin_fit = mre.fit(rk, fitfunc=mre.f_linear)
return lin_fit.ssres < exp_fit.ssres
def get_auto_correlation_time(counts_from_spikes, min_steps, max_steps,
bin_size_ms):
rk = mre.coefficients(counts_from_spikes,
dt=bin_size_ms,
steps=(min_steps, max_steps))
fit = mre.fit(rk,
steps=(min_steps, max_steps),
fitfunc=mre.f_exponential_offset)
tau_C = fit.tau
return tau_C, fit
def get_auto_correlation_time(spikes, min_steps, max_steps, bin_size_ms):
rk = mre.coefficients(spikes, dt=bin_size_ms, steps=(min_steps, max_steps))
T = (rk.steps - rk.steps[0] + 1) * bin_size_ms
fit = mre.fit(rk, steps=(min_steps, max_steps), fitfunc=mre.f_exponential)
tau_C = fit.tau
C_raw = rk.coefficients
range = int(6 * tau_C)
tau_C_int = plots.get_T_avg(T[:range + 1], C_raw[:range + 1], 0)
print(tau_C, tau_C_int)
return tau_C_int, rk, T, fit
def plot_map_tau():
directory = "../data/Spontaneous/suite2p/plane0/"
# load traces and subtract neuropil
iscell = np.load(directory + "iscell.npy")
F = np.load(directory + "F.npy")
Fneu = np.load(directory + "Fneu.npy")
Fc = F - 1 * Fneu
stat = np.load(directory + 'stat.npy', allow_pickle=True)
x_pos = np.array([np.median(neur["xpix"]) for neur in stat])
y_pos = np.array([np.median(neur["ypix"]) for neur in stat])
mask_iscell = iscell[:, 1] > 0.5
Fc = Fc[mask_iscell, :]
Fc = Fc[:]
x_pos = x_pos[mask_iscell]
y_pos = y_pos[mask_iscell]
x_pos = x_pos[:]
y_pos = y_pos[:]
Fc_conv = Fc
spks = deconvolve(Fc_conv, fs=30, tau=1.5)
t = np.linspace(0, Fc_conv.shape[1] / 30, Fc_conv.shape[1])
k_arr = np.arange(1, 40)
taus = []
for i, arr in enumerate(tqdm(spks[:])):
coeff_res = mre.coefficients(arr, k_arr, dt=1 / 30 * 1000)
plt.plot(coeff_res.coefficients)
plt.show()
fit_result = mre.fit(coeff_res, steps=np.arange(1, 40))
taus.append(fit_result.tau)
cm = plt.cm.get_cmap("inferno")
sc = plt.scatter(x_pos,
y_pos,
c=taus,
vmin=0,
vmax=300,
s=35,
cmap=cm,
norm=colors.PowerNorm(0.7, 10, 300))
plt.xlabel("x position")
plt.ylabel("y position")
cbar = plt.colorbar(sc)
cbar.set_label("Timescale [ms]")
plt.tight_layout()
plt.savefig("../reports/estimating_timescales/figures/map_taus.pdf")
plt.show()
def get_auto_correlation_time(counts_from_spikes, min_steps, max_steps,
bin_size_ms):
rk = mre.coefficients(counts_from_spikes,
dt=bin_size_ms,
steps=(min_steps, max_steps))
fit = mre.fit(rk,
steps=(min_steps, max_steps),
fitfunc=mre.f_exponential_offset)
tau_C = fit.tau
# Computing the generalized timescale on offset corrected coefficients (does not work well because of huge fluctuations in autocorrelation)
# rk_offset = fit.popt[2]
# C_raw = rk.coefficients-rk
# range = int(6*tau_C)
# tau_C_generalized = plots.get_T_avg(T[:range+1], C_raw[:range+1], 0)
return tau_C, fit
"""Load and preprocess data"""
DATA_DIR = '{}/data/{}'.format(CODE_DIR, recorded_system)
spiketimes = np.load('{}/spiketimes_tau{}ms_{}.npy'.format(
DATA_DIR, int(tau), rec_length))
# same processing as in the HD toolbox
spiketimes = spiketimes - spiketimes[0]
Trec = spiketimes[-1] - spiketimes[0]
counts_from_sptimes = utl.get_binned_neuron_activity(spiketimes, bin_size)
"""Compute measures"""
# Corr
rk = mre.coefficients(counts_from_sptimes,
dt=bin_size_ms,
steps=(min_step_autocorrelation,
max_step_autocorrelation))
T_C_ms = rk.steps * bin_size_ms
fit = mre.fit(rk, steps=(5, 500), fitfunc=mre.f_exponential_offset)
tau_est = fit.tau
rk_offset = fit.popt[2]
# computing integrated timescale on raw data
C_raw = rk.coefficients - rk_offset
tau_C_raw = plots.get_T_avg(T_C_ms, C_raw, T_0_ms)
# computing integrated timescale on fitted curve
C_fit = mre.f_exponential_offset(rk.steps, fit.tau / bin_size_ms, *
fit.popt[1:]) - rk_offset
tau_C_fit = plots.get_T_avg(T_C_ms, C_fit, T_0_ms)
# R and Delta R
ANALYSIS_DIR, analysis_num_str, R_tot, T_D, T_R, R, R_CI_lo, R_CI_hi = plots.load_analysis_results(
recorded_system,
rec_length,
sample_index,
#----#
# calculate autocorrelation coefficients and
# embed information about the time steps
rks = mre.coefficients(src,
steps=(1, 500),
dt=1,
dtunit=' bp steps',
method='trialseparated')
#----#
# 3. #
#----#
# fit an exponential autocorrelation function
fit1 = mre.fit(rks, fitfunc='exponential')
fit2 = mre.fit(rks, fitfunc='exponential_offset')
#----#
# 4. #
#----#
# create an output handler instance
out = mre.OutputHandler([rks, fit1, fit2])
# save to disk
out.save('~/mre_example/result')
#----#
# 5. #
#----#
def branching_ratio_figure(bpath):
nsp = None
try:
with open(bpath + '/raw/namespace.p', 'rb') as pfile:
nsp = pickle.load(pfile)
except FileNotFoundError:
print(bpath[-4:], "reports: No namespace data. Skipping.")
return
size_factor = 2
rc('font', size=str(8 * size_factor))
fig: pl.Figure = pl.figure()
ax_lines, ax_cols = 2, 2
axs: Dict[str, pl.Axes] = {}
for x, y in itertools.product(range(ax_lines), range(ax_cols)):
axs['%d,%d' % (x + 1, y + 1)] = pl.subplot2grid((ax_lines, ax_cols),
(x, y))
fig.set_size_inches(ax_cols * size_factor * 6 * 1.6 / 2,
ax_lines * size_factor * 2. * 1.6)
tmin5 = nsp['T1'] + nsp['T2'] + nsp['T3'] + nsp['T4']
raster_plot(axs['1,1'], bpath, nsp, tmin=tmin5, tmax=tmin5 + nsp["T5"])
bin_w = 5 * ms
rk, ft, (counts, bins) = branching_ratio('', bpath, nsp, bin_w)
expoffset_fit = mre.fit(rk, fitfunc=mre.f_exponential_offset)
Hs = h_offset(ft, expoffset_fit), h_tau(ft, expoffset_fit), h_lin(
rk, ft), h_mean(rk), h_q1_0(rk)
H_offset_rej, H_tau_rej, H_lin_rej, H_mean_rej, H_q1_0_rej = [
"rej" if H else "acc" for H in Hs
]
H_q1_0_rej = "Poisson" if H_q1_0_rej == "rej" else "rej" # for this test True/rej means Poisson
# if False and H_mean True then reject
complex_fit = mre.fit(rk, fitfunc=mre.f_complex)
tbl = axs['2,1'].table(loc="center",
cellText=(
("$\Delta{}t$", str(bin_w)),
("mre", f"{ft.mre:0.6f}"),
("mre (complex)", f"{complex_fit.mre:0.6f}"),
("$H_{offset}$", H_offset_rej),
(
"$H_{\\tau}$",
H_tau_rej,
),
("$H_{lin}$", H_lin_rej),
("$H_{mean}$", H_mean_rej),
("$H_{q1\\_0}$ (if $H_{mean}$ is rej)",
H_q1_0_rej),
("ATotalMax", nsp['ATotalMax']),
("iATotalMax", nsp['iATotalMax']),
("$mu_e$", nsp["mu_e"]),
("$mu_i$", nsp["mu_i"]),
))
tbl.scale(1, 0.6 * size_factor)
axs['2,1'].axis('tight')
axs['2,1'].axis('off')
bin_scales = [0.25, 0.5, 2, 3, 4, 5, 6, 7, 8, 9, 10]
def fit_bin(bin_w, bin_s):
rk, _, _ = branching_ratio('', bpath, nsp, bin_w * bin_s)
return mre.fit(rk, fitfunc=mre.f_exponential)
fits = [fit_bin(bin_w, bin_s).mre for bin_s in bin_scales]
fits_expected = [ft.mre**bin_s for bin_s in bin_scales]
bin_scales.insert(2, 1)
fits.insert(2, ft.mre)
fits_expected.insert(2, ft.mre)
markersize = str(int(5 * size_factor))
axs['1,2'].plot(bin_scales,
fits,
label="actual",
linestyle="None",
marker="D",
markersize=markersize)
axs['1,2'].plot(bin_scales,
fits_expected,
label="expected",
linestyle="None",
marker="P",
markersize=markersize)
axs['1,2'].legend()
axs['1,2'].set_title(
f"$actual=m(binsize=k\\Delta{{}}t)=m(binsize=\\Delta{{}}t)^k=expected$, $\Delta{{}}t={bin_w}$"
)
axs['1,2'].set_xlabel("$k$")
axs['1,2'].set_ylabel("$\hat{m}$")
axs['2,2'].bar(bins[1:],
counts,
width=1.0,
facecolor="tab:blue",
edgecolor="tab:blue")
axs['2,2'].set_xlim(0, bins[-1])
axs['2,2'].set_title(f"Activity with binsize={bin_w}")
pl.tight_layout()
directory = "figures/branching_ratio"
if not os.path.exists(directory):
os.makedirs(directory)
pl.savefig(directory + "/{:s}.png".format(bpath[-4:]),
dpi=300,
bbox_inches='tight')
def fit_bin(bin_w, bin_s):
rk, _, _ = branching_ratio('', bpath, nsp, bin_w * bin_s)
return mre.fit(rk, fitfunc=mre.f_exponential)
dtunit="steps",
description="fully sampled")
rk_subs = mre.coefficients(bp_subs,
steps=(1, 300),
dt=1,
method="trialseparated",
dtunit="steps",
description="subsampled to 2%")
rk_subz = mre.coefficients(bp_subz,
steps=(1, 300),
dt=1,
method="trialseparated",
dtunit="steps",
description="subsampled to 0.1%")
fit_full = mre.fit(rk_full)
fit_subs = mre.fit(rk_subs)
fit_subz = mre.fit(rk_subz)
fig, ax = plt.subplots(figsize=(4, 2.0))
out = mre.OutputHandler([rk_full, rk_subs, rk_subz], ax=ax)
fig.tight_layout()
os.chdir(os.path.dirname(os.path.abspath(__file__)))
out.save_plot("./coefficients_and_subsampling")
ax.set_yscale("log")
ax.set_ylim(1e-3, 1)
ax.set_yticks([1e0, 1e-1, 1e-2, 1e-3])
ax.set_ylabel(r"$r_k$ (log scale)")
ax.get_legend().set_visible(False)
def correlation_different_resolutions5():
cells = {
1: [8, 2, 14, 0],
2: [15, 11, 87, 1],
3: [19, 240, 196, 4],
4: [16, 51, 124, 2],
5: [24, 255, 305, 3],
6: [46, None, None, 9],
7: [26, 438, None, 17],
8: [28, 250, 793, 6]
}
labels = [
'60 Hz, 1.37$\mu$m, 15 min', '30 Hz, 1.37$\mu$m, 15 min (downsampled)',
'20 Hz, 1.37$\mu$m, 15 min (downsampled)',
'15 Hz, 1.37$\mu$m, 15 min (downsampled)',
'10 Hz, 1.37$\mu$m, 15 min (downsampled)',
'7.5 Hz, 1.37$\mu$m, 15 min (downsampled)',
'5 Hz 1.37$\mu$m, 15 min (downsampled)'
]
tau = 1.5
fs_list = [60, 30, 20, 15, 10, 7.5, 5]
k_arr_list = []
Fc_list = []
directory = '/data.nst/share/data/packer_calcium_mice/2019-11-08_RL065/'
subpath_F = 'suite2p/plane0/F.npy'
subpath_Fneu = 'suite2p/plane0/Fneu.npy'
paths = ['2019-11-08_RL065_t-002']
for path in paths:
F = np.load(os.path.join(directory, path, subpath_F))
Fneu = np.load(os.path.join(directory, path, subpath_Fneu))
Fc_list.append(F - 0.7 * Fneu)
del F, Fneu
spks_2dlist = []
for cell in cells.keys():
spks_2dlist.append([])
for Fc, cell_num, fs in zip(Fc_list, cells[cell][1:], fs_list):
if cell_num is not None:
spks_2dlist[-1].append(
deconvolve(Fc[cell_num][None, :], fs, tau=tau)[0])
else:
spks_2dlist[-1].append(None)
subsampling_list = [30, 20, 15, 10, 7.5, 5]
for i, subs in enumerate(subsampling_list):
for i_cell, cell in enumerate(cells.keys()):
if cells[cell][1] is not None:
cell_num = cells[cell][1]
Fc = Fc_list[0][cell_num][None, ::round(60 // subs)]
spks1 = deconvolve(Fc, subs, tau=tau)[0]
spks_2dlist[i_cell].append(spks1)
else:
spks_2dlist[i_cell].append(None)
mpl.rcParams["axes.spines.right"] = False
mpl.rcParams["axes.spines.top"] = False
f, axes_list = plt.subplots(4, 2, figsize=(16, 20))
axes_list = [ax for axes in axes_list for ax in axes]
colors = [
'tab:blue', 'tab:pink', 'tab:red', 'tab:cyan', 'tab:olive',
'tab:green', 'tab:purple', 'tab:gray'
]
tau_res_list2d = []
def mult_coeff_res(coeff_res, mult):
return coeff_res._replace(
coefficients=coeff_res.coefficients * mult,
stderrs=coeff_res.stderrs *
mult if coeff_res.stderrs is not None else None)
to_plot = [0, 1, 3, 4]
for cell in range(len(cells)):
taus = []
plot = mre.OutputHandler([], ax=axes_list[cell])
tau_res_list2d.append([])
for i in range(len(labels)):
k_arr = np.arange(1, fs_list[i] * 1)
if spks_2dlist[cell][i] is not None:
act = spks_2dlist[cell][i]
print(act.shape)
len_trial = round(40 * fs_list[i])
act = act[:-(len(act) % len_trial)].reshape((-1, len_trial))
coeff_res = mre.coefficients(act,
k_arr,
dt=1 / fs_list[i] * 1000,
numboot=500,
method='ts')
#norm = 1/coeff_res.coefficients[0]
#coeff_res = mult_coeff_res(coeff_res, norm)
#for j, bootstrapcrs in enumerate(coeff_res.bootstrapcrs):
# coeff_res.bootstrapcrs[j] = mult_coeff_res(bootstrapcrs, norm)
#axes_list[cell].plot(k_arr/fs_list[i]*1000 ,coeff_res.coefficients, color=colors[i],
# label = labels[i] if cell==0 else None)
if i in to_plot:
plot.add_coefficients(
coeff_res,
color=colors[i],
label=labels[i] if cell == 0 else None)
tau_res_list2d[-1].append(
mre.fit(coeff_res,
fitfunc='exponentialoffset',
numboot=500))
else:
tau_res_list2d[-1].append(None)
#axes_list[cell].set_ylim(-0.3,1.2)
axes_list[cell].set_title('cell {}'.format(cell))
plt.legend()
plt.tight_layout()
plt.savefig(
'../reports/different_zoom_levels/autocorrelation_plots_tempSubs2.pdf')
#axes_list[0].set_ylabel('Autocorrelation')
plt.show()
x_ticks = np.arange(len(cells))
offsets = np.linspace(-0.25, 0.25, len(labels))
f, axes_list = plt.subplots(figsize=(10, 6))
for i_ana, offset in enumerate(offsets):
res_list = [
tau_res_list2d[i_cell][i_ana] for i_cell in range(len(cells))
]
taus = np.array(
[res.tau if res is not None else None for res in res_list],
dtype=np.float)
yerr_lower = taus - np.array([
res.tauquantiles[0] if res is not None else None
for res in res_list
],
dtype=np.float)
yerr_upper = np.array([
res.tauquantiles[-1] if res is not None else None
for res in res_list
],
dtype=np.float) - taus
plt.errorbar(x_ticks + offset,
y=taus,
yerr=[yerr_lower, yerr_upper],
color=colors[i_ana],
label=labels[i_ana],
marker="x",
elinewidth=1.5,
capsize=2,
markersize=6,
markeredgewidth=1,
linestyle="")
plt.ylim(0, 400)
plt.legend()
plt.xlabel("Cell number")
plt.ylabel('Timescale (ms)')
#fit_result = mre.fit(coeff_res, steps=k_arr)
#taus.append(fit_result.tau)
plt.savefig('../reports/different_zoom_levels/timescales_tempSubs2.pdf')
plt.show()
def correlation_spatial_subsampling():
#cells = {1: [8,0,1,1,8,8],
# 2: [15,4,3,3,15,15],
# 3: [19,23,19,18,19,19],
# 4: [16,40,88,75,16,16],
# 5: [24,25,43,46,24,24],
# 6: [46,58,48,109,46,46],
# 7: [26,41,37,41,26,26],
# 8: [28,45,31,37,28,28]}
cells = {
1: [8, 0, 1, 1, 1, 2, 2],
2: [15, 4, 3, 3, 2, 1, 11],
3: [19, 23, 19, 18, 18, 19, 240],
4: [16, 40, 88, 75, 85, 22, 51],
5: [24, 25, 43, 46, 38, 39, 255],
6: [46, 58, 48, 109, 44, 42, None],
7: [26, 41, 37, 41, 39, 40, 438],
8: [28, 45, 31, 37, 29, 25, 250]
}
labels = [
'60 Hz, 0.54x0.54$\mu$m, 30 min',
'60 Hz, 0.54x1.08$\mu$m, 30 min (subs.)',
'60 Hz, 1.08x1.08$\mu$m, 30 min (subs.)',
'60 Hz, 1.62x1.62$\mu$m, 30 min (subs.)',
'60 Hz, 2.16x2.16$\mu$m, 30 min (subs.)',
'60 Hz, 2.7x2.7$\mu$m, 30 min (subs.)', '60 Hz, 1.37$\mu$m, 15 min'
]
tau = 1.5
fs_list = [60, 60, 60, 60, 60, 60, 60]
k_arr_list = []
Fc_list = []
directory = '/data.nst/jdehning/packer_data/calcium_subsampled/'
subpath_F = 'suite2p/plane0/F.npy'
subpath_Fneu = 'suite2p/plane0/Fneu.npy'
paths = [
'2019-11-08_RL065_t-001_1x1', '2019-11-08_RL065_t-001_1x2',
'2019-11-08_RL065_t-001_2x2', '2019-11-08_RL065_t-001_3x3',
'2019-11-08_RL065_t-001_4x4', '2019-11-08_RL065_t-001_5x5',
'2019-11-08_RL065_t-002'
]
for path in paths:
F = np.load(os.path.join(directory, path, subpath_F))
Fneu = np.load(os.path.join(directory, path, subpath_Fneu))
Fc_list.append(F - 0.7 * Fneu)
del F, Fneu
spks_2dlist = []
for cell in cells.keys():
spks_2dlist.append([])
for Fc, cell_num, fs, i_rec in zip(Fc_list, cells[cell], fs_list,
range(1000)):
if cell_num is not None:
if i_rec == 4:
beg, end = (0, Fc.shape[1] // 2)
elif i_rec == 5:
beg, end = (Fc.shape[1] // 2, -1)
else:
beg, end = (0, -1)
spks_2dlist[-1].append(
deconvolve(Fc[cell_num][None, beg:end], fs, tau=tau)[0])
else:
spks_2dlist[-1].append(None)
mpl.rcParams["axes.spines.right"] = False
mpl.rcParams["axes.spines.top"] = False
f, axes_list = plt.subplots(4, 2, figsize=(16, 20))
axes_list = [ax for axes in axes_list for ax in axes]
colors = [
'tab:blue', 'tab:pink', 'tab:red', 'tab:cyan', 'tab:olive',
'tab:green', 'k'
]
tau_res_list2d = []
def mult_coeff_res(coeff_res, mult):
return coeff_res._replace(
coefficients=coeff_res.coefficients * mult,
stderrs=coeff_res.stderrs *
mult if coeff_res.stderrs is not None else None)
to_plot = [0, 1, 2, 3, 4, 5]
for cell in range(len(cells)):
taus = []
plot = mre.OutputHandler([], ax=axes_list[cell])
tau_res_list2d.append([])
for i in range(len(labels)):
k_arr = np.arange(1, fs_list[i] * 1)
if spks_2dlist[cell][i] is not None:
act = spks_2dlist[cell][i]
print(act.shape)
len_trial = round(40 * fs_list[i])
act = act[:-(len(act) % len_trial)].reshape((-1, len_trial))
coeff_res = mre.coefficients(act,
k_arr,
dt=1 / fs_list[i] * 1000,
numboot=500,
method='ts')
#norm = 1/coeff_res.coefficients[0]
#coeff_res = mult_coeff_res(coeff_res, norm)
#for j, bootstrapcrs in enumerate(coeff_res.bootstrapcrs):
# coeff_res.bootstrapcrs[j] = mult_coeff_res(bootstrapcrs, norm)
#axes_list[cell].plot(k_arr/fs_list[i]*1000 ,coeff_res.coefficients, color=colors[i],
# label = labels[i] if cell==0 else None)
if i in to_plot:
plot.add_coefficients(
coeff_res,
color=colors[i],
label=labels[i] if cell == 0 else None)
tau_res_list2d[-1].append(
mre.fit(coeff_res,
fitfunc='exponentialoffset',
numboot=500))
else:
tau_res_list2d[-1].append(None)
#axes_list[cell].set_ylim(-0.3,1.2)
axes_list[cell].set_title('cell {}'.format(cell))
plt.legend()
plt.tight_layout()
plt.savefig(
'../reports/spatial_subsampling/autocorrelation_plots_spatialSubs3.pdf'
)
#axes_list[0].set_ylabel('Autocorrelation')
plt.show()
x_ticks = np.arange(len(cells))
offsets = np.linspace(-0.25, 0.25, len(labels))
f, axes_list = plt.subplots(figsize=(10, 6))
for i_ana, offset in enumerate(offsets):
res_list = [
tau_res_list2d[i_cell][i_ana] for i_cell in range(len(cells))
]
taus = np.array(
[res.tau if res is not None else None for res in res_list],
dtype=np.float)
yerr_lower = taus - np.array([
res.tauquantiles[0] if res is not None else None
for res in res_list
],
dtype=np.float)
yerr_upper = np.array([
res.tauquantiles[-1] if res is not None else None
for res in res_list
],
dtype=np.float) - taus
plt.errorbar(x_ticks + offset,
y=taus,
yerr=[yerr_lower, yerr_upper],
color=colors[i_ana],
label=labels[i_ana],
marker="x",
elinewidth=1.5,
capsize=2,
markersize=6,
markeredgewidth=1,
linestyle="")
plt.ylim(0, 400)
plt.legend()
plt.xlabel("Cell number")
plt.ylabel('Timescale (ms)')
#fit_result = mre.fit(coeff_res, steps=k_arr)
#taus.append(fit_result.tau)
plt.savefig('../reports/spatial_subsampling/timescales_spatialSubs3.pdf')
plt.show()
def full_analysis(
data,
dt,
kmax=None,
dtunit=' time unit',
fitfuncs=None,
coefficientmethod=None,
tmin=None, # include somehow into 'missing' req. arg
tmax=None,
steps=None, # dt conversion? optional replace tmin/tmax
substracttrialaverage=False,
targetdir=None,
title=None, # overwrites old files in same targetdir
numboot='auto', # optional. default depends on fitfunc
seed=1, # default: 1 uses hard coded seeds
loglevel=None, # only concerns local logfile
targetplot=None,
showoverview=True,
saveoverview=False,
):
"""
Wrapper function that performs the following four steps:
- check `data` with `input_handler()`
- calculate correlation coefficients via `coefficients()`
- fit autocorrelation function with `fit()`
- export/plot using the `OutputHandler`
Usually it should suffice to tweak the arguments and call this
wrapper function (multiple times).
Calling the underlying functions individually
gives slightly more control, though.
Parameters
----------
data: str, list or numpy.ndarray
Passed to `input_handler()`. Ideally, import and check data first.
A `string` is assumed to be the path
to file(s) that is then imported as pickle or plain text.
Alternatively, you can provide a `list` or `ndarray` containing
strings or already imported data. In the latter case,
`input_handler()` attempts to convert it to the right format.
dt: float
How many `dtunits` separate the measurements of the provided data.
For example, if measurements are taken every 4ms:
`dt=4`, `dtunit=\'ms\'`.
kmax: int
Maximum time lag k (in time steps of size `dt`) to use for
coefficients. Alternatively, `tmax` or `steps` can be specified
Other Parameters
----------------
dtunit: str, optional
Unit description/name of the time steps of the provided data.
fitfuncs: list, optional
Which fitfunctions to use e.g. ``fitfuncs=['e', 'eo', 'c']``.
Renamed from `fitfunctions` in v0.1.4.
coefficientmethod: str, optional
`ts` or `sm`, method used for determining the correlation
coefficients. See the :func:`coefficients` function for details.
Default is `ts`.
tmin: float
Smallest time separation to use for coefficients, in units of
`dtunit`.
Only one argument is possible, either `kmax` or `steps` or
`tmin` and `tmax`.
tmax: float
Maximum time separation to use for coefficients.
For example, to fit the autocorrelation between 8ms and
2s set: `tmin=8`, `tmax=2000`, `dtunit=\'ms\'`
(independent of `dt`).
steps : ~numpy.array, optional
Specify the fitrange in steps :math:`k` for which to compute
coefficients :math:`r_k`.
Note that :math:`k` provided here would need
to be multiplied with units of [`dt` * `dtunit`] to convert
back to (real) time.
If an array of length two is provided, e.g.
``steps=(minstep, maxstep)``, all enclosed integer values will be
used.
Arrays larger than two are assumed to contain a manual choice of
steps. Strides other than one are possible.
Only one argument is possible, either `steps` or `kmax` or
`tmin` and `tmax`.
substracttrialaverage: bool, optional
Substract the average across all trials before calculating
correlation coefficients.
Default is `False`.
targetdir: str, optional
String containing the path to the target directory where files
are saved with the filename `title`.
Per default, `targetdir=None` and no files are written to disk.
title: str, optional
String for the filenames. Also sets the main title of the
overview panel.
numboot: int or 'auto', optional
Number of bootstrap samples to draw.
This repeats every fit `numboot` times so that we can
provide an uncertainty estimate of the resulting branching
parameter and autocorrelation time.
Per default, bootstrapping is only applied in
`coefficeints()` as most of computing time is needed for the
fitting. Thereby we have uncertainties on the :math:`r_k`
(which will be plotted) but each fit is only
done once.
Default is `numboot='auto'` where the number of samples depends on
the fitfunction (100 for the exponential).
seed : int, None or 'random', optional
If `numboot` is not zero, provide a seed for the random number
generator. If `seed=None`, seeding will be skipped.
Per default, the rng is (re)seeded every time `full_analysis()` is
called so that every repeated call returns the same error
estimates.
loglevel: str, optional
The loglevel to use for the logfile created
as `title.log` in the `targetdir`.
'ERROR', 'WARNING', 'INFO' or 'DEBUG'.
Per default, no log is written unless `loglevel` and `targetdir`
are provided.
targetplot: `matplotlib.axes.Axes`, optional
You can provide a matplotlib axes element (i.e. a subplot of an
existing figure) to plot the correlations into.
The axis will be passed to the `OutputHandler`
and all plotting will happen within that axes.
Per default, a new figure is created - that cannot be added
as a subplot to any other figure later on. This is due to
the way matplotlib handles subplots.
showoverview: bool, optional
Wether to show the overview panel. Default is 'True'.
Note that even when set to 'True' the panel might not show if
`full_analysis()` is called through a script instead of an
(interactive) shell.
saveoverview: bool, optional
Wether to save the overview panel in `targetdir`.
Default is 'False'.
Returns
-------
OutputHandler
that is associated
with the correlation plot, fits and coefficients.
Also saves meta data and plotted pdfs to `targetdir`.
Example
-------
.. code-block:: python
# test data, subsampled branching process
bp = mre.simulate_branching(m=0.95, h=10, subp=0.1, numtrials=50)
mre.full_analysis(
data=bp,
dt=1,
tmin=0, tmax=1500,
dtunit='step',
fitfuncs=['exp', 'exp_offs', 'complex'],
targetdir='./output',
title='Branching Process')
..
"""
# ------------------------------------------------------------------ #
# Arguments
# ------------------------------------------------------------------ #
# workaround: if full_analysis() does not reach its end where we remove
# the local loghandler, it survives and keps logging with the old level
for hdlr in log.handlers:
if isinstance(hdlr, logging.FileHandler):
if hdlr != ut._logfilehandler:
hdlr.close()
log.removeHandler(hdlr)
if kmax is None and tmax is None and steps is None:
log.exception("full_analysis() requires one of the following keyword" +
"arguments: 'kmax', 'tmax' or 'steps'")
raise TypeError
# if there is a targetdir specified, create and use for various output
if targetdir is not None:
if isinstance(targetdir, str):
td = os.path.abspath(os.path.expanduser(targetdir + '/'))
os.makedirs(td, exist_ok=True)
ut._set_permissions(td)
targetdir = td
else:
log.exception("Argument 'targetdir' needs to be of type 'str'")
raise TypeError
# setup log early so argument errors appear in the logfile
if loglevel is None:
# dont create a logfile
pass
else:
if isinstance(loglevel, int) and loglevel > 0:
pass
elif str(loglevel).upper() in [
'ERROR', 'WARNING', 'INFO', 'DEBUG'
]:
loglevel = str(loglevel).upper()
else:
log.debug(
"Unrecognized log level {}, using 'INFO'".format(loglevel))
loglevel = 'INFO'
# open new handler and add it to logging module
loghandler = logging.handlers.RotatingFileHandler(
targetdir + '/{}.log'.format(
'full_analysis' if title is None else title, 'a'),
maxBytes=5 * 1024 * 1024,
backupCount=1)
loghandler.setLevel(logging.getLevelName(loglevel))
loghandler.setFormatter(
ut.CustomExceptionFormatter(
'%(asctime)s %(levelname)8s: %(message)s',
"%Y-%m-%d %H:%M:%S"))
log.addHandler(loghandler)
else:
if saveoverview:
log.warning("Cannot save overview since no targetdir specified, "+\
"skipping")
log.debug("full_analysis()")
if (ut._log_locals):
log.debug('Locals: {}'.format(locals()))
try:
dt = float(dt)
assert (dt > 0)
except Exception as e:
log.exception("Argument 'dt' needs to be a float > 0")
raise
if not isinstance(dtunit, str):
log.exception("Argument 'dtunit' needs to be of type 'str'")
raise TypeError
if steps is None:
if kmax is not None:
try:
kmax = float(kmax)
assert (kmax > 0)
except Exception as e:
log.exception("Argument 'kmax' needs to be a number > 0")
raise
if tmax is not None:
log.exception("Arguments do not match: Please provide either"+\
" 'kmax' or 'tmin' and 'tmax' or 'steps'")
raise TypeError
else:
tmax = kmax * dt
if tmin is None:
tmin = 1
try:
tmin = float(tmin)
tmax = float(tmax)
assert (tmin >= 0 and tmax > tmin)
except Exception as e:
log.exception("Arguments: 'tmax' and 'tmin' " +
"need to be floats with 'tmax' > 'tmin' >= 0")
raise
steps = (int(tmin / dt), int(tmax / dt))
else:
if tmin is not None or tmax is not None or kmax is not None:
log.exception("Arguments do not match: Please provide either "+\
"'kmax' or 'tmin' and 'tmax' or 'steps'")
raise TypeError
log.debug("Argument 'steps' was provided to full_analysis()")
defaultfits = False
if fitfuncs is None:
fitfuncs = ['e', 'eo']
defaultfits = True
elif isinstance(fitfuncs, str):
fitfuncs = [fitfuncs]
if not isinstance(fitfuncs, list):
log.exception("Argument 'fitfuncs' needs to be of type 'str' or " +\
"a list e.g. ['exponential', 'exponential_offset']")
raise TypeError
if coefficientmethod is None:
coefficientmethod = 'trialseparated'
if coefficientmethod not in [
'trialseparated', 'ts', 'stationarymean', 'sm'
]:
log.exception("Optional argument 'coefficientmethod' needs " +
"to be either 'trialseparated' or 'stationarymean'")
raise TypeError
if targetplot is not None \
and not isinstance(targetplot, matplotlib.axes.Axes):
log.exception("Optional argument 'targetplot' needs " +
"to be an instance of 'matplotlib.axes.Axes'")
raise TypeError
if title is not None:
title = str(title)
if (ut._log_locals):
log.debug('Finished argument check. Locals: {}'.format(locals()))
# ------------------------------------------------------------------ #
# Continue with trusted arguments
# ------------------------------------------------------------------ #
src = input_handler(data)
if substracttrialaverage:
src = src - np.mean(src, axis=0)
log.debug('full_analysis() seeding to {}'.format(seed))
if seed is None or seed == 'random':
rkseed = seed
ftseed = seed
else:
rkseed = seed * 5330
ftseed = seed * 101
if numboot == 'auto':
nbt = 100
else:
nbt = numboot
rks = coefficients(src,
steps,
dt,
dtunit,
method=coefficientmethod,
numboot=nbt,
seed=rkseed)
fits = []
for f in fitfuncs:
if numboot == 'auto':
if fitfunc_check(f) is f_exponential or \
fitfunc_check(f) is f_exponential_offset:
nbt = 100
elif fitfunc_check(f) is f_complex:
nbt = 0
else:
nbt = 100
else:
nbt = numboot
fits.append(
fit(data=rks, fitfunc=f, steps=steps, numboot=nbt, seed=ftseed))
# ------------------------------------------------------------------ #
# Output and Consistency Checks
# ------------------------------------------------------------------ #
warning = None
if defaultfits:
shownfits = [fits[0]]
# no trials, no confidence
if src.shape[0] == 1:
warning = 'Not enough trials to calculate confidence intervals.'
# check that tau from exp and exp_off
elif not ut._c_fits_consistent(fits[0], fits[1]):
# warning = 'Exponential with offset resulted in ' + \
# '$\\tau = {:.2f}$ {}'.format(fits[1].tau, fits[1].dtunit)
warning = 'Results from other fits differed beyond confidence.\n'+\
"Try the 'fitfuncs' argument!"
else:
shownfits = fits
warning = None
if showoverview or saveoverview:
panel = overview(src, [rks], shownfits, title=title, warning=warning)
res = OutputHandler([rks] + shownfits, ax=targetplot)
if targetdir is not None:
if (title is not None and title != ''):
res.save(targetdir + "/" + title)
if saveoverview:
panel.savefig(targetdir + "/" + title + "_overview.pdf")
else:
res.save(targetdir + "/full_analysis")
if saveoverview:
panel.savefig(targetdir + "/full_analysis_overview.pdf")
if showoverview:
panel.show()
elif saveoverview:
# if interactive mode is on, panel would still be shown
try:
plt.close(panel)
except:
log.debug('Exception passed', exc_info=True)
try:
log.removeHandler(loghandler)
except:
log.debug('No handler to remove')
log.info("full_analysis() done")
return res
def fit_tau(act, fs, numboot = 0, k_arr = None):
if k_arr is None:
k_arr = np.arange(1, fs * 1)
coeff_res = mre.coefficients(act, k_arr, dt=1/fs* 1000, numboot=numboot, method='ts')
tau_res = mre.fit(coeff_res, fitfunc='exponentialoffset', numboot=numboot)
return tau_res.tau
def plot(self, directories, nsps, fig, axs):
ax = axs[0][0]
fontsize = 18
revreg, subcrit = "reverberating", "sub-critical"
colors = {revreg: "green", subcrit: "blue"}
data, mres = {revreg: [], subcrit: []}, {revreg: [], subcrit: []}
for dir, nsp in zip(directories, nsps):
# we assume that only directories with data have been selected
# that passes the stationary tests
rk, ft, _ = branching_ratio('', dir, nsp, self.bin_w)
fit = mre.fit(rk, fitfunc=mre.f_exponential_offset)
if fit.mre > 0.995:
print(f"skipping: mre is critical or super-critical: {dir}")
continue
group = revreg if fit.mre > 0.9 else subcrit
data[group].append(self.unpickle(dir, "survival_full_t"))
mres[group].append(fit.mre)
assert (len(data[revreg]) > 0 and len(data[subcrit]) > 0)
# strct plasticity is only applied every 1000ms
# so bin size needs to be at least 1s, which logspace doesn't
# do for the range <10e1
bins = np.logspace(np.log10(10e1),
np.log10(
np.max([
data[revreg][i]['t_split'] / second
for i in range(0, len(data[revreg]))
]) + 0.1),
num=70) # 82 exhibits jump in blue curve
bins = np.hstack((np.arange(1, 10, 1), bins))
centers = (bins[:-1] + bins[1:]) / 2.
ax.loglog()
for label, dfs in data.items():
color = colors[label]
counts = [
np.histogram(df['full_t'], bins=bins, density=True)[0]
for df in dfs
]
avg, std = np.mean(counts, axis=0), np.std(counts, axis=0)
ax.plot(centers, avg, label=label, color=color)
ax.fill_between(centers,
avg - std,
avg + std,
facecolor=f"light{color}",
linewidth=0)
group_info = [(label, len(dfs), np.mean(mres[label]),
np.std(mres[label])) for label, dfs in data.items()]
group_info_str = [
f"{label}: N={no:2d}, $\hat{{m}}={mean:.2f}\pm{std:.2f}$"
for label, no, mean, std in group_info
]
ax.legend(loc="lower left", fontsize=fontsize)
# ax.text(1.0, -0.25, '\n'.join(group_info_str), transform=ax.transAxes, ha='right')
ax.set_xlabel("survival time [s]", fontsize=fontsize)
ax.set_ylabel("probaiblity density", fontsize=fontsize)
ax.set_title("Survival times of EE synapses", fontsize=fontsize)
ax.tick_params(axis='both', which='major', labelsize=fontsize)
# ------------------------------------------------------------------ #
# analyzing
# ------------------------------------------------------------------ #
# correlation coefficients with default settings, assumes 1ms time bins
rkdefault = mre.coefficients(srcful)
print(rkdefault)
print('this guy has the following attributes: ', rkdefault._fields)
# specify the range of time steps (from, to) for which coefficients are wanted
# also, set the unit and the number of time steps per bin e.g. 4ms per k:
rk = mre.coefficients(srcsub, steps=(1, 5000), dt=4, dtunit='ms', desc='mydat')
# fit with defaults: exponential over the full range of rk
m = mre.fit(rk)
# specify a custom fit range and fitfunction.
m2 = mre.fit(rk, steps=(1, 3000), fitfunc='offset')
# you could also provide an np array containing all the steps you want to
# use, e.g. with strides other than one
# m2 = mre.fit(rk, steps=np.arange(1, 3000, 100), fitfunc='offset')
# Plot with a new handler, you can provide multiple things to add
ores = mre.OutputHandler([rkdefault, m])
# or add them individually, later
ores.add_coefficients(rk)
ores.add_fit(m2)
# Note the different time scales
# The description provided to mre.coefficients is automatically used for
# subsequent steps and becomes the axis label
