def plot_autocorr(analyspar, sesspar, stimpar, extrapar, autocorrpar,
sess_info, autocorr_data, figpar=None, savedir=None):
"""
plot_autocorr(analyspar, sesspar, stimpar, extrapar, autocorrpar,
sess_info, autocorr_data)
From dictionaries, plots autocorrelation during stimulus blocks.
Required args:
- analyspar (dict) : dictionary with keys of AnalysPar namedtuple
- sesspar (dict) : dictionary with keys of SessPar namedtuple
- stimpar (dict) : dictionary with keys of StimPar namedtuple
- extrapar (dict) : dictionary containing additional analysis
parameters
["analysis"] (str): analysis type (e.g., "a")
["datatype"] (str): datatype (e.g., "run", "roi")
- autocorrpar (dict) : dictionary with keys of AutocorrPar namedtuple
- sess_info (dict) : dictionary containing information from each
session
["mouse_ns"] (list) : mouse numbers
["sess_ns"] (list) : session numbers
["lines"] (list) : mouse lines
["planes"] (list) : imaging planes
["nrois"] (list) : number of ROIs in session
- autocorr_data (dict): dictionary containing data to plot:
["xrans"] (list): list of lag values in seconds for each session
["stats"] (list): list of 3D arrays (or nested lists) of
autocorrelation statistics, structured as:
sessions stats (me, err)
x ROI or 1x and 10x lag
x lag
Optional args:
- figpar (dict): dictionary containing the following figure parameter
dictionaries
default: None
["init"] (dict): dictionary with figure initialization parameters
["save"] (dict): dictionary with figure saving parameters
["dirs"] (dict): dictionary with additional figure parameters
- savedir (str): path of directory in which to save plots.
default: None
Returns:
- fulldir (str) : final path of the directory in which the figure is
saved (may differ from input savedir, if datetime
subfolder is added.)
- savename (str): name under which the figure is saved
"""
statstr_pr = sess_str_util.stat_par_str(
analyspar["stats"], analyspar["error"], "print")
stimstr_pr = sess_str_util.stim_par_str(
stimpar["stimtype"], stimpar["visflow_dir"], stimpar["visflow_size"],
stimpar["gabk"], "print")
dendstr_pr = sess_str_util.dend_par_str(
analyspar["dend"], sesspar["plane"], extrapar["datatype"], "print")
sessstr = sess_str_util.sess_par_str(
sesspar["sess_n"], stimpar["stimtype"], sesspar["plane"],
stimpar["visflow_dir"],stimpar["visflow_size"], stimpar["gabk"])
dendstr = sess_str_util.dend_par_str(
analyspar["dend"], sesspar["plane"], extrapar["datatype"])
datatype = extrapar["datatype"]
if datatype == "roi":
fluorstr_pr = sess_str_util.fluor_par_str(
analyspar["fluor"], str_type="print")
if autocorrpar["byitem"]:
title_str = u"{}\nautocorrelation".format(fluorstr_pr)
else:
title_str = "\nautocorr. acr. ROIs" .format(fluorstr_pr)
elif datatype == "run":
datastr = sess_str_util.datatype_par_str(datatype)
title_str = u"\n{} autocorrelation".format(datastr)
if stimpar["stimtype"] == "gabors":
seq_bars = [-1.5, 1.5] # light lines
else:
seq_bars = [-1.0, 1.0] # light lines
# extract some info from sess_info
keys = ["mouse_ns", "sess_ns", "lines", "planes"]
[mouse_ns, sess_ns, lines, planes] = [sess_info[key] for key in keys]
nroi_strs = sess_str_util.get_nroi_strs(sess_info, empty=(datatype!="roi"))
n_sess = len(mouse_ns)
xrans = autocorr_data["xrans"]
stats = [np.asarray(stat) for stat in autocorr_data["stats"]]
lag_s = autocorrpar["lag_s"]
xticks = np.linspace(-lag_s, lag_s, lag_s*2+1)
yticks = np.linspace(0, 1, 6)
if figpar is None:
figpar = sess_plot_util.init_figpar()
byitemstr = ""
if autocorrpar["byitem"]:
byitemstr = "_byroi"
fig, ax = plot_util.init_fig(n_sess, **figpar["init"])
for i in range(n_sess):
sub_ax = plot_util.get_subax(ax, i)
title = (f"Mouse {mouse_ns[i]} - {stimstr_pr}, "
u"{} ".format(statstr_pr) + f"{title_str} (sess "
f"{sess_ns[i]}, {lines[i]} {planes[i]}{dendstr_pr}{nroi_strs[i]})")
# transpose to ROI/lag x stats x series
sess_stats = stats[i].transpose(1, 0, 2)
for s, sub_stats in enumerate(sess_stats):
lab = None
if not autocorrpar["byitem"]:
lab = ["actual lag", "10x lag"][s]
plot_util.plot_traces(
sub_ax, xrans[i], sub_stats[0], sub_stats[1:], xticks=xticks,
yticks=yticks, alpha=0.2, label=lab)
plot_util.add_bars(sub_ax, hbars=seq_bars)
sub_ax.set_ylim([0, 1])
sub_ax.set_title(title, y=1.02)
if sub_ax.is_last_row():
sub_ax.set_xlabel("Lag (s)")
plot_util.turn_off_extra(ax, n_sess)
if savedir is None:
savedir = Path(
figpar["dirs"][datatype],
figpar["dirs"]["autocorr"])
savename = (f"{datatype}_autocorr{byitemstr}_{sessstr}{dendstr}")
fulldir = plot_util.savefig(fig, savename, savedir, **figpar["save"])
return fulldir, savename
def plot_traces_by_qu_lock_sess(analyspar, sesspar, stimpar, extrapar,
quantpar, sess_info, trace_stats,
figpar=None, savedir=None, modif=False):
"""
plot_traces_by_qu_lock_sess(analyspar, sesspar, stimpar, extrapar,
quantpar, sess_info, trace_stats)
From dictionaries, plots traces by quantile, locked to transitions from
unexpected to expected or v.v. with each session in a separate subplot.
Returns figure name and save directory path.
Required args:
- analyspar (dict) : dictionary with keys of AnalysPar namedtuple
- sesspar (dict) : dictionary with keys of SessPar namedtuple
- stimpar (dict) : dictionary with keys of StimPar namedtuple
- extrapar (dict) : dictionary containing additional analysis
parameters
["analysis"] (str): analysis type (e.g., "l")
["datatype"] (str): datatype (e.g., "run", "roi")
- quantpar (dict) : dictionary with keys of QuantPar namedtuple
- sess_info (dict) : dictionary containing information from each
session
["mouse_ns"] (list) : mouse numbers
["sess_ns"] (list) : session numbers
["lines"] (list) : mouse lines
["planes"] (list) : imaging planes
if datatype ==
["nrois"] (list) : number of ROIs in session
- trace_stats (dict): dictionary containing trace stats information
["xrans"] (list) : time values for the 2p frames for each
session
["all_stats"] (list) : list of 4D arrays or lists of trace
data statistics across ROIs for each
session, structured as:
(unexp_len x) quantiles x
stats (me, err) x frames
["all_counts"] (array-like): number of sequences, structured as:
sess x (unexp_len x) quantiles
["lock"] (str) : value to which segments are locked:
"unexp", "exp" or "unexp_split"
["baseline"] (num) : number of seconds used for baseline
["exp_stats"] (list) : list of 3D arrays or lists of trace
data statistics across ROIs for
expected sampled sequences,
structured as:
quantiles (1) x stats (me, err)
x frames
["exp_counts"] (array-like): number of sequences corresponding to
exp_stats, structured as:
sess x quantiles (1)
if data is by unexp_len:
["unexp_lens"] (list) : number of consecutive segments for
each unexp_len, structured by session
Optional args:
- figpar (dict): dictionary containing the following figure parameter
dictionaries
default: None
["init"] (dict): dictionary with figure initialization parameters
["save"] (dict): dictionary with figure saving parameters
["dirs"] (dict): dictionary with additional figure parameters
- savedir (str): path of directory in which to save plots.
default: None
- modif (bool) : if True, modified (slimmed-down) plots are created
instead
default: False
Returns:
- fulldir (str) : final path of the directory in which the figure is
saved (may differ from input savedir, if datetime
subfolder is added.)
- savename (str): name under which the figure is saved
"""
analyspar["dend"] = None
stimstr_pr = sess_str_util.stim_par_str(
stimpar["stimtype"], stimpar["visflow_dir"], stimpar["visflow_size"],
stimpar["gabk"], "print")
statstr_pr = sess_str_util.stat_par_str(
analyspar["stats"], analyspar["error"], "print")
dendstr_pr = sess_str_util.dend_par_str(
analyspar["dend"], sesspar["plane"], extrapar["datatype"], "print")
sessstr = sess_str_util.sess_par_str(
sesspar["sess_n"], stimpar["stimtype"], sesspar["plane"],
stimpar["visflow_dir"], stimpar["visflow_size"], stimpar["gabk"])
dendstr = sess_str_util.dend_par_str(
analyspar["dend"], sesspar["plane"], extrapar["datatype"])
basestr = sess_str_util.base_par_str(trace_stats["baseline"])
basestr_pr = sess_str_util.base_par_str(trace_stats["baseline"], "print")
datatype = extrapar["datatype"]
dimstr = sess_str_util.datatype_dim_str(datatype)
# extract some info from sess_info
keys = ["mouse_ns", "sess_ns", "lines", "planes"]
[mouse_ns, sess_ns, lines, planes] = [sess_info[key] for key in keys]
nroi_strs = sess_str_util.get_nroi_strs(sess_info, empty=(datatype!="roi"))
n_sess = len(mouse_ns)
xrans = [np.asarray(xran) for xran in trace_stats["xrans"]]
all_stats = [np.asarray(sessst) for sessst in trace_stats["all_stats"]]
exp_stats = [np.asarray(expst) for expst in trace_stats["exp_stats"]]
all_counts = trace_stats["all_counts"]
exp_counts = trace_stats["exp_counts"]
lock = trace_stats["lock"]
col_idx = 0
if "unexp" in lock:
lock = "unexp"
col_idx = 1
# plot unexp_lens default values
if stimpar["stimtype"] == "gabors":
DEFAULT_UNEXP_LEN = [3.0, 4.5, 6.0]
if stimpar["gabfr"] not in ["any", "all"]:
offset = sess_str_util.gabfr_nbrs(stimpar["gabfr"])
else:
DEFAULT_UNEXP_LEN = [2.0, 3.0, 4.0]
offset = 0
unexp_lab, len_ext = "", ""
unexp_lens = [[None]] * n_sess
unexp_len_default = True
if "unexp_lens" in trace_stats.keys():
unexp_len_default = False
unexp_lens = trace_stats["unexp_lens"]
len_ext = "_bylen"
if stimpar["stimtype"] == "gabors":
unexp_lens = [
[sl * 1.5/5 - 0.3 * offset for sl in sls] for sls in unexp_lens
]
inv = 1 if lock == "unexp" else -1
# RANGE TO PLOT
if modif:
st_val = -2.0
end_val = 6.0
n_ticks = int((end_val - st_val) // 2 + 1)
else:
n_ticks = 21
if figpar is None:
figpar = sess_plot_util.init_figpar()
figpar = copy.deepcopy(figpar)
if modif:
figpar["init"]["subplot_wid"] = 6.5
else:
figpar["init"]["subplot_wid"] *= 2
fig, ax = plot_util.init_fig(n_sess, **figpar["init"])
exp_min, exp_max = np.inf, -np.inf
for i, (stats, counts) in enumerate(zip(all_stats, all_counts)):
sub_ax = plot_util.get_subax(ax, i)
# plot expected data
if exp_stats[i].shape[0] != 1:
raise ValueError("Expected only one quantile for exp_stats.")
n_lines = quantpar["n_quants"] * len(unexp_lens[i])
cols = sess_plot_util.get_quant_cols(n_lines)[0][col_idx]
if len(cols) < n_lines:
cols = [None] * n_lines
if modif:
line = "2/3" if "23" in lines[i] else "5"
plane = "somata" if "soma" in planes[i] else "dendrites"
title = f"M{mouse_ns[i]} - layer {line} {plane}{dendstr_pr}"
lab = "exp" if i == 0 else None
y_ax = None if i == 0 else ""
st, end = 0, len(xrans[i])
st_vals = list(filter(
lambda j: xrans[i][j] <= st_val, range(len(xrans[i]))
))
end_vals = list(filter(
lambda j: xrans[i][j] >= end_val, range(len(xrans[i]))
))
if len(st_vals) != 0:
st = st_vals[-1]
if len(end_vals) != 0:
end = end_vals[0] + 1
time_slice = slice(st, end)
else:
title = (f"Mouse {mouse_ns[i]} - {stimstr_pr}, "
u"{} ".format(statstr_pr) + f"{lock} locked across {dimstr}"
f"{basestr_pr}\n(sess {sess_ns[i]}, {lines[i]} {planes[i]}"
f"{dendstr_pr}{nroi_strs[i]})")
lab = f"exp (no lock) ({exp_counts[i][0]})"
y_ax = None
st = 0
end = len(xrans[i])
time_slice = slice(None) # use all
# add length markers
use_unexp_lens = unexp_lens[i]
if unexp_len_default:
use_unexp_lens = DEFAULT_UNEXP_LEN
leng_col = sess_plot_util.get_quant_cols(1)[0][col_idx][0]
for leng in use_unexp_lens:
if leng is None:
continue
edge = leng * inv
if edge < 0:
edge = np.max([xrans[i][st], edge])
elif edge > 0:
edge = np.min([xrans[i][end - 1], edge])
plot_util.add_vshade(
sub_ax, 0, edge, color=leng_col, alpha=0.1)
sess_plot_util.add_axislabels(
sub_ax, fluor=analyspar["fluor"], datatype=datatype, y_ax=y_ax
)
plot_util.add_bars(sub_ax, hbars=0)
alpha = np.min([0.4, 0.8 / n_lines])
if stimpar["stimtype"] == "gabors":
sess_plot_util.plot_gabfr_pattern(
sub_ax, xrans[i], offset=offset, bars_omit=[0] + unexp_lens[i]
)
plot_util.plot_traces(
sub_ax, xrans[i][time_slice], exp_stats[i][0][0, time_slice],
exp_stats[i][0][1:, time_slice], n_xticks=n_ticks,
alpha=alpha, label=lab, alpha_line=0.8, color="darkgray",
xticks="auto")
# get expected data range to adjust y lims
exp_min = np.min([exp_min, np.nanmin(exp_stats[i][0][0])])
exp_max = np.max([exp_max, np.nanmax(exp_stats[i][0][0])])
n = 0 # count lines plotted
for s, unexp_len in enumerate(unexp_lens[i]):
if unexp_len is not None:
counts, stats = all_counts[i][s], all_stats[i][s]
# remove offset
unexp_lab = f"unexp len {unexp_len + 0.3 * offset}"
else:
unexp_lab = "unexp" if modif else f"{lock} lock"
for q, qu_idx in enumerate(quantpar["qu_idx"]):
qu_lab = ""
if quantpar["n_quants"] > 1:
qu_lab = "{} ".format(sess_str_util.quantile_str(
qu_idx, quantpar["n_quants"], str_type="print"
))
lab = f"{qu_lab}{unexp_lab}"
if modif:
lab = lab if i == 0 else None
else:
lab = f"{lab} ({counts[q]})"
if n == 2 and cols[n] is None:
sub_ax.plot([], []) # to advance the color cycle (past gray)
plot_util.plot_traces(sub_ax, xrans[i][time_slice],
stats[q][0, time_slice], stats[q][1:, time_slice], title,
alpha=alpha, label=lab, n_xticks=n_ticks, alpha_line=0.8,
color=cols[n], xticks="auto")
n += 1
if unexp_len is not None:
plot_util.add_bars(
sub_ax, hbars=unexp_len, color=sub_ax.lines[-1].get_color(),
alpha=1)
plot_util.turn_off_extra(ax, n_sess)
if savedir is None:
savedir = Path(
figpar["dirs"][datatype],
figpar["dirs"]["unexp_qu"],
f"{lock}_lock", basestr.replace("_", ""))
if not modif:
if stimpar["stimtype"] == "visflow":
plot_util.rel_confine_ylims(sub_ax, [exp_min, exp_max], 5)
qu_str = f"_{quantpar['n_quants']}q"
if quantpar["n_quants"] == 1:
qu_str = ""
savename = (f"{datatype}_av_{lock}_lock{len_ext}{basestr}_{sessstr}"
f"{dendstr}{qu_str}")
fulldir = plot_util.savefig(fig, savename, savedir, **figpar["save"])
return fulldir, savename
def plot_traces_by_qu_unexp_sess(analyspar, sesspar, stimpar, extrapar,
quantpar, sess_info, trace_stats, figpar=None,
savedir=None, modif=False):
"""
plot_traces_by_qu_unexp_sess(analyspar, sesspar, stimpar, extrapar,
quantpar, sess_info, trace_stats)
From dictionaries, plots traces by quantile/unexpected with each session in a
separate subplot.
Returns figure name and save directory path.
Required args:
- analyspar (dict) : dictionary with keys of AnalysPar namedtuple
- sesspar (dict) : dictionary with keys of SessPar namedtuple
- stimpar (dict) : dictionary with keys of StimPar namedtuple
- extrapar (dict) : dictionary containing additional analysis
parameters
["analysis"] (str): analysis type (e.g., "t")
["datatype"] (str): datatype (e.g., "run", "roi")
- quantpar (dict) : dictionary with keys of QuantPar namedtuple
- sess_info (dict) : dictionary containing information from each
session
["mouse_ns"] (list) : mouse numbers
["sess_ns"] (list) : session numbers
["lines"] (list) : mouse lines
["planes"] (list) : imaging planes
if extrapar["datatype"] == "roi":
["nrois"] (list) : number of ROIs in session
- trace_stats (dict): dictionary containing trace stats information
["xrans"] (list) : time values for the frames, for each
session
["all_stats"] (list) : list of 4D arrays or lists of trace
data statistics across ROIs for each
session, structured as:
sess x unexp x quantiles x
stats (me, err) x frames
["all_counts"] (array-like): number of sequences, structured as:
sess x unexp x quantiles
Optional args:
- figpar (dict): dictionary containing the following figure parameter
dictionaries
default: None
["init"] (dict): dictionary with figure initialization parameters
["save"] (dict): dictionary with figure saving parameters
["dirs"] (dict): dictionary with additional figure parameters
- savedir (str): path of directory in which to save plots.
default: None
- modif (bool) : if True, modified (slimmed-down) plots are created
instead
default: False
Returns:
- fulldir (str) : final path of the directory in which the figure is
saved (may differ from input savedir, if datetime
subfolder is added.)
- savename (str): name under which the figure is saved
"""
stimstr_pr = sess_str_util.stim_par_str(
stimpar["stimtype"], stimpar["visflow_dir"], stimpar["visflow_size"],
stimpar["gabk"], "print")
statstr_pr = sess_str_util.stat_par_str(
analyspar["stats"], analyspar["error"], "print")
dendstr_pr = sess_str_util.dend_par_str(
analyspar["dend"], sesspar["plane"], extrapar["datatype"], "print")
sessstr = sess_str_util.sess_par_str(
sesspar["sess_n"], stimpar["stimtype"], sesspar["plane"],
stimpar["visflow_dir"], stimpar["visflow_size"], stimpar["gabk"])
dendstr = sess_str_util.dend_par_str(
analyspar["dend"], sesspar["plane"], extrapar["datatype"])
datatype = extrapar["datatype"]
dimstr = sess_str_util.datatype_dim_str(datatype)
# extract some info from sess_info
keys = ["mouse_ns", "sess_ns", "lines", "planes"]
[mouse_ns, sess_ns, lines, planes] = [sess_info[key] for key in keys]
nroi_strs = sess_str_util.get_nroi_strs(sess_info, empty=(datatype!="roi"))
n_sess = len(mouse_ns)
xrans = [np.asarray(xran) for xran in trace_stats["xrans"]]
all_stats = [np.asarray(sessst) for sessst in trace_stats["all_stats"]]
all_counts = trace_stats["all_counts"]
cols, lab_cols = sess_plot_util.get_quant_cols(quantpar["n_quants"])
alpha = np.min([0.4, 0.8 / quantpar["n_quants"]])
unexps = ["exp", "unexp"]
n = 6
if figpar is None:
figpar = sess_plot_util.init_figpar()
fig, ax = plot_util.init_fig(n_sess, **figpar["init"])
for i in range(n_sess):
sub_ax = plot_util.get_subax(ax, i)
for s, [col, leg_ext] in enumerate(zip(cols, unexps)):
for q, qu_idx in enumerate(quantpar["qu_idx"]):
qu_lab = ""
if quantpar["n_quants"] > 1:
qu_lab = "{} ".format(sess_str_util.quantile_str(
qu_idx, quantpar["n_quants"], str_type="print"
))
if modif:
line = "2/3" if "23" in lines[i] else "5"
plane = "somata" if "soma" in planes[i] else "dendrites"
title = f"M{mouse_ns[i]} - layer {line} {plane}{dendstr_pr}"
leg = f"{qu_lab}{leg_ext}" if i == 0 else None
y_ax = None if i == 0 else ""
else:
title=(f"Mouse {mouse_ns[i]} - {stimstr_pr}, "
u"{}\n".format(statstr_pr) + f"across {dimstr} (sess "
f"{sess_ns[i]}, {lines[i]} {planes[i]}{dendstr_pr}"
f"{nroi_strs[i]})")
leg = f"{qu_lab}{leg_ext} ({all_counts[i][s][q]})"
y_ax = None
plot_util.plot_traces(
sub_ax, xrans[i], all_stats[i][s, q, 0],
all_stats[i][s, q, 1:], title, color=col[q], alpha=alpha,
label=leg, n_xticks=n, xticks="auto")
sess_plot_util.add_axislabels(
sub_ax, fluor=analyspar["fluor"], datatype=datatype,
y_ax=y_ax)
plot_util.turn_off_extra(ax, n_sess)
if stimpar["stimtype"] == "gabors":
sess_plot_util.plot_labels(
ax, stimpar["gabfr"], "both", pre=stimpar["pre"],
post=stimpar["post"], cols=lab_cols,
sharey=figpar["init"]["sharey"])
if savedir is None:
savedir = Path(
figpar["dirs"][datatype],
figpar["dirs"]["unexp_qu"])
qu_str = f"_{quantpar['n_quants']}q"
if quantpar["n_quants"] == 1:
qu_str = ""
savename = f"{datatype}_av_{sessstr}{dendstr}{qu_str}"
fulldir = plot_util.savefig(fig, savename, savedir, **figpar["save"])
return fulldir, savename
def plot_full_traces(analyspar, sesspar, extrapar, sess_info, trace_info,
roi_tr=None, figpar=None, savedir=None):
"""
plot_full_traces(analyspar, sesspar, extrapar, sess_info, trace_info)
From dictionaries, plots full traces for each session in a separate subplot.
Returns figure name and save directory path.
Required args:
- analyspar (dict) : dictionary with keys of AnalysPar namedtuple
- sesspar (dict) : dictionary with keys of SessPar namedtuple
- extrapar (dict) : dictionary containing additional analysis
parameters
["analysis"] (str): analysis type (e.g., "f")
["datatype"] (str): datatype (e.g., "run", "roi")
- sess_info (dict) : dictionary containing information from each
session
["mouse_ns"] (list) : mouse numbers
["sess_ns"] (list) : session numbers
["lines"] (list) : mouse lines
["planes"] (list) : imaging planes
["nrois"] (list) : number of ROIs in session
- trace_info (dict): dictionary containing trace information
["all_tr"] (nested list): trace values structured as
sess x
(me/err if datatype is "roi" x)
frames
["all_edges"] (list) : frame edge values for each parameter,
structured as sess x block x
edges ([start, end])
["all_pars"] (list) : stimulus parameter strings structured as
sess x block
Optional args:
- roi_tr (list): trace values for each ROI, structured as
sess x ROI x frames
default: None
- figpar (dict): dictionary containing the following figure parameter
dictionaries
default: None
["init"] (dict): dictionary with figure initialization parameters
["save"] (dict): dictionary with figure saving parameters
["dirs"] (dict): dictionary with additional figure parameters
- savedir (str): path of directory in which to save plots.
default: None
Returns:
- fulldir (str) : final path of the directory in which the figure is
saved (may differ from input savedir, if datetime
subfolder is added.)
- savename (str): name under which the figure is saved
"""
statstr_pr = sess_str_util.stat_par_str(
analyspar["stats"], analyspar["error"], "print")
dendstr_pr = sess_str_util.dend_par_str(
analyspar["dend"], sesspar["plane"], extrapar["datatype"], "print")
sessstr = f"sess{sesspar['sess_n']}_{sesspar['plane']}"
dendstr = sess_str_util.dend_par_str(
analyspar["dend"], sesspar["plane"], extrapar["datatype"])
datatype = extrapar["datatype"]
# extract some info from sess_info
keys = ["mouse_ns", "sess_ns", "lines", "planes"]
[mouse_ns, sess_ns, lines, planes] = [sess_info[key] for key in keys]
nroi_strs = sess_str_util.get_nroi_strs(sess_info, empty=(datatype!="roi"))
n_sess = len(mouse_ns)
if figpar is None:
figpar = sess_plot_util.init_figpar()
figpar = copy.deepcopy(figpar)
figpar["init"]["subplot_wid"] = 10
figpar["init"]["ncols"] = n_sess
if datatype == "roi":
figpar["save"]["fig_ext"] = "jpg"
figpar["init"]["sharex"] = False
figpar["init"]["sharey"] = False
# set subplot ratios and removes extra space between plots vertically
gs = {"height_ratios": [5, 1], "hspace": 0.1}
n_rows = 2
if roi_tr is None:
raise ValueError("Cannot plot data as ROI traces are missing "
"(not recorded in dictionary, likely due to size).")
else:
gs = None
n_rows = 1
if datatype == "roi" and not figpar["save"]["save_fig"]:
warnings.warn("Figure plotting is being skipped. Since full ROI traces "
"are not saved to dictionary, to actually plot traces, analysis "
"will have to be rerun with 'save_fig' set to True.",
stacklevel=1)
fig, ax = plot_util.init_fig(n_sess*n_rows, gs=gs, **figpar["init"])
label_height = 0.8
if datatype == "roi":
fig.subplots_adjust(top=0.92) # remove extra white space at top
label_height = 0.55
for i in range(n_sess):
title = (f"Mouse {mouse_ns[i]} (sess {sess_ns[i]}, {lines[i]} "
f"{planes[i]}{dendstr_pr}{nroi_strs[i]})")
sub_axs = ax[:, i]
sub_axs[0].set_title(title, y=1.02)
if datatype == "roi":
xran = range(len(trace_info["all_tr"][i][1]))
# each ROI (top subplot)
plot_util.plot_sep_data(sub_axs[0], np.asarray(roi_tr[i]), 0.1)
sess_plot_util.add_axislabels(
sub_axs[0], fluor=analyspar["fluor"], scale=True,
datatype=datatype, x_ax="")
# average across ROIs (bottom subplot)
av_tr = np.asarray(trace_info["all_tr"][i])
subtitle = u"{} across ROIs".format(statstr_pr)
plot_util.plot_traces(
sub_axs[1], xran, av_tr[0], av_tr[1:], lw=0.2, xticks="auto",
title=subtitle
)
else:
xran = range(len(trace_info["all_tr"][i]))
run_tr = np.asarray(trace_info["all_tr"][i])
sub_axs[0].plot(run_tr, lw=0.2)
for b, block in enumerate(trace_info["all_edges"][i]):
# all block labels to the lower plot
plot_util.add_labels(
sub_axs[-1], trace_info["all_pars"][i][b], np.mean(block),
label_height, color="k")
sess_plot_util.add_axislabels(
sub_axs[-1], fluor=analyspar["fluor"], datatype=datatype,
x_ax="")
plot_util.remove_ticks(sub_axs[-1], True, False)
plot_util.remove_graph_bars(sub_axs[-1], bars="horiz")
# add lines to both plots
for r in range(n_rows):
plot_util.add_bars(sub_axs[r], bars=block)
if savedir is None:
savedir = Path(
figpar["dirs"][datatype],
figpar["dirs"]["full"])
y = 1 if datatype == "run" else 0.98
fig.suptitle("Full traces across sessions", fontsize="xx-large", y=y)
savename = f"{datatype}_tr_{sessstr}{dendstr}"
fulldir = plot_util.savefig(
fig, savename, savedir, dpi=400, **figpar["save"])
return fulldir, savename
def plot_traces(sub_ax, time_values, trace_st, split="by_exp", col="k",
lab=True, ls=None, exp_col="gray", hline=True):
"""
plot_traces(sub_ax, time_values, trace_st)
Plot data trace splits (a single set).
Required args:
- sub_ax (plt Axis subplot):
subplot
- time_values (array-like):
values for each frame, in seconds (only from 0, if lock)
- trace_st (3D array):
trace statistics
dims: split x frame x stats
Optional args:
- split (str):
data split, e.g. "exp_lock", "unexp_lock", "stim_onset" or
"stim_offset"
default: False
- col (str):
colour for non-regular/non-locked data
default: "k"
- lab (bool):
if True, data label is included for legend
default: True
- ls (float):
trace line style
default: None
- exp_col (str):
color for expected data
default: "gray"
- hline (bool):
if True, horizontal line at y=0 is added
default: False
"""
lock = (split != "by_exp")
names = ["exp", "unexp"]
if lock and "stim" in split:
names = ["off", "on"]
if lock:
cols = [col, col]
else:
cols = [exp_col, col]
alphas_line = [0.8, 0.8]
alphas_shade = [0.5, 0.5]
# alpha exceptions for red
if col in ["red", plot_util.LINCLAB_COLS["red"]]:
updates = [0, 1] if lock else [1]
for update in updates:
alphas_line[update] = 0.75
alphas_shade[update] = 0.2
time_values = np.asarray(time_values)
# horizontal 0 line
if hline:
all_rows = True
if not sub_ax.is_last_row() or all_rows:
alpha = 0.5
else:
# to prevent infinite expansion bug
alpha = 0
sub_ax.axhline(
y=0, ls=plot_helper_fcts.HDASH, c="k", lw=3.0, alpha=alpha,
zorder=-13
)
# vertical 0 line
if lock or (time_values.min() <= 0 and time_values.max() >= 0):
alpha = 0.5
else:
alpha = 0 # same as before - prevents infinite expansion
sub_ax.axvline(x=0, ls=plot_helper_fcts.VDASH, c="k", lw=3.0, alpha=alpha,
zorder=-13)
# x ticks
xticks = "auto"
if lock:
xticks = np.linspace(-np.max(time_values), np.max(time_values), 5)
rev_time_values = time_values[::-1] * -1
trace_st = np.asarray(trace_st)
for i, (col, name) in enumerate(zip(cols, names)):
label = name if lab and not lock else None
if lock and f"_{name}" not in f"_{split}": # hack, since "exp" is in "unexp"
time_values_use = rev_time_values
else:
time_values_use = time_values
if split in ["exp_lock", "stim_offset"]:
i = 1 - i # data ordered as [non-reg, reg] instead of vv
plot_util.plot_traces(sub_ax, time_values_use, trace_st[i, :, 0],
trace_st[i, :, 1:], label=label, alpha_line=alphas_line[i],
alpha=alphas_shade[i], color=col, xticks=xticks, ls=ls)
def run_sess_lstm(sessid, args):
if args.parallel and args.plt_bkend is not None:
plt.switch_backend(args.plt_bkend) # needs to be repeated within joblib
args.seed = rand_util.seed_all(args.seed, args.device, seed_torch=True)
train_p = 0.8
lr = 1. * 10**(-args.lr_ex)
if args.conv:
conv_str = "_conv"
outch_str = f"_{args.out_ch}outch"
else:
conv_str = ""
outch_str = ""
# Input output parameters
n_stim_s = 0.6
n_roi_s = 0.3
# Stim/traces for training
train_gabfr = 0
train_post = 0.9 # up to C
roi_train_pre = 0 # from A
stim_train_pre = 0.3 # from preceeding grayscreen
# Stim/traces for testing (separated for unexp vs exp)
test_gabfr = 3
test_post = 0.6 # up to grayscreen
roi_test_pre = 0 # from D/U
stim_test_pre = 0.3 # from preceeding C
sess = sess_gen_util.init_sessions(
sessid, args.datadir, args.mouse_df, args.runtype, full_table=False,
fluor="dff", dend="extr", run=True, temp_log="warning")[0]
analysdir = sess_gen_util.get_analysdir(
sess.mouse_n, sess.sess_n, sess.plane, stimtype=args.stimtype,
comp=None)
dirname = Path(args.output, analysdir)
file_util.createdir(dirname, log_dir=False)
# Must not scale ROIs or running BEFOREHAND. Must do after to use only
# network available data.
# seq x frame x gabor x par
logger.info("Preparing stimulus parameter dataframe",
extra={"spacing": "\n"})
train_stim_wins, run_stats = sess_data_util.get_stim_data(
sess, args.stimtype, n_stim_s, train_gabfr, stim_train_pre,
train_post, gabk=16, run=True)
logger.info("Adding ROI data")
xran, train_roi_wins, roi_stats = sess_data_util.get_roi_data(
sess, args.stimtype, n_roi_s, train_gabfr, roi_train_pre, train_post,
gabk=16)
logger.warning("Preparing windowed datasets (too slow - to be improved)")
raise NotImplementedError("Not implemented properly - some error leads "
"to excessive memory requests.")
test_stim_wins = []
test_roi_wins = []
for unexp in [0, 1]:
stim_wins = sess_data_util.get_stim_data(
sess, args.stimtype, n_stim_s, test_gabfr, stim_test_pre,
test_post, unexp, gabk=16, run_mean=run_stats[0],
run_std=run_stats[1])
test_stim_wins.append(stim_wins)
roi_wins = sess_data_util.get_roi_data(sess, args.stimtype, n_roi_s,
test_gabfr, roi_test_pre, test_post, unexp, gabk=16,
roi_means=roi_stats[0], roi_stds=roi_stats[1])[1]
test_roi_wins.append(roi_wins)
n_pars = train_stim_wins.shape[-1] # n parameters (121)
n_rois = train_roi_wins.shape[-1] # n ROIs
hyperstr = (f"{args.hidden_dim}hd_{args.num_layers}hl_{args.lr_ex}lrex_"
f"{args.batchsize}bs{outch_str}{conv_str}")
dls = data_util.create_dls(train_stim_wins, train_roi_wins, train_p=train_p,
test_p=0, batchsize=args.batchsize, thresh_cl=0,
train_shuff=True)[0]
train_dl, val_dl, _ = dls
test_dls = []
for s in [0, 1]:
dl = data_util.init_dl(test_stim_wins[s], test_roi_wins[s],
batchsize=args.batchsize)
test_dls.append(dl)
logger.info("Running LSTM")
if args.conv:
lstm = ConvPredROILSTM(args.hidden_dim, n_rois, out_ch=args.out_ch,
num_layers=args.num_layers, dropout=args.dropout)
else:
lstm = PredLSTM(n_pars, args.hidden_dim, n_rois,
num_layers=args.num_layers, dropout=args.dropout)
lstm = lstm.to(args.device)
lstm.loss_fn = torch.nn.MSELoss(size_average=False)
lstm.opt = torch.optim.Adam(lstm.parameters(), lr=lr)
loss_df = pd.DataFrame(
np.nan, index=range(args.n_epochs), columns=["train", "val"])
min_val = np.inf
for ep in range(args.n_epochs):
logger.info(f"====> Epoch {ep}", extra={"spacing": "\n"})
if ep == 0:
train_loss = run_dl(lstm, train_dl, args.device, train=False)
else:
train_loss = run_dl(lstm, train_dl, args.device, train=True)
val_loss = run_dl(lstm, val_dl, args.device, train=False)
loss_df["train"].loc[ep] = train_loss/train_dl.dataset.n_samples
loss_df["val"].loc[ep] = val_loss/val_dl.dataset.n_samples
logger.info(f"Training loss : {loss_df['train'].loc[ep]}")
logger.info(f"Validation loss: {loss_df['val'].loc[ep]}")
# record model if training is lower than val, and val reaches a new low
if ep == 0 or val_loss < min_val:
prev_model = glob.glob(str(Path(dirname, f"{hyperstr}_ep*.pth")))
prev_df = glob.glob(str(Path(dirname, f"{hyperstr}.csv")))
min_val = val_loss
saved_ep = ep
if len(prev_model) == 1 and len(prev_df) == 1:
Path(prev_model[0]).unlink()
Path(prev_df[0]).unlink()
savename = f"{hyperstr}_ep{ep}"
savefile = Path(dirname, savename)
torch.save({"net": lstm.state_dict(), "opt": lstm.opt.state_dict()},
f"{savefile}.pth")
file_util.saveinfo(loss_df, hyperstr, dirname, "csv")
plot_util.linclab_plt_defaults(font=["Arial", "Liberation Sans"],
fontdir=DEFAULT_FONTDIR)
fig, ax = plt.subplots(1)
for dataset in ["train", "val"]:
plot_util.plot_traces(ax, range(args.n_epochs), np.asarray(loss_df[dataset]),
label=dataset, title=f"Average loss (MSE) ({n_rois} ROIs)",
xticks="auto")
fig.savefig(Path(dirname, f"{hyperstr}_loss"))
savemod = Path(dirname, f"{hyperstr}_ep{saved_ep}.pth")
checkpoint = torch.load(savemod)
lstm.load_state_dict(checkpoint["net"])
n_samples = 20
val_idx = np.random.choice(range(val_dl.dataset.n_samples), n_samples)
val_samples = val_dl.dataset[val_idx]
xrans = data_util.get_win_xrans(xran, val_samples[1].shape[1], val_idx.tolist())
fig, ax = plot_util.init_fig(n_samples, ncols=4, sharex=True, subplot_hei=2,
subplot_wid=5)
lstm.eval()
with torch.no_grad():
batch_len, seq_len, n_items = val_samples[1].shape
pred_tr = lstm(val_samples[0].transpose(1, 0).to(args.device))
pred_tr = pred_tr.view([seq_len, batch_len, n_items]).transpose(1, 0)
for lab, data in zip(["target", "pred"], [val_samples[1], pred_tr]):
data = data.numpy()
for n in range(n_samples):
roi_n = np.random.choice(range(data.shape[-1]))
sub_ax = plot_util.get_subax(ax, n)
plot_util.plot_traces(sub_ax, xrans[n], data[n, :, roi_n],
label=lab, xticks="auto")
plot_util.set_ticks(sub_ax, "x", xran[0], xran[-1], n=7)
sess_plot_util.plot_labels(ax, train_gabfr, plot_vals="exp", pre=roi_train_pre,
post=train_post)
fig.suptitle(f"Target vs predicted validation traces ({n_rois} ROIs)")
fig.savefig(Path(dirname, f"{hyperstr}_traces"))