def gabor_Uori_decoding_sess123(sessions,
analyspar,
sesspar,
stimpar,
logregpar,
permpar,
figpar,
seed=None,
parallel=False):
"""
gabor_Uori_decoding_sess123(sessions, analyspar, sesspar, stimpar,
logregpar, permpar, figpar)
Runs decoding analyses (U orientations).
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list):
Session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
- sesspar (SessPar):
named tuple containing session parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- logregpar (LogRegPar):
named tuple containing logistic regression parameters
- permpar (PermPar):
named tuple containing permutation parameters
- figpar (dict):
dictionary containing figure parameters
Optional args:
- seed (int):
seed value to use. (-1 treated as None)
default: None
- parallel (bool):
if True, some of the analysis is run in parallel across CPU cores
default: False
"""
if logregpar.comp != "Uori":
raise ValueError("logregpar.comp should be Uori.")
# ctrl doesn't apply to U orientation decoding
logregpar = sess_ntuple_util.get_modif_ntuple(logregpar, "ctrl", False)
gab_ori = sess_gen_util.filter_gab_oris("U", stimpar.gab_ori)
stimpar = sess_ntuple_util.get_modif_ntuple(stimpar, "gab_ori", gab_ori)
gabor_decoding_sess123(sessions,
analyspar=analyspar,
sesspar=sesspar,
stimpar=stimpar,
logregpar=logregpar,
permpar=permpar,
figpar=figpar,
seed=seed,
parallel=parallel)
def set_multcomp(permpar, sess_df, CIs=True, pairs=True, factor=1):
"""
set_multcomp(permpar)
Returns permpar updated with the number of comparisons computed from the
sessions.
Required args:
- permpar (PermPar):
named tuple containing permutation parameters
- sess_df (pd.DataFrame):
dataframe containing session information, including the following
keys: "sess_ns", "lines", "planes"
Optional args:
- CIs (bool):
include comparisons to CIs comparisons
default: True
- pairs (bool):
include paired comparisons
default: True
- factor (int):
additional factor by which to multiply the number of comparisons
default: 1
Returns:
- permpar (PermPar):
updated permutation parameter named tuple
"""
if isinstance(permpar, dict):
permpar = sess_ntuple_util.init_permpar(**permpar)
n_comps = 0
for _, sess_df_grp in sess_df.groupby(["lines", "planes"]):
n_sess = len(sess_df_grp)
# sessions compared to CIs
if CIs:
n_comps += n_sess
# session pair comparisons
if pairs:
k = 2
if n_sess >= k:
fact = np.math.factorial
n_comps += fact(n_sess) / (fact(k) * fact(n_sess - k))
# multiplied by specified factor
n_comps *= factor
permpar = sess_ntuple_util.get_modif_ntuple(
permpar, "multcomp", int(n_comps)
)
return permpar
def set_multcomp(permpar, sessions, analyspar, consec_only=True, factor=1):
"""
set_multcomp(permpar, sessions, analyspar)
Returns permpar updated with the number of comparisons computed from the
sessions.
Required args:
- permpar (PermPar or dict):
named tuple containing permutation parameters
- sessions (list):
Session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
Optional args:
- consec_only (bool):
if True, only consecutive session numbers are correlated
default: True
- factor (int):
multiplicative factor
default: 1
Returns:
- permpar (PermPar):
updated permutation parameter named tuple
"""
sess_df = misc_analys.get_check_sess_df(sessions, analyspar=analyspar)
n_comps = 0
for _, lp_df in sess_df.groupby(["lines", "planes"]):
corr_ns = get_corr_pairs(lp_df, consec_only=consec_only)
n_comps += len(corr_ns)
n_comps = n_comps * factor
permpar = sess_ntuple_util.get_modif_ntuple(permpar, "multcomp", n_comps)
return permpar
def run_mag_change(sessions,
analysis,
seed,
analyspar,
sesspar,
stimpar,
permpar,
quantpar,
figpar,
datatype="roi"):
"""
run_mag_change(sessions, analysis, seed, analyspar, sesspar, stimpar,
permpar, quantpar, figpar)
Calculates and plots the magnitude of change in activity of ROIs between
the first and last quantile for expected vs unexpected sequences.
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list) : list of Session objects
- analysis (str) : analysis type (e.g., "m")
- seed (int) : seed value to use. (-1 treated as None)
- analyspar (AnalysPar): named tuple containing analysis parameters
- sesspar (SessPar) : named tuple containing session parameters
- stimpar (StimPar) : named tuple containing stimulus parameters
- permpar (PermPar) : named tuple containing permutation parameters
- quantpar (QuantPar) : named tuple containing quantile analysis
parameters
- figpar (dict) : dictionary containing figure parameters
Optional args:
- datatype (str): type of data (e.g., "roi", "run")
"""
sessstr_pr = sess_str_util.sess_par_str(sesspar.sess_n, stimpar.stimtype,
sesspar.plane, stimpar.visflow_dir,
stimpar.visflow_size, stimpar.gabk,
"print")
dendstr_pr = sess_str_util.dend_par_str(analyspar.dend, sesspar.plane,
datatype, "print")
datastr = sess_str_util.datatype_par_str(datatype)
logger.info(
f"Calculating and plotting the magnitude changes in {datastr} "
f"activity across quantiles \n({sessstr_pr}{dendstr_pr}).",
extra={"spacing": "\n"})
if permpar.multcomp:
permpar = sess_ntuple_util.get_modif_ntuple(permpar, "multcomp",
len(sessions))
# get full data: session x unexp x quants of interest x [ROI x seq]
integ_info = quant_analys.trace_stats_by_qu_sess(sessions,
analyspar,
stimpar,
quantpar.n_quants,
quantpar.qu_idx,
by_exp=True,
integ=True,
ret_arr=True,
datatype=datatype)
all_counts = integ_info[-2]
qu_data = integ_info[-1]
# extract session info
mouse_ns = [sess.mouse_n for sess in sessions]
lines = [sess.line for sess in sessions]
if analyspar.rem_bad:
nanpol = None
else:
nanpol = "omit"
seed = rand_util.seed_all(seed, "cpu", log_seed=False)
mags = quant_analys.qu_mags(qu_data,
permpar,
mouse_ns,
lines,
analyspar.stats,
analyspar.error,
nanpol=nanpol,
op_qu="diff",
op_unexp="diff")
# convert mags items to list
mags = copy.deepcopy(mags)
mags["all_counts"] = all_counts
for key in ["mag_st", "L2", "mag_rel_th", "L2_rel_th"]:
mags[key] = mags[key].tolist()
sess_info = sess_gen_util.get_sess_info(sessions,
analyspar.fluor,
incl_roi=(datatype == "roi"),
rem_bad=analyspar.rem_bad)
extrapar = {"analysis": analysis, "datatype": datatype, "seed": seed}
info = {
"analyspar": analyspar._asdict(),
"sesspar": sesspar._asdict(),
"stimpar": stimpar._asdict(),
"extrapar": extrapar,
"permpar": permpar._asdict(),
"quantpar": quantpar._asdict(),
"mags": mags,
"sess_info": sess_info
}
fulldir, savename = gen_plots.plot_mag_change(figpar=figpar, **info)
file_util.saveinfo(info, savename, fulldir, "json")
def run_traces_by_qu_lock_sess(sessions,
analysis,
seed,
analyspar,
sesspar,
stimpar,
quantpar,
figpar,
datatype="roi"):
"""
run_traces_by_qu_lock_sess(sessions, analysis, analyspar, sesspar,
stimpar, quantpar, figpar)
Retrieves trace statistics by session x quantile at the transition of
expected to unexpected sequences (or v.v.) and plots traces across ROIs by
quantile with each session in a separate subplot.
Also runs analysis for one quantile (full data) with different unexpected
lengths grouped separated
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list) : list of Session objects
- analysis (str) : analysis type (e.g., "l")
- seed (int) : seed value to use. (-1 treated as None)
- analyspar (AnalysPar): named tuple containing analysis parameters
- sesspar (SessPar) : named tuple containing session parameters
- stimpar (StimPar) : named tuple containing stimulus parameters
- quantpar (QuantPar) : named tuple containing quantile analysis
parameters
- figpar (dict) : dictionary containing figure parameters
Optional args:
- datatype (str): type of data (e.g., "roi", "run")
"""
sessstr_pr = sess_str_util.sess_par_str(sesspar.sess_n, stimpar.stimtype,
sesspar.plane, stimpar.visflow_dir,
stimpar.visflow_size, stimpar.gabk,
"print")
dendstr_pr = sess_str_util.dend_par_str(analyspar.dend, sesspar.plane,
datatype, "print")
datastr = sess_str_util.datatype_par_str(datatype)
logger.info(
f"Analysing and plotting unexpected vs expected {datastr} "
f"traces locked to unexpected onset by quantile ({quantpar.n_quants}) "
f"\n({sessstr_pr}{dendstr_pr}).",
extra={"spacing": "\n"})
seed = rand_util.seed_all(seed, "cpu", log_seed=False)
# modify quantpar to retain all quantiles
quantpar_one = sess_ntuple_util.init_quantpar(1, 0)
n_quants = quantpar.n_quants
quantpar_mult = sess_ntuple_util.init_quantpar(n_quants, "all")
if stimpar.stimtype == "visflow":
pre_post = [2.0, 6.0]
elif stimpar.stimtype == "gabors":
pre_post = [2.0, 8.0]
else:
gen_util.accepted_values_error("stimpar.stimtype", stimpar.stimtype,
["visflow", "gabors"])
logger.warning("Setting pre to {}s and post to {}s.".format(*pre_post))
stimpar = sess_ntuple_util.get_modif_ntuple(stimpar, ["pre", "post"],
pre_post)
figpar = copy.deepcopy(figpar)
if figpar["save"]["use_dt"] is None:
figpar["save"]["use_dt"] = gen_util.create_time_str()
for baseline in [None, stimpar.pre]:
basestr_pr = sess_str_util.base_par_str(baseline, "print")
for quantpar in [quantpar_one, quantpar_mult]:
locks = ["unexp", "exp"]
if quantpar.n_quants == 1:
locks.append("unexp_split")
# get the stats (all) separating by session and quantiles
for lock in locks:
logger.info(
f"{quantpar.n_quants} quant, {lock} lock{basestr_pr}",
extra={"spacing": "\n"})
if lock == "unexp_split":
trace_info = quant_analys.trace_stats_by_exp_len_sess(
sessions,
analyspar,
stimpar,
quantpar.n_quants,
quantpar.qu_idx,
byroi=False,
nan_empty=True,
baseline=baseline,
datatype=datatype)
else:
trace_info = quant_analys.trace_stats_by_qu_sess(
sessions,
analyspar,
stimpar,
quantpar.n_quants,
quantpar.qu_idx,
byroi=False,
lock=lock,
nan_empty=True,
baseline=baseline,
datatype=datatype)
# for comparison, locking to middle of expected sample (1 quant)
exp_samp = quant_analys.trace_stats_by_qu_sess(
sessions,
analyspar,
stimpar,
quantpar_one.n_quants,
quantpar_one.qu_idx,
byroi=False,
lock="exp_samp",
nan_empty=True,
baseline=baseline,
datatype=datatype)
extrapar = {
"analysis": analysis,
"datatype": datatype,
"seed": seed,
}
xrans = [xran.tolist() for xran in trace_info[0]]
all_stats = [sessst.tolist() for sessst in trace_info[1]]
exp_stats = [expst.tolist() for expst in exp_samp[1]]
trace_stats = {
"xrans": xrans,
"all_stats": all_stats,
"all_counts": trace_info[2],
"lock": lock,
"baseline": baseline,
"exp_stats": exp_stats,
"exp_counts": exp_samp[2]
}
if lock == "unexp_split":
trace_stats["unexp_lens"] = trace_info[3]
sess_info = sess_gen_util.get_sess_info(
sessions,
analyspar.fluor,
incl_roi=(datatype == "roi"),
rem_bad=analyspar.rem_bad)
info = {
"analyspar": analyspar._asdict(),
"sesspar": sesspar._asdict(),
"stimpar": stimpar._asdict(),
"quantpar": quantpar._asdict(),
"extrapar": extrapar,
"sess_info": sess_info,
"trace_stats": trace_stats
}
fulldir, savename = gen_plots.plot_traces_by_qu_lock_sess(
figpar=figpar, **info)
file_util.saveinfo(info, savename, fulldir, "json")
def get_sess_ex_traces(sess, analyspar, stimpar, basepar, rolling_win=4):
"""
get_sess_ex_traces(sess, analyspar, stimpar, basepar)
Returns example traces selected for the session, based on SNR and Gabor
response pattern criteria.
Criteria:
- Above median SNR
- Sequence response correlation above 75th percentile.
- Mean sequence standard deviation above 75th percentile.
- Mean sequence skew above 75th percentile.
Required args:
- sess (Session):
Session object
- analyspar (AnalysPar):
named tuple containing analysis parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- basepar (BasePar):
named tuple containing baseline parameters
Optional args:
- rolling_win (int):
window to use in rolling mean over individual trial traces before
computing correlation between trials (None for no smoothing)
default: 4
Returns:
- selected_roi_data (dict):
["time_values"] (1D array): values for each frame, in seconds
(only 0 to stimpar.post, unless split is "by_exp")
["roi_ns"] (1D array): selected ROI numbers
["roi_traces_sm"] (3D array): selected ROI sequence traces,
smoothed, with dims: ROIs x seq x frames
["roi_trace_stats"] (2D array): selected ROI trace mean or median,
dims: ROIs x frames
"""
nanpol = None if analyspar.rem_bad else "omit"
if stimpar.stimtype != "gabors":
raise NotImplementedError(
"ROI selection criteria designed for Gabors, and based on their "
"cyclical responses.")
snr_analyspar = sess_ntuple_util.get_modif_ntuple(analyspar, "scale",
False)
snrs = misc_analys.get_snr(sess, snr_analyspar, "snrs")
snr_median = np.median(snrs)
# identify ROIs that meet the SNR threshold
snr_thr_rois = np.where(snrs > snr_median)[0]
# collect all data, and compute statistics
traces, time_values = basic_analys.get_split_data_by_sess(
sess,
analyspar=analyspar,
stimpar=stimpar,
split="by_exp",
baseline=basepar.baseline,
)
traces_exp = np.asarray(traces[0]) # get expected split
traces_exp_stat = math_util.mean_med(traces_exp,
stats=analyspar.stats,
axis=1,
nanpol=nanpol)
# smooth individual traces, then compute correlations
if rolling_win is not None:
traces_exp = math_util.rolling_mean(traces_exp, win=rolling_win)
triu_idx = np.triu_indices(traces_exp[snr_thr_rois].shape[1], k=1)
corr_medians = [
np.median(np.corrcoef(roi_trace)[triu_idx])
for roi_trace in traces_exp[snr_thr_rois]
]
# calculate std and skew over trace statistics
trace_stat_stds = np.std(traces_exp_stat[snr_thr_rois], axis=1)
trace_stat_skews = scist.skew(traces_exp_stat[snr_thr_rois], axis=1)
# identify ROIs that meet thresholds (from those with high enough SNR)
std_thr = np.percentile(trace_stat_stds, 75)
skew_thr = np.percentile(trace_stat_skews, 75)
corr_thr = np.percentile(corr_medians, 75)
selected_idx = np.where(
((trace_stat_stds > std_thr) * (corr_medians > corr_thr) *
(trace_stat_skews > skew_thr)))[0]
# re-index into all ROIs
roi_ns = snr_thr_rois[selected_idx]
selected_roi_data = {
"time_values": time_values,
"roi_ns": roi_ns,
"roi_traces_sm": traces_exp[roi_ns],
"roi_trace_stats": traces_exp_stat[roi_ns]
}
return selected_roi_data
def unexp_resp_stimulus_comp_sess1v3(sessions,
analyspar,
sesspar,
stimpar,
permpar,
figpar,
seed=None,
parallel=False):
"""
unexp_resp_stimulus_comp_sess1v3(sessions, analyspar, sesspar, stimpar,
permpar, figpar)
Retrieves changes in tracked ROI responses to unexpected sequences for
Gabors vs visual flow stimuli.
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list):
Session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
- sesspar (SessPar):
named tuple containing session parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- permpar (PermPar):
named tuple containing permutation parameters
- figpar (dict):
dictionary containing figure parameters
Optional args:
- seed (int):
seed value to use. (-1 treated as None)
default: None
- parallel (bool):
if True, some of the analysis is run in parallel across CPU cores
default: False
"""
logger.info(
("Compiling changes in unexpected responses to Gabor vs visual "
"flow stimuli."),
extra={"spacing": "\n"})
if analyspar.scale:
raise ValueError("analyspar.scale should be set to False.")
# calculate multiple comparisons
dummy_df = misc_analys.get_check_sess_df(
sessions, None, analyspar).drop_duplicates(subset=["lines", "planes"])
multcomp = len(dummy_df) + 1
permpar = sess_ntuple_util.get_modif_ntuple(permpar, "multcomp", multcomp)
comp_sess = [1, 3]
datatype = "rel_unexp_resp"
rel_sess = 1
pop_stats = True
unexp_comp_df = stim_analys.get_stim_stats_df(
sessions,
analyspar=analyspar,
stimpar=stimpar,
permpar=permpar,
comp_sess=comp_sess,
datatype=datatype,
rel_sess=rel_sess,
pop_stats=pop_stats,
randst=seed,
parallel=parallel,
)
extrapar = {
"comp_sess": comp_sess,
"datatype": datatype,
"rel_sess": rel_sess,
"pop_stats": pop_stats,
"seed": seed,
}
info = {
"analyspar": analyspar._asdict(),
"sesspar": sesspar._asdict(),
"stimpar": stimpar._asdict(),
"permpar": permpar._asdict(),
"extrapar": extrapar,
"unexp_comp_df": unexp_comp_df.to_dict(),
}
helper_fcts.plot_save_all(info, figpar)
def tracked_roi_usis_stimulus_comp_sess1v3(sessions,
analyspar,
sesspar,
stimpar,
basepar,
idxpar,
permpar,
figpar,
seed=None,
parallel=False):
"""
tracked_roi_usis_stimulus_comp_sess1v3(sessions, analyspar, sesspar,
stimpar, basepar, idxpar, permpar,
figpar)
Retrieves changes in tracked ROI USIs for Gabors vs visual flow stimuli.
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list):
Session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
- sesspar (SessPar):
named tuple containing session parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- basepar (BasePar):
named tuple containing baseline parameters
- idxpar (IdxPar):
named tuple containing index parameters
- permpar (PermPar):
named tuple containing permutation parameters
- figpar (dict):
dictionary containing figure parameters
Optional args:
- seed (int):
seed value to use. (-1 treated as None)
default: None
- parallel (bool):
if True, some of the analysis is run in parallel across CPU cores
default: False
"""
logger.info(
("Compiling changes in ROI USIs for Gabors vs visual flow stimuli."),
extra={"spacing": "\n"})
if not analyspar.tracked:
raise ValueError("analyspar.tracked should be set to True.")
# remove incomplete session series and warn
sessions = misc_analys.check_sessions_complete(sessions)
# calculate multiple comparisons
dummy_df = misc_analys.get_check_sess_df(
sessions, None, analyspar).drop_duplicates(subset=["lines", "planes"])
multcomp = len(dummy_df) + 1
permpar = sess_ntuple_util.get_modif_ntuple(permpar, "multcomp", multcomp)
comp_sess = [1, 3]
datatype = "usis"
pop_stats = True
usi_comp_df = stim_analys.get_stim_stats_df(
sessions,
analyspar=analyspar,
stimpar=stimpar,
basepar=basepar,
idxpar=idxpar,
permpar=permpar,
comp_sess=comp_sess,
datatype=datatype,
pop_stats=pop_stats,
randst=seed,
parallel=parallel,
)
extrapar = {
"comp_sess": comp_sess,
"datatype": datatype,
"pop_stats": pop_stats,
"seed": seed,
}
info = {
"analyspar": analyspar._asdict(),
"sesspar": sesspar._asdict(),
"stimpar": stimpar._asdict(),
"basepar": basepar._asdict(),
"idxpar": idxpar._asdict(),
"permpar": permpar._asdict(),
"extrapar": extrapar,
"usi_comp_df": usi_comp_df.to_dict(),
}
helper_fcts.plot_save_all(info, figpar)
def gabor_corrs_sess123_comps(sessions,
analyspar,
sesspar,
stimpar,
basepar,
idxpar,
permpar,
figpar,
seed=None,
parallel=False):
"""
gabor_corrs_sess123_comps(sessions, analyspar, sesspar, stimpar, basepar,
idxpar, permpar, figpar)
Retrieves tracked ROI Gabor USI correlations for session 1 to 3.
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list):
Session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
- sesspar (SessPar):
named tuple containing session parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- basepar (BasePar):
named tuple containing baseline parameters
- idxpar (IdxPar):
named tuple containing index parameters
- permpar (PermPar):
named tuple containing permutation parameters
- figpar (dict):
dictionary containing figure parameters
Optional args:
- seed (int):
seed value to use. (-1 treated as None)
default: None
- parallel (bool):
if True, some of the analysis is run in parallel across CPU cores
default: False
"""
logger.info(
"Compiling tracked ROI Gabor USI correlations for sessions 1 to 3.",
extra={"spacing": "\n"})
if not analyspar.tracked:
raise ValueError("analyspar.tracked should be set to True.")
# remove incomplete session series and warn
sessions = misc_analys.check_sessions_complete(sessions)
consec_only = True
permpar = corr_analys.set_multcomp(permpar,
sessions,
analyspar,
consec_only=consec_only)
permute = PERMUTE
corr_type = CORR_TYPE
sig_only = SIG_ONLY
if "R_sqr" in corr_type:
permpar = sess_ntuple_util.get_modif_ntuple(permpar, "tails", "hi")
idx_corr_df = corr_analys.get_idx_corrs_df(sessions,
analyspar=analyspar,
stimpar=stimpar,
basepar=basepar,
idxpar=idxpar,
permpar=permpar,
consec_only=consec_only,
permute=permute,
corr_type=corr_type,
sig_only=sig_only,
randst=seed,
parallel=parallel)
extrapar = {
"consec_only": consec_only,
"corr_type": corr_type,
"permute": permute,
"seed": seed,
"sig_only": sig_only,
}
info = {
"analyspar": analyspar._asdict(),
"sesspar": sesspar._asdict(),
"stimpar": stimpar._asdict(),
"basepar": basepar._asdict(),
"idxpar": idxpar._asdict(),
"permpar": permpar._asdict(),
"extrapar": extrapar,
"idx_corr_df": idx_corr_df.to_dict()
}
helper_fcts.plot_save_all(info, figpar)
def run_mag_permute(all_data_perm, act_mag_me_rel, act_L2_rel, n_exps, permpar,
op_qu="diff", op_grp="diff", stats="mean", nanpol=None):
"""
run_mag_permute(all_data_perm, act_mag_rel, act_L2_rel, n_exp, permpar)
Returns the results of a permutation analysis of difference or ratio
between 2 quantiles of the magnitude change or L2 norm between expected and
unexpected activity.
Required args:
- all_data_perm (2D array): Data from both groups for permutation,
structured as:
ROI x seqs
- act_mag_rel (num) : Real mean/median magnitude difference
between quantiles
- act_L2_rel (num) : Real L2 difference between quantiles
- n_exps (list) : List of number of expected sequences in
each quantile
- permpar (PermPar) : named tuple containing permutation
parameters
Optional args:
- op_qu (str) : Operation to use in comparing the last vs first
quantile ("diff" or "ratio")
default: "diff"
- op_grp (str): Operation to use in comparing groups
(e.g., unexpected vs expected data) ("diff" or "ratio")
default: "diff"
- stats (str) : Statistic to take across group sequences, and then
across magnitude differences ("mean" or "median")
default: "mean"
- nanpol (str): Policy for NaNs, "omit" or None when taking statistics
default: None
Returns:
- signif (list) : list of significance results ("hi", "lo" or "no") for
magnitude, L2
- threshs (list): list of thresholds (1 if 1-tailed analysis,
2 if 2-tailed) for magnitude, L2
"""
if permpar.multcomp:
permpar = sess_ntuple_util.get_modif_ntuple(
permpar, ["multcomp", "p_val"],
[False, permpar.p_val / permpar.multcomp]
)
if len(all_data_perm) != 2 or len(n_exps) !=2:
raise ValueError("all_data_perm and n_exps must have length of 2.")
all_rand_vals = [] # qu x grp x ROI x perms
# for each quantile
for q, perm_data in enumerate(all_data_perm):
qu_vals = rand_util.permute_diff_ratio(
perm_data, n_exps[q], permpar.n_perms, stats, nanpol=nanpol,
op="none")
all_rand_vals.append(qu_vals)
all_rand_vals = np.asarray(all_rand_vals)
# get absolute change stats and retain mean/median only
rand_mag_me = math_util.calc_mag_change(
all_rand_vals, 0, 2, order="stats", op=op_qu, stats=stats)[0]
rand_L2 = math_util.calc_mag_change(all_rand_vals, 0, 2, order=2, op=op_qu)
# take diff/ratio between grps
rand_mag_rel = math_util.calc_op(rand_mag_me, op_grp, dim=0)
rand_L2_rel = math_util.calc_op(rand_L2, op_grp, dim=0)
# check significance (returns list although only one result tested)
mag_sign, mag_th = rand_util.id_elem(
rand_mag_rel, act_mag_me_rel, permpar.tails, permpar.p_val, ret_th=True)
L2_sign, L2_th = rand_util.id_elem(
rand_L2_rel, act_L2_rel, permpar.tails, permpar.p_val, ret_th=True)
mag_signif, L2_signif = ["no", "no"]
if str(permpar.tails) == "2":
if len(mag_sign[0]) == 1:
mag_signif = "lo"
elif len(mag_sign[1]) == 1:
mag_signif = "hi"
if len(L2_sign[0]) == 1:
L2_signif = "lo"
elif len(L2_sign[1]) == 1:
L2_signif = "hi"
elif permpar.tails in ["lo", "hi"]:
if len(mag_sign) == 1:
mag_signif = permpar.tails
if len(L2_sign) == 1:
L2_signif = permpar.tails
signif = [mag_signif, L2_signif]
threshs = [mag_th[0], L2_th[0]]
return signif, threshs
def trace_stats_by_exp_len_sess(sessions, analyspar, stimpar, n_quants=4,
qu_idx="all", byroi=True, integ=False,
ret_arr=False, nan_empty=False,
baseline=None, datatype="roi"):
"""
trace_stats_by_exp_len_sess(sessions, analyspar, stimpar)
Returns trace statistics for the quantiles of interest for each
session and unexpected length value, for the datatype of interest.
Required args:
- sessions (list) : list of Session objects
- analyspar (AnalysPar): named tuple containing analysis parameters
- stimpar (StimPar) : named tuple containing stimulus parameters
Optional args:
- n_quants (int) : number of quantiles to divide sessions into
default: 4
- qu_idx (str or list): indices of quantiles to retain
default: "all"
- byroi (bool) : If datatype is "roi", if True, returns
statistics for each ROI. If False, returns
statistics across ROIs.
default: True
- integ (bool) : if True, dF/F is integrated over sequences
default: False
- ret_arr (bool) : if True, data arrays are returned also
default: False
- nan_empty (bool) : if a quantile is empty, return NaN arrays
(avoids an error)
default: False
- baseline (num) : number of seconds to use as baseline. If None,
data is not baselined.
default: None
- datatype (str) : datatype, i.e. ROIs or running
default: "roi"
Returns:
- xrans (list) : time values for the 2p frames (None if
integ), for each session
- all_stats (list) : list of 2 to 5D arrays of trace data
statistics for each session, structured as:
unexp_len x
quantiles x
stats (me, err) x
(ROIs if byroi x)
(frames if not integ)
- all_counts (nested list) : list of number of sequences,
structured as:
sess x unexp_len x quantiles
- all_n_consec (list) : unique values of number of consecutive
segments, by session
if ret_arr:
- all_arrays (nested lists): list of data trace arrays, structured as:
session x unexp_len x quantile
of 1 to 3D arrays:
(ROI x) sequences
(x frames if not integ)
"""
shift_gab_segs = False
if stimpar.stimtype == "gabors" and stimpar.gabfr not in ["any", "all"]:
shift_gab_segs = True
orig_gabfr = stimpar.gabfr
stimpar = sess_ntuple_util.get_modif_ntuple(stimpar, "gabfr", "any")
all_counts, all_stats, all_arrays, all_n_consec = [], [], [], []
xrans = []
for sess in sessions:
stim = sess.get_stim(stimpar.stimtype)
sess_counts, sess_stats, sess_arrays = [], [], []
qu_segs, _, qu_n_consec = quant_segs(
stim, stimpar, n_quants, qu_idx, 1, empty_ok=nan_empty,
by_exp_len=True)
if shift_gab_segs: # shift to requested gabor frame
qu_segs = [[s + orig_gabfr for s in segs] for segs in qu_segs]
n_consec_flat = [n for sub_ns in qu_n_consec for n in sub_ns]
all_n_consec.append(sorted(set(n_consec_flat)))
for n_consec in all_n_consec[-1]:
sub_segs, sub_counts = [], []
# retain segments with correct number of consecutive values
for segs, ns in zip(qu_segs, qu_n_consec):
idx = np.where(np.asarray(ns) == n_consec)[0]
sub_segs.append([segs[i] for i in idx])
sub_counts.append(len(idx))
sess_counts.append(sub_counts)
trace_info = trace_stats_by_qu(stim, sub_segs, stimpar.pre,
stimpar.post, analyspar, byroi=byroi, integ=integ,
ret_arr=ret_arr, nan_empty=nan_empty,
baseline=baseline, datatype=datatype)
sess_stats.append(trace_info[1])
if ret_arr:
sess_arrays.append(trace_info[2])
xrans.append(trace_info[0])
all_counts.append(sess_counts)
all_stats.append(np.asarray(sess_stats))
if ret_arr:
all_arrays.append(sess_arrays)
if ret_arr:
return xrans, all_stats, all_counts, all_n_consec, all_arrays
else:
return xrans, all_stats, all_counts, all_n_consec
def trace_stats_by_qu_sess(sessions, analyspar, stimpar, n_quants=4,
qu_idx="all", byroi=True, by_exp=False, integ=False,
ret_arr=False, nan_empty=False, lock="no",
baseline=None, datatype="roi", randst=None):
"""
trace_stats_by_qu_sess(sessions, analyspar, stimpar)
Returns trace statistics for the quantiles of interest for each
session and unexpected value, for the datatype of interest.
Required args:
- sessions (list) : list of Session objects
- analyspar (AnalysPar): named tuple containing analysis parameters
- stimpar (StimPar) : named tuple containing stimulus parameters
Optional args:
- n_quants (int) : number of quantiles to divide sessions into
default: 4
- qu_idx (str or list): indices of quantiles to retain
default: "all"
- byroi (bool) : If datatype is "roi", if True, returns
statistics for each ROI. If False, returns
statistics across ROIs.
default: True
- by_exp (bool) : if True, quantiles are separated into unexpected
and expected groups.
default: False
- integ (bool) : if True, dF/F is integrated over sequences
default: False
- ret_arr (bool) : if True, data arrays are returned also
default: False
- nan_empty (bool) : if a quantile is empty, return NaN arrays
(avoids an error)
default: False
- lock (bool) : if "unexp", "exp", "exp_samp", only the first
unexpected or expected segments are retained.
If "both"
(by_exp is ignore).
default: False
- baseline (num) : number of seconds to use as baseline. If None,
data is not baselined.
default: None
- datatype (str) : datatype, i.e. ROIs or running
default: "roi"
- randst (int) : random state or seed for sampling segments
default: None
Returns:
- xrans (list) : time values for the 2p frames (None if
integ), for each session
- all_stats (list) : list of 2 to 5D arrays of trace data
statistics for each session, structured as:
(unexp if by_exp x)
quantiles x
stats (me, err) x
(ROIs if byroi x)
(frames if not integ)
- all_counts (nested list) : list of number of sequences,
structured as:
sess
x (unexp if by_exp or lock is "both")
x quantiles
if ret_arr:
- all_arrays (nested lists): list of data trace arrays, structured as:
session (x unexp if by_exp) x quantile
of 1 to 3D arrays:
(ROI x) sequences
(x frames if not integ)
"""
incr_unexp_ori = False
if stimpar.stimtype == "gabors":
if by_exp and isinstance(stimpar.gab_ori, int):
if stimpar.gabfr == 3:
incr_unexp_ori = True
warnings.warn(
"Incrementing orientation for unexpected segments to "
"ensure data is paired for by_exp split.",
category=RuntimeWarning, stacklevel=1
)
shift_gab_segs = False
remconsec, sample = False, False
unexp_vals = ["any"]
if lock in ["unexp", "exp", "both"]:
remconsec = True
unexp_vals = [1, 0]
if lock == "exp":
unexp_vals = [0]
elif lock == "unexp":
unexp_vals = [1]
if stimpar.stimtype == "gabors" and stimpar.gabfr not in ["any", "all"]:
shift_gab_segs = True
orig_gabfr = stimpar.gabfr
stimpar = sess_ntuple_util.get_modif_ntuple(stimpar, "gabfr", "any")
elif lock == "exp_samp":
remconsec, sample = False, True
unexp_vals = [0]
elif by_exp:
unexp_vals = [0, 1]
all_counts, all_stats, all_arrays = [], [], []
xrans = []
for sess in sessions:
stim = sess.get_stim(stimpar.stimtype)
sess_counts, sess_stats, sess_arrays = [], [], []
for unexp in unexp_vals:
stimpar_use = stimpar
if incr_unexp_ori and unexp == 1:
incr_ori = sess_gen_util.get_unexp_gab_ori(stimpar.gab_ori)
stimpar_use = sess_ntuple_util.get_modif_ntuple(
stimpar, "gab_ori", incr_ori
)
qu_segs, qu_counts = quant_segs(
stim, stimpar_use, n_quants, qu_idx, unexp, empty_ok=nan_empty,
remconsec=remconsec)
if shift_gab_segs: # shift to requested gabor frame
qu_segs = [[s + orig_gabfr for s in segs] for segs in qu_segs]
if sample:
pre_seg = stimpar.pre / stim.seg_len_s
post_seg = stimpar.post / stim.seg_len_s
qu_segs, qu_counts = samp_quant_segs(
qu_segs, pre_seg, post_seg, randst=randst
)
sess_counts.append(qu_counts)
trace_info = trace_stats_by_qu(
stim, qu_segs, stimpar.pre, stimpar.post, analyspar,
byroi=byroi, integ=integ, ret_arr=ret_arr, nan_empty=nan_empty,
baseline=baseline, datatype=datatype)
sess_stats.append(trace_info[1])
if ret_arr:
sess_arrays.append(trace_info[2])
xrans.append(trace_info[0])
if len(unexp_vals) > 1:
sess_stats = np.asarray(sess_stats)
else:
sess_stats = np.asarray(sess_stats[0]) # list of length 1
sess_counts = sess_counts[0]
if ret_arr:
sess_arrays = sess_arrays[0] # list of length 1
all_counts.append(sess_counts)
all_stats.append(sess_stats)
if ret_arr:
all_arrays.append(sess_arrays)
if ret_arr:
return xrans, all_stats, all_counts, all_arrays
else:
return xrans, all_stats, all_counts
def pupil_run_block_diffs(sessions,
analyspar,
sesspar,
stimpar,
permpar,
figpar,
seed=None,
parallel=False):
"""
pupil_run_block_diffs(sessions, analyspar, sesspar, stimpar, permpar,
figpar)
Retrieves pupil and running block differences for Gabor sequences for
session 1.
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list):
Session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
- sesspar (SessPar):
named tuple containing session parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- permpar (PermPar):
named tuple containing permutation parameters
- figpar (dict):
dictionary containing figure parameters
Optional args:
- seed (int):
seed value to use. (-1 treated as None)
default: None
- parallel (bool):
if True, some of the analysis is run in parallel across CPU cores
default: False
"""
logger.info("Compiling pupil and running block differences for session 1.",
extra={"spacing": "\n"})
permpar = sess_ntuple_util.get_modif_ntuple(permpar, "multcomp", False)
block_df = behav_analys.get_pupil_run_block_stats_df(sessions,
analyspar=analyspar,
stimpar=stimpar,
permpar=permpar,
randst=seed,
parallel=parallel)
extrapar = {"seed": seed}
info = {
"analyspar": analyspar._asdict(),
"sesspar": sesspar._asdict(),
"stimpar": stimpar._asdict(),
"permpar": permpar._asdict(),
"extrapar": extrapar,
"block_df": block_df.to_dict()
}
helper_fcts.plot_save_all(info, figpar)
def gabor_roi_usi_sig(sessions, analyspar, sesspar, stimpar, basepar,
idxpar, permpar, figpar, common_oris=False, seed=None,
parallel=False):
"""
gabor_roi_usi_sig(sessions, analyspar, sesspar, stimpar, basepar,
idxpar, permpar, figpar)
Retrieves percentage of signifiant ROI Gabor USIs.
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list):
Session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
- sesspar (SessPar):
named tuple containing session parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- basepar (BasePar):
named tuple containing baseline parameters
- idxpar (IdxPar):
named tuple containing index parameters
- permpar (PermPar):
named tuple containing permutation parameters
- figpar (dict):
dictionary containing figure parameters
Optional args:
- common_oris (bool):
if True, data is for common orientations
default: False
- seed (int):
seed value to use. (-1 treated as None)
default: None
- parallel (bool):
if True, some of the analysis is run in parallel across CPU cores
default: False
"""
common_str = ", with common orientations" if common_oris else ""
logger.info(
f"Compiling percentages of significant Gabor USIs{common_str}.",
extra={"spacing": "\n"}
)
if common_oris:
gab_ori = sess_gen_util.gab_oris_common_U(stimpar.gab_ori)
stimpar = sess_ntuple_util.get_modif_ntuple(
stimpar, "gab_ori", gab_ori
)
by_mouse = False
idx_df = usi_analys.get_idx_df(
sessions,
analyspar=analyspar,
stimpar=stimpar,
basepar=basepar,
idxpar=idxpar,
permpar=permpar,
common_oris=common_oris,
by_mouse=by_mouse,
randst=seed,
parallel=parallel,
)
permpar = misc_analys.set_multcomp(permpar, sess_df=idx_df, factor=2)
perc_sig_df = usi_analys.get_perc_sig_df(idx_df, analyspar, permpar, seed)
extrapar = {
"common_oris": common_oris,
"by_mouse" : by_mouse,
"seed" : seed,
}
info = {"analyspar" : analyspar._asdict(),
"sesspar" : sesspar._asdict(),
"stimpar" : stimpar._asdict(),
"basepar" : basepar._asdict(),
"idxpar" : idxpar._asdict(),
"permpar" : permpar._asdict(),
"extrapar" : extrapar,
"perc_sig_df": perc_sig_df.to_dict()
}
helper_fcts.plot_save_all(info, figpar)
def gabor_example_roi_usis(sessions, analyspar, sesspar, stimpar, basepar,
idxpar, permpar, figpar, seed=None, parallel=False):
"""
gabor_example_roi_usis(sessions, analyspar, sesspar, stimpar, basepar,
idxpar, permpar, figpar)
Retrieves example ROI Gabor USI traces.
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list):
Session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
- sesspar (SessPar):
named tuple containing session parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- basepar (BasePar):
named tuple containing baseline parameters
- idxpar (IdxPar):
named tuple containing index parameters
- permpar (PermPar):
named tuple containing permutation parameters
- figpar (dict):
dictionary containing figure parameters
Optional args:
- seed (int):
seed value to use. (-1 treated as None)
default: None
- parallel (bool):
if True, some of the analysis is run in parallel across CPU cores
default: False
"""
logger.info("Compiling Gabor example ROI USI data.",
extra={"spacing": "\n"}
)
# check stimpar.pre
if (not isinstance(stimpar.pre, list)) or (len(stimpar.pre) == 1):
pre_list = gen_util.list_if_not(stimpar.pre)
stimpar = sess_ntuple_util.get_modif_ntuple(stimpar, "pre", pre_list)
elif len(stimpar.pre) != 2:
raise ValueError("Expected 2 values for stimpar.pre: one for "
"index calculation and one for traces.")
# use first stimpar.pre for idx calculation
stimpar_idx = sess_ntuple_util.get_modif_ntuple(
stimpar, "pre", stimpar.pre[0]
)
chosen_rois_df = usi_analys.get_chosen_roi_df(
sessions,
analyspar=analyspar,
stimpar=stimpar_idx,
basepar=basepar,
idxpar=idxpar,
permpar=permpar,
target_idx_vals = [0.5, 0, -0.5],
target_idx_sigs = ["sig", "not_sig", "sig"],
randst=seed,
parallel=parallel
)
# use second stimpar.pre for traces
stimpar_tr = sess_ntuple_util.get_modif_ntuple(
stimpar, "pre", stimpar.pre[1]
)
chosen_rois_df = usi_analys.add_chosen_roi_traces(
sessions,
chosen_rois_df,
analyspar=analyspar,
stimpar=stimpar_tr,
basepar=basepar,
split=idxpar.feature,
parallel=parallel
)
extrapar = {"seed": seed}
info = {
"analyspar": analyspar._asdict(),
"stimpar": stimpar._asdict(),
"sesspar": sesspar._asdict(),
"basepar": basepar._asdict(),
"idxpar": idxpar._asdict(),
"permpar": permpar._asdict(),
"extrapar": extrapar,
"chosen_rois_df": chosen_rois_df.to_dict()
}
helper_fcts.plot_save_all(info, figpar)
def get_stim_stats_df(sessions,
analyspar,
stimpar,
permpar,
comp_sess=[1, 3],
datatype="rel_unexp_resp",
rel_sess=1,
basepar=None,
idxpar=None,
pop_stats=True,
randst=None,
parallel=False):
"""
get_stim_stats_df(sessions, analyspar, stimpar, permpar)
Returns dataframe with comparison of absolute fractional data changes
between sessions for different stimuli.
Required args:
- sessions (list):
session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- permpar (PermPar):
named tuple containing permutation parameters
Optional args:
- comp_sess (int):
sessions for which to obtain absolute fractional change
[x, y] => |(y - x) / x|
default: [1, 3]
- datatype (str):
type of data to retrieve
default: "rel_unexp_resp"
- rel_sess (int):
number of session relative to which data should be scaled, for each
mouse
default: 1
- basepar (BasePar):
named tuple containing baseline parameters
(needed if datatype is "usis")
default: None
- idxpar (IdxPar):
named tuple containing index parameters
(needed if datatype is "usis")
default: None
- pop_stats (bool):
if True, analyses are run on population statistics, and not
individual tracked ROIs
default: True
- randst (int or np.random.RandomState):
random state or seed value to use. (-1 treated as None)
default: None
- parallel (bool):
if True, some of the analysis is run in parallel across CPU cores
default: False
Returns:
- stim_stats_df (pd.DataFrame):
dataframe with one row per line/plane and one for all line/planes
together, and the basic sess_df columns, in addition to,
for each stimtype:
- {stimtype} (list): absolute fractional change statistics (me, err)
- raw_p_vals (float): uncorrected p-value for data differences
between stimulus types
- p_vals (float): p-value for data differences between stimulus
types, corrected for multiple comparisons and tails
"""
if not pop_stats:
if analyspar.tracked:
misc_analys.check_sessions_complete(sessions, raise_err=True)
else:
raise ValueError(
"If analysis is run for individual ROIs and not population "
"statistics, analyspar.tracked must be set to True.")
if set(stimpar.stimtype) != set(["gabors", "visflow"]):
raise ValueError(
"Expected stimpar.stimtype to list 'gabors' and 'visflow'.")
if (not (isinstance(stimpar.pre, list) and isinstance(stimpar.post, list))
or not (len(stimpar.pre) == 2 and len(stimpar.post) == 2)):
raise ValueError(
"stimpar.pre and stimpar.post must be provided as lists of "
"length 2 (one value per stimpar.stimtype, in order).")
if datatype == "usis":
if (not isinstance(idxpar.feature, list)
or not len(idxpar.feature) == 2):
raise ValueError(
"idxpar.feature must be provided as a list of length 2 "
"(one value per stimpar.stimtype, in order).")
stim_stats_df = None
for s, stimtype in enumerate(stimpar.stimtype):
stim_stimpar = sess_ntuple_util.get_modif_ntuple(
stimpar, ["stimtype", "pre", "post"],
[stimtype, stimpar.pre[s], stimpar.post[s]])
stim_idxpar = idxpar
if datatype == "usis":
stim_idxpar = sess_ntuple_util.get_modif_ntuple(
idxpar, "feature", idxpar.feature[s])
stim_stats_df = get_stim_data_df(sessions,
analyspar,
stim_stimpar,
stim_data_df=stim_stats_df,
comp_sess=comp_sess,
datatype=datatype,
rel_sess=rel_sess,
basepar=basepar,
idxpar=stim_idxpar,
abs_usi=pop_stats,
parallel=parallel)
# add statistics and p-values
add_stim_stats = add_stim_pop_stats if pop_stats else add_stim_roi_stats
stim_stats_df = add_stim_stats(stim_stats_df,
sessions,
analyspar,
stimpar,
permpar,
comp_sess=comp_sess,
in_place=True,
randst=randst)
# adjust data columns
data_cols = []
for s, stimtype in enumerate(stimpar.stimtype):
for n in comp_sess:
data_cols.append(f"{stimtype}_s{n}")
stim_stats_df = stim_stats_df.drop(data_cols, axis=1)
stim_stats_df["sess_ns"] = f"comp{comp_sess[0]}v{comp_sess[1]}"
# corrected p-values
stim_stats_df = misc_analys.add_corr_p_vals(stim_stats_df, permpar)
return stim_stats_df