def get_correlation(session, analyspar, rolling_win=4):
"""
get_correlation(session, analyspar)
Returns ROI correlations for a session.
Required args:
- session (Session):
Session object
- analyspar (AnalysPar):
named tuple containing analysis parameters
Optional args:
- rolling_win (int):
window to use in rolling mean over individual traces before
computing correlation between ROIs (None for no smoothing)
default: 4
Returns:
- corr_triu (1D array):
all correlations
"""
if session.only_tracked_rois != analyspar.tracked:
raise RuntimeError(
"session.only_tracked_rois should match analyspar.tracked."
)
if analyspar.scale:
raise ValueError(
"analyspar.scale must be False for correlation analysis."
)
full_traces_df = session.get_roi_traces(
fluor=analyspar.fluor,
rem_bad=analyspar.rem_bad
)
full_traces = gen_util.reshape_df_data(full_traces_df, squeeze_cols=True)
if rolling_win is not None:
full_traces = math_util.rolling_mean(full_traces, win=rolling_win)
corrs = np.corrcoef(full_traces)
corr_triu = corrs[np.triu_indices(len(corrs), k=1)]
return corr_triu
def run_autocorr(sessions,
analysis,
analyspar,
sesspar,
stimpar,
autocorrpar,
figpar,
datatype="roi"):
"""
run_autocorr(sessions, analysis, analyspar, sesspar, stimpar, autocorrpar,
figpar)
Calculates and plots autocorrelation during stimulus blocks.
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list) : list of Session objects
- analysis (str) : analysis type (e.g., "a")
- analyspar (AnalysPar) : named tuple containing analysis parameters
- sesspar (SessPar) : named tuple containing session parameters
- stimpar (StimPar) : named tuple containing stimulus parameters
- autocorrpar (AutocorrPar): named tuple containing autocorrelation
analysis parameters
- figpar (dict) : dictionary containing figure parameters
Optional args:
- datatype (str): type of data (e.g., "roi", "run")
"""
sessstr_pr = sess_str_util.sess_par_str(sesspar.sess_n, stimpar.stimtype,
sesspar.plane, stimpar.visflow_dir,
stimpar.visflow_size, stimpar.gabk,
"print")
dendstr_pr = sess_str_util.dend_par_str(analyspar.dend, sesspar.plane,
datatype, "print")
datastr = sess_str_util.datatype_par_str(datatype)
logger.info(
f"Analysing and plotting {datastr} autocorrelations "
f"({sessstr_pr}{dendstr_pr}).",
extra={"spacing": "\n"})
xrans = []
stats = []
for sess in sessions:
if datatype == "roi" and (sess.only_tracked_rois != analyspar.tracked):
raise RuntimeError(
"sess.only_tracked_rois should match analyspar.tracked.")
stim = sess.get_stim(stimpar.stimtype)
all_segs = stim.get_segs_by_criteria(visflow_dir=stimpar.visflow_dir,
visflow_size=stimpar.visflow_size,
gabk=stimpar.gabk,
by="block")
sess_traces = []
for segs in all_segs:
if len(segs) == 0:
continue
segs = sorted(segs)
# check that segs are contiguous
if max(np.diff(segs)) > 1:
raise NotImplementedError("Segments used for autocorrelation "
"must be contiguous within blocks.")
if datatype == "roi":
frame_edges = stim.get_fr_by_seg(
[min(segs), max(segs)], fr_type="twop")
fr = list(range(min(frame_edges[0]), max(frame_edges[1]) + 1))
traces = gen_util.reshape_df_data(sess.get_roi_traces(
fr,
fluor=analyspar.fluor,
rem_bad=analyspar.rem_bad,
scale=analyspar.scale),
squeeze_cols=True)
elif datatype == "run":
if autocorrpar.byitem != False:
raise ValueError("autocorrpar.byitem must be False for "
"running data.")
frame_edges = stim.get_fr_by_seg(
[min(segs), max(segs)], fr_type="stim")
fr = list(range(min(frame_edges[0]), max(frame_edges[1]) + 1))
traces = sess.get_run_velocity_by_fr(
fr,
fr_type="stim",
rem_bad=analyspar.rem_bad,
scale=analyspar.scale).to_numpy().reshape(1, -1)
sess_traces.append(traces)
# Calculate autocorr stats while filtering some warnings
msgs = ["Degrees of freedom", "invalid value encountered"]
categs = [RuntimeWarning, RuntimeWarning]
with gen_util.TempWarningFilter(msgs, categs):
xran, ac_st = math_util.autocorr_stats(sess_traces,
autocorrpar.lag_s,
sess.twop_fps,
byitem=autocorrpar.byitem,
stats=analyspar.stats,
error=analyspar.error)
if not autocorrpar.byitem: # also add a 10x lag
lag_fr = 10 * int(autocorrpar.lag_s * sess.twop_fps)
_, ac_st_10x = math_util.autocorr_stats(sess_traces,
lag_fr,
byitem=autocorrpar.byitem,
stats=analyspar.stats,
error=analyspar.error)
downsamp = range(0, ac_st_10x.shape[-1], 10)
if len(downsamp) != ac_st.shape[-1]:
raise RuntimeError("Failed to downsample correctly. "
"Check implementation.")
ac_st = np.stack([ac_st, ac_st_10x[:, downsamp]], axis=1)
xrans.append(xran)
stats.append(ac_st)
autocorr_data = {
"xrans": [xran.tolist() for xran in xrans],
"stats": [stat.tolist() for stat in stats]
}
sess_info = sess_gen_util.get_sess_info(sessions,
analyspar.fluor,
incl_roi=(datatype == "roi"),
rem_bad=analyspar.rem_bad)
extrapar = {
"analysis": analysis,
"datatype": datatype,
}
info = {
"analyspar": analyspar._asdict(),
"sesspar": sesspar._asdict(),
"stimpar": stimpar._asdict(),
"extrapar": extrapar,
"autocorrpar": autocorrpar._asdict(),
"autocorr_data": autocorr_data,
"sess_info": sess_info
}
fulldir, savename = gen_plots.plot_autocorr(figpar=figpar, **info)
file_util.saveinfo(info, savename, fulldir, "json")
def run_full_traces(sessions,
analysis,
analyspar,
sesspar,
figpar,
datatype="roi"):
"""
run_full_traces(sessions, analysis, analyspar, sesspar, figpar)
Plots full traces across an entire session. If ROI traces are plotted,
each ROI is scaled and plotted separately and an average is plotted.
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list) : list of Session objects
- analysis (str) : analysis type (e.g., "f")
- analyspar (AnalysPar): named tuple containing analysis parameters
- sesspar (SessPar) : named tuple containing session parameters
- figpar (dict) : dictionary containing figure parameters
Optional args:
- datatype (str): type of data (e.g., "roi", "run")
"""
dendstr_pr = sess_str_util.dend_par_str(analyspar.dend, sesspar.plane,
datatype, "print")
sessstr_pr = (f"session: {sesspar.sess_n}, "
f"plane: {sesspar.plane}{dendstr_pr}")
datastr = sess_str_util.datatype_par_str(datatype)
logger.info(
f"Plotting {datastr} traces across an entire "
f"session\n({sessstr_pr}).",
extra={"spacing": "\n"})
figpar = copy.deepcopy(figpar)
if figpar["save"]["use_dt"] is None:
figpar["save"]["use_dt"] = gen_util.create_time_str()
all_tr, roi_tr, all_edges, all_pars = [], [], [], []
for sess in sessions:
# get the block edges and parameters
edge_fr, par_descrs = [], []
for stim in sess.stims:
stim_str = stim.stimtype
if stim.stimtype == "visflow":
stim_str = "vis. flow"
if datatype == "roi":
fr_type = "twop"
elif datatype == "run":
fr_type = "stim"
else:
gen_util.accepted_values_error("datatype", datatype,
["roi", "run"])
for b in stim.block_params.index:
row = stim.block_params.loc[b]
edge_fr.append([
int(row[f"start_frame_{fr_type}"]),
int(row[f"stop_frame_{fr_type}"])
])
par_vals = [row[param] for param in stim.stim_params]
pars_str = "\n".join([str(par) for par in par_vals][0:2])
par_descrs.append(
sess_str_util.pars_to_descr(
f"{stim_str.capitalize()}\n{pars_str}"))
if datatype == "roi":
if sess.only_tracked_rois != analyspar.tracked:
raise RuntimeError(
"sess.only_tracked_rois should match analyspar.tracked.")
nanpol = None
if not analyspar.rem_bad:
nanpol = "omit"
all_rois = gen_util.reshape_df_data(sess.get_roi_traces(
None, analyspar.fluor, analyspar.rem_bad,
analyspar.scale)["roi_traces"],
squeeze_cols=True)
full_tr = math_util.get_stats(all_rois,
analyspar.stats,
analyspar.error,
axes=0,
nanpol=nanpol).tolist()
roi_tr.append(all_rois.tolist())
elif datatype == "run":
full_tr = sess.get_run_velocity(
rem_bad=analyspar.rem_bad,
scale=analyspar.scale).to_numpy().squeeze().tolist()
roi_tr = None
all_tr.append(full_tr)
all_edges.append(edge_fr)
all_pars.append(par_descrs)
extrapar = {
"analysis": analysis,
"datatype": datatype,
}
trace_info = {
"all_tr": all_tr,
"all_edges": all_edges,
"all_pars": all_pars
}
sess_info = sess_gen_util.get_sess_info(sessions,
analyspar.fluor,
incl_roi=(datatype == "roi"),
rem_bad=analyspar.rem_bad)
info = {
"analyspar": analyspar._asdict(),
"sesspar": sesspar._asdict(),
"extrapar": extrapar,
"sess_info": sess_info,
"trace_info": trace_info
}
fulldir, savename = gen_plots.plot_full_traces(roi_tr=roi_tr,
figpar=figpar,
**info)
file_util.saveinfo(info, savename, fulldir, "json")
def get_decoding_data(sess, analyspar, stimpar, comp="Dori", ctrl=False):
"""
get_decoding_data(sess, analyspar, stimpar)
Retrieves data for decoding.
Required args:
- sess (Session):
Session object
- analyspar (AnalysPar):
named tuple containing analysis parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
Optional args:
- comp (str):
comparison used to define classes ("Dori" or "Eori")
default: "Dori"
- ctrl (bool):
if True, the number of examples per class for the unexpected data
is returned (applies to "Dori" comp only).
default: False
Returns:
- all_input_data (3D array):
input data, dims: seq x frames x ROIs
- all_target_data (1D array):
class target for each input sequence
- ctrl_ns (list):
number of examples per class for the unexpected data
(None if it doesn't apply)
"""
if stimpar.stimtype != "gabors":
raise ValueError("Expected stimpar.stimtype to be 'gabors'.")
if comp == "Dori":
unexp = 0
ctrl_ns = []
elif comp == "Uori":
unexp = 1
ctrl = False
ctrl_ns = False
else:
gen_util.accepted_values_error("comp", comp, ["Dori", "Uori"])
gab_oris = sess_gen_util.filter_gab_oris(comp[0], stimpar.gab_ori)
stim = sess.get_stim(stimpar.stimtype)
all_input_data = []
all_target_data = []
for g, gab_ori in enumerate(gab_oris):
segs = stim.get_segs_by_criteria(gabfr=stimpar.gabfr,
gabk=stimpar.gabk,
gab_ori=gab_ori,
unexp=unexp,
remconsec=False,
by="seg")
fr_ns = stim.get_fr_by_seg(segs, start=True,
fr_type="twop")["start_frame_twop"]
# sample as many sequences as are usable for unexpected data
if ctrl:
ctrl_gab_ori = sess_gen_util.get_unexp_gab_ori(gab_ori)
segs_ctrl = stim.get_segs_by_criteria(gabfr=stimpar.gabfr,
gabk=stimpar.gabk,
gab_ori=ctrl_gab_ori,
unexp=1,
remconsec=False,
by="seg")
fr_ns_ctrl = stim.get_fr_by_seg(segs_ctrl,
start=True,
ch_fl=[stimpar.pre, stimpar.post],
fr_type="twop")["start_frame_twop"]
ctrl_ns.append(len(fr_ns_ctrl))
ori_data_df = stim.get_roi_data(fr_ns,
stimpar.pre,
stimpar.post,
rem_bad=analyspar.rem_bad,
scale=analyspar.scale)
# seq x frames x ROIs
ori_data = np.transpose(
gen_util.reshape_df_data(ori_data_df, squeeze_cols=True),
[1, 2, 0])
all_input_data.append(ori_data)
all_target_data.append(np.full(len(ori_data), g))
all_input_data = np.concatenate(all_input_data, axis=0)
all_target_data = np.concatenate(all_target_data)
return all_input_data, all_target_data, ctrl_ns
def get_stim_data(sess,
stimtype,
win_leng_s,
gabfr=0,
pre=0,
post=1.5,
unexp="any",
step_size=1,
gabk=16,
run=True,
run_mean=None,
run_std=None):
"""
get_stim_data(sess, stimtype, win_leng_s)
Returns stimulus data (x position, y position, size, orientation, each
scaled based on its maximal range), and optionally running for windows
taken for the each of the segments of interest.
Required args:
- sess (Session) : session
- stimtype (str) : stimulus type ("gabors"or "visflow")
- win_leng_s (num) : window length in seconds
Optional args:
- gabfr (int) : gabor reference frame for determining the
2p frames in each sequence
default: 0
- pre (num) : number of frames to include before reference
gabor frame in each sequence (in sec)
default: 0
- post (num) : number of frames to include after reference
gabor frame in each sequence (in sec)
default: 1.5
- unexp (str, list or int): unexpected value criteria for including
reference gabor frames (0, 1, or "any")
default: "any"
- step_size (int) : step size between windows
default: 1
- gabk (int or list) : gabor kappa criteria for including reference
gabor frames (4, 16 or "any")
default: 16
- run (bool) : if True, running data is appended to the
end of the stimulus data
default: True
- run_mean (num) : mean value with which to scale running
data, if running data is included. If
run_mean or run_std is None, both are
calculated from the running data retrieved
and returned as outputs.
default: None
- run_std (num) : standard deviation value with which to
scale running data, if running data is
included. If run_mean or run_std is None,
both are calculated from the running data
retrieved and returned as outputs.
default: None
Returns:
- stim_wins (3D array): array of stimulus data, structured as:
seq wins x frames x pars, where the pars are:
- for each gabor: x_pos, y_pos, size, ori
- run velocity
if run and run_mean or run_std is None:
- (list):
- run_mean (num) : mean of retrieved running values
- run_std (num) : standard deviation of retrieved running values
"""
if win_leng_s > (pre + post):
raise ValueError("Windows cannot be longer than the sequences.")
stim = sess.get_stim(stimtype)
segs = stim.get_segs_by_criteria(gabfr=gabfr,
gabk=gabk,
unexp=unexp,
by="seg")
twopfr = stim.get_fr_by_seg(segs, start=True,
fr_type="twop")["start_frame_twop"]
# get stim params in df with indices seg x frame x gabor x par
# (x, y, ori, size). Each param scaled to between -1 and 1 based on known
# ranges from which they were sampled
pars_df = stim.get_stim_par_by_fr(twopfr,
pre,
post,
scale=True,
fr_type="twop")
targ = [len(pars_df.index.unique(lev)) for lev in pars_df.index.names] + \
[len(pars_df.columns.unique("parameters"))]
targ[1] = -1 # 2p frame number is not repeated across sequences
pars = pars_df.to_numpy().reshape(targ)
if run:
twop_fr_seqs = gen_util.reshape_df_data(sess.get_fr_ran(
twopfr, pre, post, fr_type="twop"),
squeeze_cols=True)
run_velocity = gen_util.reshape_df_data(sess.get_run_velocity_by_fr(
twop_fr_seqs, rem_bad=True, scale=False),
squeeze_cols=True)
# scale running array to mean 0 with std 1
ret_run_stats = False
if run_mean is None or run_std is None:
ret_run_stats = True
run_mean = np.mean(run_velocity)
run_std = np.std(run_velocity)
run_velocity = 2. * (run_velocity - run_mean) / run_std - 1.
# stim params: seq x frame x flat (gab x pars)
all_pars = pars.reshape([pars.shape[0], pars.shape[1], -1])
if run:
all_pars = np.concatenate([all_pars, run_velocity[:, :, np.newaxis]],
axis=2)
win_leng = int(np.floor(win_leng_s * sess.twop_fps))
stim_wins = []
for seq_pars in all_pars:
stim_wins.append(
data_util.window_2d(seq_pars, win_leng, step_size=step_size))
stim_wins = np.concatenate(stim_wins, axis=0).transpose([0, 2, 1])
if run and ret_run_stats:
return stim_wins, [run_mean, run_std]
else:
return stim_wins
def get_roi_data(sess,
stimtype,
win_leng_s,
gabfr=0,
pre=0,
post=1.5,
unexp="any",
step_size=1,
gabk=16,
roi_means=None,
roi_stds=None):
"""
get_roi_data(sess, stimtype, win_leng_s)
Returns stimulus data (x position, y position, size, orientation, each
scaled based on its maximal range), and optionally running for windows
taken for the each of the segments of interest.
Required args:
- sess (Session) : session
- stimtype (str) : stimulus type ("gabors"or "visflow")
- win_leng_s (num) : window length in seconds
Optional args:
- gabfr (int) : gabor reference frame for determining the
2p frames in each sequence
default: 0
- pre (num) : number of frames to include before reference
gabor frame in each sequence (in sec)
default: 0
- post (num) : number of frames to include after reference
gabor frame in each sequence (in sec)
default: 1.5
- unexp (str, list or int): unexpected value criteria for including
reference gabor frames (0, 1, or "any")
default: "any"
- step_size (int) : step size between windows
default: 1
- gabk (int or list) : gabor kappa criteria for including reference
gabor frames (4, 16 or "any")
default: 16
- roi_means (1D array) : mean values for each ROI with which to
scale trace data. If roi_means or
roi_stds is None, both are calculated from
the trace data retrieved and returned as
outputs.
default: None
- roi_stds (1D array) : standard deviation values for each ROI with
which to scale trace data. If roi_means
or roi_stds is None, both are calculated
from the trace data retrieved and returned
as outputs.
default: None
Returns:
- xran (1D array) : time values for the 2p frames
- trace_wins (3D array): array of ROI data, structured as:
seq wins x frames x ROI
if roi_means or roi_stds is None:
- (list):
- roi_means (1D array): trace means for each ROI
- roi_stds (1D array) : trace standard deviations for each ROI
"""
if win_leng_s > (pre + post):
raise ValueError("Windows cannot be longer than the sequences.")
stim = sess.get_stim(stimtype)
segs = stim.get_segs_by_criteria(gabfr=gabfr,
gabk=gabk,
unexp=unexp,
by="seg")
twopfr = stim.get_fr_by_seg(segs, start=True,
fr_type="twop")["start_frame_twop"]
roi_data_df = stim.get_roi_data(twopfr,
pre,
post,
rem_bad=True,
scale=False)
ret_roi_stats = False
xran = roi_data_df.index.unique("time_values").to_numpy()
traces = gen_util.reshape_df_data(roi_data_df, squeeze_cols=True)
# scale each ROI to mean 0, std 1
if roi_means is None or roi_stds is None:
ret_roi_stats = True
roi_means = np.mean(traces.reshape(traces.shape[0], -1), axis=1)
roi_stds = np.std(traces.reshape(traces.shape[0], -1), axis=1)
traces = 2. * (traces - roi_means.reshape([-1, 1, 1]))/ \
roi_stds.reshape([-1, 1, 1]) - 1.
# traces: seq x frames x ROI
traces = traces.transpose(1, 2, 0)
trace_wins = []
win_leng = int(np.floor(sess.twop_fps * win_leng_s))
for seq_traces in traces:
# n_wins x n_ROIs x win_leng
trace_wins.append(
data_util.window_2d(seq_traces, win_leng, step_size=step_size))
# concatenate windows
trace_wins = np.concatenate(trace_wins, axis=0).transpose([0, 2, 1])
if ret_roi_stats:
return xran, trace_wins, [roi_means, roi_stds]
else:
return xran, trace_wins
def trace_stats_by_qu(stim, qu_segs, pre, post, analyspar, byroi=True,
integ=False, ret_arr=False, nan_empty=False,
baseline=None, datatype="roi"):
"""
trace_stats_by_qu(stim, qu_seg, pre, post, analyspar)
Returns trace statistics for the quantiles of interest. If ret_arr, also
returns trace data arrays.
Required args:
- stim (Stim object) : stim object
- qu_segs (dict) : list of sublists for each quantile, each
containing segment numbers for that quantile
- pre (num) : range of frames to include before each frame
reference (in s)
- post (num) : range of frames to include after each frame
reference (in s)
- analyspar (AnalysPar): named tuple containing analysis parameters
Optional args:
- byroi (bool) : If datatype is "roi", if True, returns statistics
for each ROI. If False, returns statistics
across ROIs.
default: True
- integ (bool) : if True, dF/F is integrated over sequences
default: False
- ret_arr (bool) : if True, data arrays are returned also
default: False
- nan_empty (bool): if a quantile is empty, returns NaN arrays instead
of an error (1 sequence, for qu_array)
default: False
- baseline (num) : number of seconds to use as baseline. If None,
data is not baselined.
default: None
- datatype (str) : datatype, i.e. ROIs or running
default: "roi"
Returns:
- xran (1D array) : time values for the 2p frames (None if
integ)
- qu_stats (2 to 4D array) : trace data statistics, structured as:
quantiles x
stats (me, err) x
(ROIs if byroi x)
(frames if not integ)
if ret_arr, also:
- qu_array (list) : list per quantile of 1-3D arrays of trace
data structured as:
(ROIs x) sequences
(x frames if not integ)
"""
if datatype == "roi" and (stim.sess.only_tracked_rois != analyspar.tracked):
raise RuntimeError(
"stim.sess.only_tracked_rois should match analyspar.tracked."
)
qu_stats, qu_array = [], []
xran = None
for segs in qu_segs:
rep_nan = False
for _ in range(2): # allows retrying if nan_empty is True
try:
if datatype == "roi":
twop_fr = stim.get_fr_by_seg(
segs, start=True, fr_type="twop")["start_frame_twop"]
trace_df = stim.get_roi_stats_df(twop_fr, pre, post,
byroi=byroi, fluor=analyspar.fluor,
rem_bad=analyspar.rem_bad,
stats=analyspar.stats, error=analyspar.error,
integ=integ, ret_arr=ret_arr, scale=analyspar.scale,
baseline=baseline)
elif datatype == "run":
stim_fr = stim.get_fr_by_seg(
segs, start=True, fr_type="stim")["start_frame_stim"]
trace_df = stim.get_run_stats_df(stim_fr, pre, post,
rem_bad=analyspar.rem_bad,
stats=analyspar.stats, error=analyspar.error,
integ=integ, ret_arr=ret_arr, scale=analyspar.scale,
baseline=baseline)
else:
gen_util.accepted_values_error(
"datatype", datatype, ["roi", "run"])
break # break out of for loop if successful
except Exception as err: # RuntimeError or ValueError
empty = ("No frames" in str(err) or "No segments" in str(err))
if nan_empty and empty:
segs = [10] # dummy segment to use
rep_nan = True # later, replace values with NaNs
else:
raise err
if not integ:
xran = trace_df.index.unique("time_values").to_numpy()
# array: stats [me, err] (x ROI) (x frames)
# (catch performance warning for unsorted large dataframes)
msg, categ = ["indexing past lexsort", pd.errors.PerformanceWarning]
with gen_util.TempWarningFilter(msg, categ):
trace_stats = gen_util.reshape_df_data(
trace_df.loc["stats", ], squeeze_cols=True)
if datatype == "roi":
if not byroi:
trace_stats = trace_stats.squeeze(0)
else:
trace_stats = trace_stats.transpose(
1, 0, *range(len(trace_stats.shape))[2:])
if rep_nan: # replace dummy values with NaNs
trace_stats = np.full_like(trace_stats, np.nan)
qu_stats.append(trace_stats)
if ret_arr:
trace_array = gen_util.reshape_df_data(
trace_df.loc["data", ], squeeze_cols=True)
if rep_nan: # replace dummy values with NaNs
trace_array = np.full_like(trace_array, np.nan)
qu_array.append(trace_array)
qu_stats = np.asarray(qu_stats)
if ret_arr:
return xran, qu_stats, qu_array
else:
return xran, qu_stats
def get_data(stim,
refs,
analyspar,
pre=0,
post=1,
ch_fl=None,
integ=False,
ref_type="segs",
datatype="roi"):
"""
get_data(stim, refs, analyspar)
Returns data for a specific stimulus around sequence references provided.
Required args:
- stim (Stim):
Stimulus object
- refs (1D array):
Sequences references (either segments or frames, specified by
ref_type)
- analyspar (AnalysPar):
named tuple containing analysis parameters
Optional args:
- pre (num):
number of seconds to keep before refs
default: 0
- post (num):
number of seconds to keep after refs
default: 1
- ch_fl (list):
flanks to check for discarding refs with insufficient flanks
default: None
- integ (bool):
if True, sequence data is integrated
default: False
- ref_type (str):
type of references provided
(segments, twop frames or stimulus frames)
("segs", "twop", "stim")
default: "segs"
- datatype (str):
type of data to return ("roi", "run" or "pupil")
default: "roi"
Returns:
- data_arr (1-3D array):
sequence data array
dims: (ROIs x) seq (x frames)
- time_values (1D array):
values for each frame, in seconds
"""
if stim.sess.only_tracked_rois != analyspar.tracked:
raise RuntimeError(
"stim.sess.only_tracked_rois should match analyspar.tracked.")
fr_ns, _ = get_frame_numbers(stim,
refs,
ch_fl=ch_fl,
ref_type=ref_type,
datatype=datatype)
# obtain data
if datatype == "roi":
data_df = stim.get_roi_data(fr_ns,
pre,
post,
rem_bad=analyspar.rem_bad,
scale=analyspar.scale)
col_name = "roi_traces"
integ_dt = stim.sess.twop_fps
elif datatype == "run":
data_df = stim.get_run_data(fr_ns,
pre,
post,
rem_bad=analyspar.rem_bad,
scale=analyspar.scale)
col_name = "run_velocity"
integ_dt = stim.sess.stim_fps
elif datatype == "pupil":
data_df = stim.get_pup_diam_data(fr_ns,
pre,
post,
rem_bad=analyspar.rem_bad,
scale=analyspar.scale)
col_name = "pup_diam"
integ_dt = stim.sess.twop_fps
else:
gen_util.accepted_values_error("datatype", datatype,
["roi", "run", "pupil"])
time_values = data_df.index.unique("time_values").to_numpy()
data_arr = gen_util.reshape_df_data(data_df[col_name], squeeze_cols=True)
if integ:
nanpol = None if analyspar.rem_bad else "omit"
data_arr = math_util.integ(data_arr,
1. / integ_dt,
axis=-1,
nanpol=nanpol)
return data_arr, time_values
def calc_tune_curvs(sess,
analyspar,
stimpar,
nrois="all",
ngabs="all",
grp2="unexp",
comb_gabs=True,
vm_estim=False,
collapse=True,
parallel=True):
"""
calc_tune_curvs(sess, analyspar, stimpar)
Returns orientations and corresponding fluorescence levels for the sessions
of interest.
Required args:
- sess (Session) : session
- analyspar (AnalysPar): named tuple containing analysis parameters
- stimpar (StimPar) : named tuple containing stimulus parameters
Optional args:
- nrois (int) : number of ROIs to include in analysis
default: "all"
- ngabs (int) : number of gabors to include in analysis (set to 1
if comb_gabs, as all gabors are combined)
default: "all"
- grp2 (str) : second group: either unexp, exp or rand (random
subsample of exp, the size of unexp)
default: "unexp"
- comb_gabs (bool): if True, all gabors have been combined for
gab_oris and roi_data
default: False
- vm_estim (bool) : if True, analysis is run using a von Mises tuning
curve estimation method
- collapse (bool) : if True, opposite orientations in the 0 to 360
range are collapsed to the 0 to 180 range
- parallel (bool) : if True, some of the analysis is parallelized
across CPU cores
Returns:
- tc_oris (list) : list of orientation values corresponding to the
tc_data:
unexp x gabor (1 if comb_gabs) x oris
- tc_data (list) : list of mean integrated fluorescence data per
orientation, for each ROI, structured as
ROI x unexp x gabor (1 if comb_gabs)
x oris
- tc_nseqs (list) : number of sequences per unexp
if vm_estim, also:
- tc_vm_pars (list) : nested list of Von Mises parameters for each ROI:
ROI x unexp x gabor (1 if comb_gabs) x par
- tc_vm_mean (list) : nested list of mean Von Mises means for each ROI,
not weighted by kappa value or weighted (if not
comb_gabs) (in rad):
ROI x unexp x kappa weighted (False, (True))
- tc_hist_pars (list): parameters used to convert tc_data to histogram
values (sub, mult) used in Von Mises parameter
estimation, structured as:
ROI x unexp x gabor (1 if comb_gabs) x
param (sub, mult)
"""
if sess.only_tracked_rois != analyspar.tracked:
raise RuntimeError(
"sess.only_tracked_rois should match analyspar.tracked.")
gabfrs = gen_util.list_if_not(stimpar.gabfr)
if len(gabfrs) == 1:
gabfrs = gabfrs * 2
if grp2 == "unexp":
unexps = [0, 1]
elif grp2 in ["exp", "rand"]:
unexps = [0, 0]
else:
gen_util.accepted_values_error("grp2", grp2, ["unexp", "exp", "rand"])
stim = sess.get_stim(stimpar.stimtype)
nrois_tot = sess.get_nrois(analyspar.rem_bad, analyspar.fluor)
ngabs_tot = stim.n_patches
if nrois == "all":
sess_nrois = nrois_tot
else:
sess_nrois = np.min([nrois_tot, nrois])
if ngabs == "all":
sess_ngabs = stim.n_patches
else:
sess_ngabs = np.min([stim.n_patches, ngabs])
tc_data, tc_oris, tc_nseqs = [], [], []
# estimate tuning curves by fitting a Von Mises distribution
if vm_estim:
tc_vm_pars, tc_vm_mean, tc_hist_pars = [], [], []
for i, (gf, e) in enumerate(zip(gabfrs, unexps)):
# get segments
segs = stim.get_segs_by_criteria(gabfr=gf,
visflow_dir=stimpar.visflow_dir,
visflow_size=stimpar.visflow_size,
gabk=stimpar.gabk,
unexp=e,
by="seg")
if grp2 == "rand" and i == 1:
n_segs = len(
stim.get_segs_by_criteria(gabfr=gf,
visflow_dir=stimpar.visflow_dir,
visflow_size=stimpar.visflow_size,
gabk=stimpar.gabk,
unexp=1,
by="seg"))
np.random.shuffle(segs)
segs = sorted(segs[:n_segs])
tc_nseqs.append(len(segs))
twopfr = stim.get_fr_by_seg(segs, start=True,
fr_type="twop")["start_frame_twop"]
# ROI x seq
roi_data = gen_util.reshape_df_data(stim.get_roi_data(
twopfr,
stimpar.pre,
stimpar.post,
analyspar.fluor,
integ=True,
rem_bad=analyspar.rem_bad,
scale=analyspar.scale)["roi_traces"],
squeeze_cols=True)[:sess_nrois]
# gab x seq
gab_oris = gen_util.reshape_df_data(stim.get_stim_par_by_seg(
segs, pos=False, ori=True, size=False),
squeeze_cols=True).T
if collapse:
gab_oris = collapse_dir(gab_oris)
if comb_gabs:
ngabs = 1
gab_oris = gab_oris.reshape([1, -1])
roi_data = np.tile(roi_data, [1, ngabs_tot])
tc_oris.append(gab_oris.tolist())
tc_data.append(roi_data.tolist())
# estimate tuning curves by fitting a Von Mises distribution
if vm_estim:
[
gab_tc_oris, gab_tc_data, gab_vm_pars, gab_vm_mean,
gab_hist_pars
] = tune_curv_estims(gab_oris,
roi_data,
ngabs_tot,
sess_nrois,
sess_ngabs,
comb_gabs,
collapse=False,
parallel=parallel)
tc_oris[i] = gab_tc_oris
tc_data[i] = gab_tc_data
tc_vm_pars.append(gab_vm_pars)
tc_vm_mean.append(gab_vm_mean)
tc_hist_pars.append(gab_hist_pars)
if vm_estim:
return tc_oris, tc_data, tc_nseqs, tc_vm_pars, tc_vm_mean, tc_hist_pars
else:
return tc_oris, tc_data, tc_nseqs
def peristim_data(sess,
stimpar,
ran_s=None,
datatype="both",
returns="diff",
fluor="dff",
stats="mean",
rem_bad=True,
scale=False,
first_unexp=True,
trans_all=False):
"""
peristim_data(sess, stimpar)
Returns pupil, ROI and run data around unexpected onset, or the difference
between post and pre unexpected onset, or both.
Required args:
- sess (Session) : session object
- stimpar (StimPar): named tuple containing stimulus parameters
Optional args:
- ran_s (dict, list or num): number of frames to take before and after
unexpected for each datatype (ROI, run,
pupil) (in sec).
If dictionary, expected keys are:
"pup_pre", "pup_post",
("roi_pre", "roi_post"),
("run_pre", "run_post"),
If list, should be structured as
[pre, post] and the same values will be
used for all datatypes.
If num, the same value will be used
for all keys.
If None, the values are taken from the
stimpar pre and post attributes.
default: None
- datatype (str) : type of data to include with pupil data,
"roi", "run" or "both"
default: "roi"
- returns (str) : type of data to return (data around
unexpected, difference between post and pre
unexpected)
default: "diff"
- fluor (str) : if "dff", dF/F is used, if "raw", ROI
traces
default: "dff"
- stats (str) : measure on which to take the pre and post
unexpected difference: either mean ("mean")
or median ("median")
default: "mean"
- rem_bad (bool) : if True, removes ROIs with NaN/Inf values
anywhere in session and running array with
NaNs linearly interpolated is used. If
False, NaNs are ignored in calculating
statistics for the ROI and running data
(always ignored for pupil data)
default: True
- scale (bool) : if True, data is scaled
default: False
- first_unexp (bool) : if True, only the first of consecutive
unexpecteds are retained
default: True
- trans_all (bool) : if True, only ROIs with transients are
retained
default: False
Returns:
if datatype == "data" or "both":
- datasets (list): list of 2-3D data arrays, structured as
datatype (pupil, (ROI), (running)) x
[trial x frames (x ROI)]
elif datatype == "diff" or "both":
- diffs (list) : list of 1-2D data difference arrays, structured as
datatype (pupil, (ROI), (running)) x
[trial (x ROI)]
"""
stim = sess.get_stim(stimpar.stimtype)
# initialize ran_s dictionary if needed
if ran_s is None:
ran_s = [stimpar.pre, stimpar.post]
ran_s = get_ran_s(ran_s, datatype)
if first_unexp:
unexp_segs = stim.get_segs_by_criteria(
visflow_dir=stimpar.visflow_dir,
visflow_size=stimpar.visflow_size,
gabk=stimpar.gabk,
unexp=1,
remconsec=True,
by="seg")
if stimpar.stimtype == "gabors":
unexp_segs = [seg + stimpar.gabfr for seg in unexp_segs]
else:
unexp_segs = stim.get_segs_by_criteria(
visflow_dir=stimpar.visflow_dir,
visflow_size=stimpar.visflow_size,
gabk=stimpar.gabk,
gabfr=stimpar.gabfr,
unexp=1,
remconsec=False,
by="seg")
unexp_twopfr = stim.get_fr_by_seg(unexp_segs, start=True,
fr_type="twop")["start_frame_twop"]
unexp_stimfr = stim.get_fr_by_seg(unexp_segs, start=True,
fr_type="stim")["start_frame_stim"]
# get data dataframes
pup_data = gen_util.reshape_df_data(stim.get_pup_diam_data(
unexp_twopfr,
ran_s["pup_pre"],
ran_s["pup_post"],
rem_bad=rem_bad,
scale=scale)["pup_diam"],
squeeze_cols=True)
datasets = [pup_data]
datanames = ["pup"]
if datatype in ["roi", "both"]:
# ROI x trial x fr
roi_data = gen_util.reshape_df_data(stim.get_roi_data(
unexp_twopfr,
ran_s["roi_pre"],
ran_s["roi_post"],
fluor=fluor,
integ=False,
rem_bad=rem_bad,
scale=scale,
transients=trans_all)["roi_traces"],
squeeze_cols=True)
datasets.append(roi_data.transpose([1, 2, 0])) # ROIs last
datanames.append("roi")
if datatype in ["run", "both"]:
run_data = gen_util.reshape_df_data(stim.get_run_data(
unexp_stimfr,
ran_s["run_pre"],
ran_s["run_post"],
rem_bad=rem_bad,
scale=scale),
squeeze_cols=True)
datasets.append(run_data)
datanames.append("run")
if rem_bad:
nanpolgen = None
else:
nanpolgen = "omit"
if returns in ["diff", "both"]:
for key in ran_s.keys():
if "pre" in key and ran_s[key] == 0:
raise ValueError(
"Cannot set pre to 0 if returns is 'diff' or 'both'.")
# get avg for first and second halves
diffs = []
for dataset, name in zip(datasets, datanames):
if name == "pup":
nanpol = "omit"
else:
nanpol = nanpolgen
n_fr = dataset.shape[1]
pre_s = ran_s[f"{name}_pre"]
post_s = ran_s[f"{name}_post"]
split = int(np.round(pre_s / (pre_s + post_s) *
n_fr)) # find 0 mark
pre = math_util.mean_med(dataset[:, :split], stats, 1, nanpol)
post = math_util.mean_med(dataset[:, split:], stats, 1, nanpol)
diffs.append(post - pre)
if returns == "data":
return datasets
elif returns == "diff":
return diffs
elif returns == "both":
return datasets, diffs
else:
gen_util.accepted_values_error("returns", returns,
["data", "diff", "both"])