def pilot_visflow_omit(visflow_dir, visflow_size):
"""
pilot_visflow_omit(visflow_dir, visflow_size)
Returns numbers of pilot mice to omit based on visual flow direction and
square size values to include.
Required args:
- visflow_dir (str or list) : visual flow direction values
("right", "left")
- visflow_size (int or list): visual flow square size values
(128, 256, [128, 256])
Returns:
- omit_mice (list): list IDs of mice to omit
"""
visflow_dir = gen_util.list_if_not(visflow_dir)
visflow_size = gen_util.list_if_not(visflow_size)
omit_mice = []
right_incl = len(list(filter(lambda x: "right" in x, visflow_dir)))
left_incl = len(list(filter(lambda x: "left" in x, visflow_dir)))
if not right_incl:
# mouse 4 only got visflow_dir="right"
omit_mice.extend([4])
if 128 not in visflow_size:
# mouse 3 only got visflow_dir="left" with visflow_size=128
omit_mice.extend([3])
elif not left_incl and 256 not in visflow_size:
# mouse 3 only got visflow_dir="right" with visflow_size=256
omit_mice.extend([3])
return omit_mice
def all_omit(stimtype="gabors",
runtype="prod",
visflow_dir="both",
visflow_size=128,
gabk=16):
"""
all_omit()
Returns list of mice and sessions to omit, based on analysis parameters
(runtype, gabk, visflow_dir and visflow_size) and throws an error if the
parameter combination requested does not occur in the dataset.
Required args:
- stimtype (str) : stimulus to analyse ("visflow", "gabors")
- runtype (str) : runtype ("pilot", "prod")
- visflow_dir (str or list) : visual flow direction values
("right", "left")
- visflow_size (int or list): visual flow square size values to include
(128, 256, [128, 256])
- gabk (int or list) : gabor kappa values to include
(4, 16 or [4, 16])
Returns:
- omit_sess (list): sessions to omit
- omit_mice (list): mice to omit
"""
omit_sess = []
omit_mice = []
if runtype == "pilot":
omit_sess = [714893802] # no data
if stimtype == "gabors":
omit_mice = pilot_gab_omit(gabk)
elif stimtype == "visflow":
omit_mice = pilot_visflow_omit(visflow_dir, visflow_size)
elif runtype == "prod":
if stimtype == "gabors":
if 16 not in gen_util.list_if_not(gabk):
logger.warning("The production data only includes gabor "
"stimuli with kappa=16")
omit_mice = list(range(1, 9)) # all
elif stimtype == "visflow":
if 128 not in gen_util.list_if_not(visflow_size):
logger.warning("The production data only includes visual flow "
"stimuli with square size=128")
omit_mice = list(range(1, 9)) # all
return omit_sess, omit_mice
def get_pupil_data_h5_path(maindir, check_name=True):
"""
get_pupil_data_h5_path(maindir)
Returns path to pupil data h5 file.
Required args:
- maindir (Path): path of the main data directory
Optional args:
- check_name (bool): if True, pupil file name is checked to have the
expected structure.
default: True
Returns:
- pup_data_h5 (Path or list): full path name(s) of the pupil h5 file
"""
name_part = "*pupil_data_df.h5"
pupil_data_files = glob.glob(str(Path(maindir, name_part)))
if len(pupil_data_files) == 1:
pup_data_h5 = Path(pupil_data_files[0])
elif len(pupil_data_files) > 1:
pup_data_h5 = [Path(data_file) for data_file in pupil_data_files]
else:
pup_data_h5 = "none"
if check_name:
for h5_name in gen_util.list_if_not(pup_data_h5):
if h5_name != "none":
get_check_pupil_data_h5_name(h5_name)
return pup_data_h5
def init_quantpar(n_quants=4, qu_idx="all"):
"""
init_quantpar()
Returns a QuantPar namedtuple with the inputs arguments as named attributes.
Optional args:
- n_quants (int) : nbr of quantiles
default: 4
- qu_idx (list) : indices of quantiles used in analyses
default: "all"
Returns:
- quantpar (QuantPar namedtuple): QuantPar with input arguments as
attributes
"""
if qu_idx == "all":
qu_idx = list(range(n_quants))
qu_idx = gen_util.list_if_not(qu_idx)
quant_pars = [n_quants, qu_idx]
quant_keys = ["n_quants", "qu_idx"]
QuantPar = namedtuple("QuantPar", quant_keys)
quantpar = QuantPar(*quant_pars)
return quantpar
def reformat_sess_n(sess_n):
"""
reformat_sess_n(sess_n)
Returns reformatted sess_n argument, converting ranges to lists.
Required args:
- sess_n (str):
session number or range (e.g., "1-1", "all")
Returns:
- sess_n (str or list):
session number or range (e.g., [1, 2, 3], "all")
"""
if "-" in str(sess_n):
vals = str(sess_n).split("-")
if len(vals) != 2:
raise ValueError("If sess_n is a range, must have format 1-3.")
st = int(vals[0])
end = int(vals[1]) + 1
sess_n = list(range(st, end))
elif sess_n not in ["any", "all"]:
sess_n = gen_util.list_if_not(sess_n)
return sess_n
def depth_vals(plane, line):
"""
depth_vals(plane, line)
Returns depth values corresponding to a specified plane.
Required args:
- plane (str): plane (e.g., "dend", "soma", "any")
- line (str) : line (e.g., "L23", "L5", "any")
Returns:
- depths (str or list): depths corresponding to plane and line or "any"
"""
if plane in ["any", "all"] and line in ["any", "all"]:
return "any"
depth_dict = {
"L23_dend": [50, 75],
"L23_soma": [175],
"L5_dend": [20],
"L5_soma": [375]
}
all_planes = ["dend", "soma"]
if plane in ["any", "all"]:
planes = all_planes
else:
planes = gen_util.list_if_not(plane)
all_lines = ["L23", "L5"]
if line in ["any", "all"]:
lines = all_lines
else:
lines = gen_util.list_if_not(line)
depths = []
for plane in planes:
if plane not in all_planes:
allowed_planes = all_planes + ["any", "all"]
gen_util.accepted_values_error("plane", plane, allowed_planes)
for line in lines:
if line not in all_lines:
allowed_lines = all_lines + ["any", "all"]
gen_util.accepted_values_error("line", line, allowed_lines)
depths.extend(depth_dict[f"{line}_{plane}"])
return depths
def load_stimulus_images_nwb(sess_files, template_name="gabors", frame_ns=[0]):
"""
load_stimulus_images_nwb(sess_files)
Returns the stimulus images for the requested template, and frames.
Required args:
- sess_files (Path): full path names of the session files
Optional args:
- template_name (str): name of the stimulus template
default: "gabors"
- frame_ns (list) : frame numbers (must be unique and in increasing
order)
default: frame_ns
Returns:
- stim_images (list): list of stimulus images (grayscale)
(height x width x channel (1))
"""
stim_file = sess_file_util.select_nwb_sess_path(sess_files, stim=True)
frame_ns = gen_util.list_if_not(frame_ns)
unique_bool = (len(np.unique(frame_ns)) == len(frame_ns))
increasing_bool = (np.sort(frame_ns) == np.asarray(frame_ns)).all()
if not (unique_bool and increasing_bool):
raise ValueError(
"frame_ns must be provided in strictly increasing order, and "
"comprise only unique values."
)
with pynwb.NWBHDF5IO(str(stim_file), "r") as f:
nwbfile_in = f.read()
try:
stim_template = nwbfile_in.get_stimulus_template(template_name)
n_frames = stim_template.data.shape[0]
if max(frame_ns) > n_frames - 1:
raise ValueError(
"Some of the frames requested go beyond the number of "
"frames recorded for this template."
)
# loading the images can be slow
template_images = stim_template.data[frame_ns]
except KeyError as err:
raise KeyError(
f"Could not find frames for '{template_name}' stimulus "
f"template in {stim_file} due to: {err}"
)
template_images = list(template_images)
return template_images
def main(args):
"""
main(args)
Runs analyses with parser arguments.
Required args:
- args (dict): parser argument dictionary
"""
# set logger to the specified level
logger_util.set_level(level=args.log_level)
# args.device = gen_util.get_device(args.cuda)
args.fontdir = DEFAULT_FONTDIR if DEFAULT_FONTDIR.is_dir() else None
if args.dict_path != "":
plot_dicts.plot_from_dicts(Path(args.dict_path),
source="glm",
plt_bkend=args.plt_bkend,
fontdir=args.fontdir,
parallel=args.parallel,
datetime=not (args.no_datetime),
overwrite=args.overwrite)
else:
if args.datadir is None:
args.datadir = DEFAULT_DATADIR
else:
args.datadir = Path(args.datadir)
mouse_df = DEFAULT_MOUSE_DF_PATH
args = reformat_args(args)
# get numbers of sessions to analyse
if args.sess_n == "all":
all_sess_ns = sess_gen_util.get_sess_vals(mouse_df,
"sess_n",
runtype=args.runtype,
plane=args.plane,
line=args.line,
min_rois=args.min_rois,
pass_fail=args.pass_fail,
incl=args.incl,
omit_sess=args.omit_sess,
omit_mice=args.omit_mice)
else:
all_sess_ns = gen_util.list_if_not(args.sess_n)
for sess_n in all_sess_ns:
analys_pars = prep_analyses(sess_n, args, mouse_df)
analyses_parallel = bool(args.parallel * (not args.debug))
run_analyses(*analys_pars,
analyses=args.analyses,
parallel=analyses_parallel)
def gabfr_letters(gabfr, unexp="any"):
"""
gabfr_letters(gabfr)
Returns the letters corresponding to the Gabor frame numbers
(0, 1, 2, 3, 4).
Required args:
- gabfr (int or list): gabor frame number(s)
Optional args:
- unexp (str, int or list): unexpected values for all or each gabor
frame number. If only value, applies to all.
(0, 1 or "any")
default: "any"
Returns:
- gab_letts (str or list): gabor frame letter(s)
"""
if not isinstance(gabfr, list):
gabfr_list = False
gabfr = [gabfr]
else:
gabfr_list = True
unexp = gen_util.list_if_not(unexp)
if len(unexp) == 1:
unexp = unexp * len(gabfr)
else:
if len(gabfr) != len(unexp):
raise ValueError("If passing more than one unexp value, must "
"pass as many as gabfr.")
if isinstance(gabfr, int):
if min(gabfr) < 0 or max(gabfr) > 4:
raise ValueError(
"Gabor frames are only between 0 and 4, inclusively.")
all_gabfr = ["A", "B", "C", "D/U", "G"]
gab_letts = []
for i, gf in enumerate(gabfr):
if gf == 3 and unexp[i] != "any":
gab_letts.append(all_gabfr[gf][-unexp[i]]) # D or U is retained
else:
gab_letts.append(all_gabfr[gf])
if not gabfr_list:
gab_letts = gab_letts[0]
return gab_letts
def get_mouseid_sessid_nwb(nwb_files):
"""
get_mouseid_sessid_nwb(nwb_files)
Returns the mouse ID and session ID retrieve from NWB files. If several
files are passed, they are expected to be for the same session
(with identical mouse ID and session IDs).
Required args:
- nwb_files (Path or list): path(s) to the NWB file(s) for a single
session
Returns:
- mouseid (str): mouse ID (Allen)
- sessid (str) : session ID (Allen)
"""
nwb_files = gen_util.list_if_not(nwb_files)
mouseid, sessid = None, None
for nwb_file in nwb_files:
with pynwb.NWBHDF5IO(str(nwb_file), "r") as f:
nwbfile_in = f.read()
new_mouseid = nwbfile_in.subject.subject_id
if mouseid is None:
mouseid = new_mouseid
elif mouseid != new_mouseid:
nwb_filenames = [str(filename) for filename in nwb_files]
raise RuntimeError(
"Mouse IDs for different NWB files for the same session "
f"do not match: {', '.join(nwb_filenames)}."
)
new_sessid = nwbfile_in.identifier
if sessid is None:
sessid = new_sessid
elif sessid != new_sessid:
nwb_filenames = [str(filename) for filename in nwb_files]
raise RuntimeError(
"Session IDs for different NWB files for the same session "
f"do not match: {', '.join(nwb_filenames)}."
)
return mouseid, sessid
def size_par_str(size, str_type="file"):
"""
size_par_str(size)
Returns a string with stimulus type, as well as size parameters
(e.g., 128, 256), unless only 128 is passed.
Required args:
- size (int or list): visual flow square size parameter
Optional args:
- str_type (str) : use of output str, i.e., for a filename ("file") or
to print the info to console ("print")
default: "file"
Returns:
- pars (str): string containing stim type (visual flow) and size,
unless only 128 is passed.
"""
size = gen_util.list_if_not(size)
size = [int(s) for s in size]
if str_type == "file":
pars = "visflow"
elif str_type == "print":
pars = "vis. flow"
else:
gen_util.accepted_values_error("str_type", str_type, ["print", "file"])
if 256 in size:
if len(size) > 1:
if str_type == "file":
pars = f"{pars}_both_siz"
elif str_type == "print":
pars = f"{pars} (both square sizes)"
else:
if str_type == "file":
pars = f"{pars}{size[0]}"
elif str_type == "print":
pars = f"{pars} ({size[0]})"
return pars
def gabk_par_str(gabk, str_type="file"):
"""
gabk_par_str(gabk)
Returns a string with stim type, as well as kappa parameters
(e.g., 4, 16), unless only 16 is passed.
Required args:
- gabk (int or list): gabor kappa parameter
Optional args:
- str_type (str) : use of output str, i.e., for a filename ("file") or
to print the info to console ("print")
default: "file"
Returns:
- pars (str): string containing stim type (gabors) and kappa,
unless only 16 is passed.
"""
gabk = gen_util.list_if_not(gabk)
gabk = [int(g) for g in gabk]
if str_type == "file":
pars = "gab"
elif str_type == "print":
pars = "gabors"
else:
gen_util.accepted_values_error("str_type", str_type, ["print", "file"])
if 4 in gabk:
if len(gabk) > 1:
if str_type == "file":
pars = f"{pars}_both"
elif str_type == "print":
pars = f"{pars} (both)"
else:
if str_type == "file":
pars = f"{pars}{gabk[0]}"
elif str_type == "print":
pars = f"{pars} ({gabk[0]})"
return pars
def dir_par_str(direc, str_type="file"):
"""
dir_par_str(direc)
Returns a string with stim type, as well as direction parameters
(e.g., "right", "left"), unless both possible values are passed.
Required args:
- direc (str or list): visual flow direction parameter
Optional args:
- str_type (str) : use of output str, i.e., for a filename ("file") or
to print the info to console ("print")
default: "file"
Returns:
- pars (str): string containing stim type (visual flow) and direction,
unless both possible values are passed.
"""
direc = gen_util.list_if_not(direc)
if str_type == "file":
pars = "visflow"
elif str_type == "print":
pars = "vis. flow"
else:
gen_util.accepted_values_error("str_type", str_type, ["print", "file"])
if len(direc) == 1 and direc[0] != "both":
direc = direc[0]
direc_detailed = sess_gen_util.get_visflow_screen_mouse_direc(direc)
if str_type == "file":
direc = direc_detailed[:5].strip(" ") # get left/right
pars = f"{pars}_{direc}"
elif str_type == "print":
direc_detailed = direc_detailed.replace(" (",
", ").replace(")", "")
pars = f"{pars} ({direc_detailed})"
return pars
def pilot_gab_omit(gabk):
"""
pilot_gab_omit(gabk)
Returns numbers of pilot mice to omit based on gabor kappa values to
include.
Required args:
- gabk (int or list): gabor kappa values (4, 16, [4, 16])
Returns:
- omit_mice (list): list IDs of mice to omit
"""
gabk = gen_util.list_if_not(gabk)
if 4 not in gabk:
omit_mice = [3] # mouse 3 only got K=4
elif 16 not in gabk:
omit_mice = [4] # mouse 4 only got K=16
else:
omit_mice = []
return omit_mice
def get_sess_info(sessions,
fluor="dff",
add_none=False,
incl_roi=True,
return_df=False,
rem_bad=False):
"""
get_sess_info(sessions)
Puts information from all sessions into a dictionary. Optionally allows
None sessions.
Required args:
- sessions (list): ordered list of Session objects
Optional args:
- fluor (str) : specifies which bad_rois to include (for "dff" or
"raw")
default: "dff"
- add_none (bool): if True, None sessions are allowed and all values
are filled with None
default: False
- incl_roi (bool): if True, ROI information is included
default: True
- rem_bad (bool) : if True, NaN/Inf ROI information is removed, and the
number of subtracted from nrois
default: False
Returns:
- sess_info (dict or df): dictionary or dataframe containing
information from each session, as lists or
in dataframe rows, under the following keys
or columns
"mouse_ns", "mouseids", "sess_ns", "sessids", "lines", "planes"
if datatype == "roi":
"nrois", "twop_fps"
if not rem_bad:
"bad_rois_{}" (depending on fluor)
"""
if return_df and add_none:
raise ValueError("'add_none' cannot be True if 'return_df' is True.")
if add_none and set(sessions) == {None}:
logger.info("All None value sessions.")
sess_info = dict()
keys = ["mouse_ns", "mouseids", "sess_ns", "sessids", "lines", "planes"]
if incl_roi:
keys.extend(["nrois", "twop_fps"])
if not rem_bad:
keys.extend([f"bad_rois_{fluor}"])
for key in keys:
sess_info[key] = []
sessions = gen_util.list_if_not(sessions)
for _, sess in enumerate(sessions):
if sess is None:
if add_none:
for key in keys:
sess_info[key].append(None)
else:
raise RuntimeError("None sessions not allowed.")
else:
sess_info["mouse_ns"].append(sess.mouse_n)
sess_info["mouseids"].append(sess.mouseid)
sess_info["sess_ns"].append(sess.sess_n)
sess_info["sessids"].append(sess.sessid)
sess_info["lines"].append(sess.line)
sess_info["planes"].append(sess.plane)
if not incl_roi:
continue
nrois = sess.get_nrois(rem_bad=rem_bad, fluor=fluor)
if not rem_bad:
sess_info[f"bad_rois_{fluor}"].append(sess.get_bad_rois(fluor))
sess_info["nrois"].append(nrois)
sess_info["twop_fps"].append(sess.twop_fps)
if return_df:
sess_info = pd.DataFrame.from_dict(sess_info)
return sess_info
def init_sessions(sessids,
datadir,
mouse_df,
runtype="prod",
full_table=True,
fluor="dff",
dend="extr",
omit=False,
roi=True,
run=False,
pupil=False,
temp_log=None):
"""
init_sessions(sessids, datadir)
Creates list of Session objects for each session ID passed.
Required args:
- sessids (int or list): ID or list of IDs of sessions
- datadir (Path) : directory where sessions are stored
- mouse_df (Path) : path name of dataframe containing information
on each session
Optional args:
- runtype (str) : the type of run, either "pilot" or "prod"
default: "prod"
- full_table (bool): if True, the full stimulus dataframe is loaded
(with all the visual flow square positions and
individual Gabor patch orientations).
default: True
- dend (str) : type of dendrites to use ("allen" or "extr")
default: "extr"
- omit (bool) : if True, dendritic sessions with the wrong type of
dendrite are omitted
default: False
- roi (bool) : if True, ROI data is loaded into sessions
default: True
- run (bool) : if True, running data is loaded into sessions
default: False
- pupil (bool) : if True, pupil data is loaded into session and
only sessions with pupil data are included
default: False
- temp_log (bool) : temporary log level to set logger to. If None,
logger is left at current level.
default: None
Returns:
- sessions (list): list of Session objects
"""
from analysis import session
with logger_util.TempChangeLogLevel(level=temp_log):
sessions = []
sessids = gen_util.list_if_not(sessids)
for sessid in sessids:
logger.info(f"Creating session {sessid}...",
extra={"spacing": "\n"})
# creates a session object to work with
sess = session.Session(sessid,
datadir,
runtype=runtype,
mouse_df=mouse_df)
# extracts necessary info for analysis
sess.extract_info(full_table=full_table,
fluor=fluor,
dend=dend,
roi=roi,
run=run)
if omit and sess.plane == "dend" and sess.dend != dend:
logger.info(
f"Omitting session {sessid} ({dend} dendrites not found).")
continue
if pupil:
if sess.pup_data_available:
sess.load_pup_data()
else:
logger.info(
f"Omitting session {sessid} as no pupil data was found."
)
continue
logger.info(f"Finished creating session {sessid}.")
sessions.append(sess)
return sessions
def select_nwb_sess_path(sess_files,
ophys=False,
behav=False,
stim=False,
warn_multiple=False):
"""
select_nwb_sess_path(sess_files)
Returns an NWB session data path name, selected from a list according to
the specified criteria.
Required arguments:
- sess_files (list): full path names of the session files
Optional arguments
- ophys (bool) : if True, only session files with optical
physiology data are retained
default: False
- behav (bool) : if True, only session files with behaviour
data are retained
default: False
- stim (bool) : if True, only session files with stimulus
images are retained
default: False
- warn_multiple (bool): if True, a warning if thrown if multiple
matching session files are found
default: False
Returns:
- sess_file (Path): full path name of the selected session file
"""
sess_files = gen_util.list_if_not(sess_files)
criterion_dict = {
"ophys": [ophys, "optical physiology"],
"behavior": [behav, "behavioral"],
"image": [stim, "stimulus template"],
}
data_names = []
for data_str, [data_bool, data_name] in criterion_dict.items():
if data_bool:
sess_files = [
sess_file for sess_file in sess_files
if data_str in str(sess_file)
]
data_names.append(data_name)
tog_str = "" if len(data_names) < 2 else " together"
data_names = ", and ".join(data_names).lower()
if len(sess_files) == 0:
raise RuntimeError(
f"{data_names.capitalize()} data not included{tog_str} in this "
"session's NWB files.")
sess_file = sess_files[0]
if len(sess_files) > 1 and warn_multiple:
data_names_str = f" with {data_names} data" if len(data_names) else ""
warnings.warn(f"Several session files{data_names_str} found{tog_str}. "
f"Using the first listed: {sess_file}.")
return sess_file
def get_mouse_df_vals(mouse_df,
returnlab,
mouse_n,
sessid,
sess_n,
runtype,
depth,
pass_fail="P",
incl="yes",
all_files=1,
any_files=1,
min_rois=1,
unique=False,
sort=False):
"""
get_mouse_df_vals(mouse_df, returnlab, mouse_n, sessid, sess_n, runtype,
depth)
Returns values from mouse dataframe under a specified label or labels that
fit the criteria.
Required args:
- mouse_df (pandas df) : dataframe containing parameters for each
session.
- returnlab (str or list) : labels for which to return values
- mouse_n (int, str or list): mouse id value(s) of interest
- sessid (int or list) : session id value(s) of interest
- sess_n (int, str or list) : session number(s) of interest
- runtype (str or list) : runtype value(s) of interest
- depth (str or list) : depth value(s) of interest (20, 50, 75,
175, 375)
Optional args:
- pass_fail (str or list) : pass/fail values of interest ("P", "F")
default: "P"
- incl (str) : which sessions to include ("yes", "no",
"any")
default: "yes"
- all_files (int, str or list): all_files values of interest (0, 1)
default: 1
- any_files (int, str or list): any_files values of interest (0, 1)
default: 1
- min_rois (int) : min number of ROIs
default: 1
- unique (bool) : whether to return a list of values
without duplicates
default: False
- sort (bool) : whether to sort output values
default: False
Returns:
- df_vals (list): list of values under the specified labels that fit
the criteria (or list of lists in multiple labels)
"""
# make sure all of these labels are lists
all_labs = [
mouse_n, sessid, sess_n, runtype, depth, pass_fail, incl, all_files,
any_files
]
all_cols = [
"mouse_n", "sessid", "sess_n", "runtype", "depth", "pass_fail", "incl",
"all_files", "any_files"
]
for i in range(len(all_labs)):
all_labs[i] = gen_util.list_if_not(all_labs[i])
col_dtype = mouse_df[all_cols[i]].dtype
if col_dtype in [bool, int, float]:
all_labs[i] = gen_util.conv_types(all_labs[i])
[
mouse_n, sessid, sess_n, runtype, depth, pass_fail, incl, all_files,
any_files
] = all_labs
df_rows = mouse_df.loc[(mouse_df["mouse_n"].isin(mouse_n))
& (mouse_df["sessid"].isin(sessid)) &
(mouse_df["sess_n"].isin(sess_n)) &
(mouse_df["runtype"].isin(runtype)) &
(mouse_df["depth"].isin(depth)) &
(mouse_df["pass_fail"].isin(pass_fail)) &
(mouse_df["incl"].isin(incl)) &
(mouse_df["all_files"].isin(all_files)) &
(mouse_df["any_files"].isin(any_files)) &
(mouse_df["nrois"].astype(int) >= min_rois)]
returnlab = gen_util.list_if_not(returnlab)
if len(returnlab) != 1 and (sort or unique):
logger.warning("Sorted and unique will be set to False as multiple "
"labels are requested.")
sort = False
unique = False
df_vals = []
for lab in returnlab:
vals = df_rows[lab].tolist()
if unique:
vals = list(set(vals))
if sort:
vals = sorted(vals)
df_vals.append(vals)
df_vals = gen_util.delist_if_not(df_vals)
return df_vals
def get_check_sess_df(sessions, sess_df=None, analyspar=None, roi=True):
"""
get_check_sess_df(sessions)
Checks a dataframe against existing sessions (that they match and are in
the same order), or returns a dataframe with session information if sess_df
is None.
Required args:
- sessions (list):
Session objects
Optional args:
- sess_df (pd.DataFrame):
dataframe containing session information (see keys under Returns)
default: None
- analyspar (AnalysPar):
named tuple containing analysis parameters, used if sess_df is None
- roi (bool):
if True, ROI data is included in sess_df, used if sess_df is None
Returns:
- sess_df (pd.DataFrame):
dataframe containing session information under the following keys:
"mouse_ns", "mouseids", "sess_ns", "sessids", "lines", "planes"
if datatype == "roi":
"nrois", "twop_fps"
if not rem_bad:
"bad_rois_{}" (depending on fluor)
"""
sessions = gen_util.list_if_not(sessions)
if sess_df is None:
roi_kwargs = dict()
if analyspar is None and roi:
raise ValueError("If sess_df is None, must pass analyspar.")
elif analyspar is not None:
roi_kwargs["fluor"] = analyspar.fluor
roi_kwargs["rem_bad"] = analyspar.rem_bad
sess_df = sess_gen_util.get_sess_info(
sessions, incl_roi=roi, return_df=True, **roi_kwargs
)
else:
if len(sess_df) != len(sessions):
raise ValueError(
"'sess_df' should have as many rows as 'sessions'.")
# check order
sessids = np.asarray([sess.sessid for sess in sessions]).astype(int)
sess_df_sessids = sess_df.sessids.to_numpy().astype(int)
if len(sessids) != len(sess_df_sessids):
raise ValueError("'sess_df' is not the same length at 'sessions'.")
elif (np.sort(sessids) != np.sort(sess_df_sessids)).any():
raise ValueError("Sessions do not match ids in 'sess_df'.")
elif (sessids != sess_df_sessids).any():
raise ValueError("Sessions do not appear in order in 'sess_df'.")
return sess_df
def calc_tune_curvs(sess,
analyspar,
stimpar,
nrois="all",
ngabs="all",
grp2="unexp",
comb_gabs=True,
vm_estim=False,
collapse=True,
parallel=True):
"""
calc_tune_curvs(sess, analyspar, stimpar)
Returns orientations and corresponding fluorescence levels for the sessions
of interest.
Required args:
- sess (Session) : session
- analyspar (AnalysPar): named tuple containing analysis parameters
- stimpar (StimPar) : named tuple containing stimulus parameters
Optional args:
- nrois (int) : number of ROIs to include in analysis
default: "all"
- ngabs (int) : number of gabors to include in analysis (set to 1
if comb_gabs, as all gabors are combined)
default: "all"
- grp2 (str) : second group: either unexp, exp or rand (random
subsample of exp, the size of unexp)
default: "unexp"
- comb_gabs (bool): if True, all gabors have been combined for
gab_oris and roi_data
default: False
- vm_estim (bool) : if True, analysis is run using a von Mises tuning
curve estimation method
- collapse (bool) : if True, opposite orientations in the 0 to 360
range are collapsed to the 0 to 180 range
- parallel (bool) : if True, some of the analysis is parallelized
across CPU cores
Returns:
- tc_oris (list) : list of orientation values corresponding to the
tc_data:
unexp x gabor (1 if comb_gabs) x oris
- tc_data (list) : list of mean integrated fluorescence data per
orientation, for each ROI, structured as
ROI x unexp x gabor (1 if comb_gabs)
x oris
- tc_nseqs (list) : number of sequences per unexp
if vm_estim, also:
- tc_vm_pars (list) : nested list of Von Mises parameters for each ROI:
ROI x unexp x gabor (1 if comb_gabs) x par
- tc_vm_mean (list) : nested list of mean Von Mises means for each ROI,
not weighted by kappa value or weighted (if not
comb_gabs) (in rad):
ROI x unexp x kappa weighted (False, (True))
- tc_hist_pars (list): parameters used to convert tc_data to histogram
values (sub, mult) used in Von Mises parameter
estimation, structured as:
ROI x unexp x gabor (1 if comb_gabs) x
param (sub, mult)
"""
if sess.only_tracked_rois != analyspar.tracked:
raise RuntimeError(
"sess.only_tracked_rois should match analyspar.tracked.")
gabfrs = gen_util.list_if_not(stimpar.gabfr)
if len(gabfrs) == 1:
gabfrs = gabfrs * 2
if grp2 == "unexp":
unexps = [0, 1]
elif grp2 in ["exp", "rand"]:
unexps = [0, 0]
else:
gen_util.accepted_values_error("grp2", grp2, ["unexp", "exp", "rand"])
stim = sess.get_stim(stimpar.stimtype)
nrois_tot = sess.get_nrois(analyspar.rem_bad, analyspar.fluor)
ngabs_tot = stim.n_patches
if nrois == "all":
sess_nrois = nrois_tot
else:
sess_nrois = np.min([nrois_tot, nrois])
if ngabs == "all":
sess_ngabs = stim.n_patches
else:
sess_ngabs = np.min([stim.n_patches, ngabs])
tc_data, tc_oris, tc_nseqs = [], [], []
# estimate tuning curves by fitting a Von Mises distribution
if vm_estim:
tc_vm_pars, tc_vm_mean, tc_hist_pars = [], [], []
for i, (gf, e) in enumerate(zip(gabfrs, unexps)):
# get segments
segs = stim.get_segs_by_criteria(gabfr=gf,
visflow_dir=stimpar.visflow_dir,
visflow_size=stimpar.visflow_size,
gabk=stimpar.gabk,
unexp=e,
by="seg")
if grp2 == "rand" and i == 1:
n_segs = len(
stim.get_segs_by_criteria(gabfr=gf,
visflow_dir=stimpar.visflow_dir,
visflow_size=stimpar.visflow_size,
gabk=stimpar.gabk,
unexp=1,
by="seg"))
np.random.shuffle(segs)
segs = sorted(segs[:n_segs])
tc_nseqs.append(len(segs))
twopfr = stim.get_fr_by_seg(segs, start=True,
fr_type="twop")["start_frame_twop"]
# ROI x seq
roi_data = gen_util.reshape_df_data(stim.get_roi_data(
twopfr,
stimpar.pre,
stimpar.post,
analyspar.fluor,
integ=True,
rem_bad=analyspar.rem_bad,
scale=analyspar.scale)["roi_traces"],
squeeze_cols=True)[:sess_nrois]
# gab x seq
gab_oris = gen_util.reshape_df_data(stim.get_stim_par_by_seg(
segs, pos=False, ori=True, size=False),
squeeze_cols=True).T
if collapse:
gab_oris = collapse_dir(gab_oris)
if comb_gabs:
ngabs = 1
gab_oris = gab_oris.reshape([1, -1])
roi_data = np.tile(roi_data, [1, ngabs_tot])
tc_oris.append(gab_oris.tolist())
tc_data.append(roi_data.tolist())
# estimate tuning curves by fitting a Von Mises distribution
if vm_estim:
[
gab_tc_oris, gab_tc_data, gab_vm_pars, gab_vm_mean,
gab_hist_pars
] = tune_curv_estims(gab_oris,
roi_data,
ngabs_tot,
sess_nrois,
sess_ngabs,
comb_gabs,
collapse=False,
parallel=parallel)
tc_oris[i] = gab_tc_oris
tc_data[i] = gab_tc_data
tc_vm_pars.append(gab_vm_pars)
tc_vm_mean.append(gab_vm_mean)
tc_hist_pars.append(gab_hist_pars)
if vm_estim:
return tc_oris, tc_data, tc_nseqs, tc_vm_pars, tc_vm_mean, tc_hist_pars
else:
return tc_oris, tc_data, tc_nseqs
def sep_grps(sign_rois, nrois, grps="all", tails="2", add_exp=False):
"""
sep_grps(sign_rois, nrois)
Separate ROIs into groups based on whether their first/last quantile was
significant in a specific tail.
Required args:
- sign_rois (nested list): list of significant ROIs, structured as:
quantile (x tail)
- nrois (int) : total number of ROIs in data (signif or not)
Optional args:
- grps (str or list): set of groups or list of sets of groups to
return, e.g., "all", "change", "no_change",
"reduc", "incr"
default: "all"
- tails (str) : tail(s) used in analysis: "hi", "lo" or 2
default: 2
- add_exp (bool) : if True, group of ROIs showing no significance in
either is included in the groups returned
default: False
Returns:
- roi_grps (list) : lists structured as follows:
if grp parameter includes only one set,
ROIs per roi group
otherwise: sets x roi grps
numbers included in the group
- grp_names (list) : if grp parameter includes only one set, list of
names of roi grps (order preserved)
otherwise: list of sublists per set, each
containing names of roi grps per set
"""
grps = gen_util.list_if_not(grps)
# get ROI numbers for each group
if tails in ["hi", "lo"]:
# sign_rois[first/last]
all_rois = list(range(nrois))
unexp_unexp = list(set(sign_rois[0]) & set(sign_rois[1]))
unexp_exp = list(set(sign_rois[0]) - set(sign_rois[1]))
exp_unexp = list(set(sign_rois[1]) - set(sign_rois[0]))
exp_exp = list(
set(all_rois) - set(unexp_unexp) - set(unexp_exp) - set(exp_unexp))
# to store stats
roi_grps = [unexp_unexp, unexp_exp, exp_unexp, exp_exp]
grp_names = ["unexp_unexp", "unexp_exp", "exp_unexp", "exp_exp"]
exp_ind = 3
grp_inds = []
for i, g in enumerate(grps):
if g == "all":
grp_ind = list(range(len(roi_grps)))
elif g == "change":
grp_ind = [1, 2]
elif g == "no_change":
grp_ind = [0, 3]
elif g == "reduc":
grp_ind = [1]
elif g == "incr":
grp_ind = [2]
else:
gen_util.accepted_values_error(
"grps", g, ["all", "change", "no_change", "reduc", "incr"])
if add_exp and exp_ind not in grp_ind:
grp_ind.extend([exp_ind])
grp_inds.append(sorted(grp_ind))
elif str(tails) == "2":
# sign_rois[first/last][lo/up]
all_rois = list(range(nrois))
unexp_up_unexp_up = list(set(sign_rois[0][1]) & set(sign_rois[1][1]))
unexp_up_unexp_lo = list(set(sign_rois[0][1]) & set(sign_rois[1][0]))
unexp_lo_unexp_up = list(set(sign_rois[0][0]) & set(sign_rois[1][1]))
unexp_lo_unexp_lo = list(set(sign_rois[0][0]) & set(sign_rois[1][0]))
unexp_up_exp = list(
(set(sign_rois[0][1]) - set(sign_rois[1][1]) - \
set(sign_rois[1][0])))
unexp_lo_exp = list(
(set(sign_rois[0][0]) - set(sign_rois[1][1]) - \
set(sign_rois[1][0])))
exp_unexp_up = list(
(set(sign_rois[1][1]) - set(sign_rois[0][1]) - \
set(sign_rois[0][0])))
exp_unexp_lo = list(
(set(sign_rois[1][0]) - set(sign_rois[0][1]) - \
set(sign_rois[0][0])))
exp_exp = list(
(set(all_rois) - set(sign_rois[0][1]) - set(sign_rois[1][1]) - \
set(sign_rois[0][0]) - set(sign_rois[1][0])))
# to store stats
roi_grps = [
unexp_up_unexp_up, unexp_up_unexp_lo, unexp_lo_unexp_up,
unexp_lo_unexp_lo, unexp_up_exp, unexp_lo_exp, exp_unexp_up,
exp_unexp_lo, exp_exp
]
exp_ind = 8 # index of exp_exp
# group names
grp_names = [
"unexp-up_unexp-up", "unexp-up_unexp-lo", "unexp-lo_unexp-up",
"unexp-lo_unexp-lo", "unexp-up_exp", "unexp-lo_exp",
"exp_unexp-up", "exp_unexp-lo", "exp_exp"
]
exp_ind = 8
grp_inds = []
for i, g in enumerate(grps):
if g == "all":
grp_ind = list(range(len(roi_grps)))
elif g == "change":
grp_ind = [1, 2, 4, 5, 6, 7]
elif g == "no_change":
grp_ind = [0, 3, 8]
elif g == "reduc":
grp_ind = [1, 4, 7]
elif g == "incr":
grp_ind = [2, 5, 6]
else:
gen_util.accepted_values_error(
"grps", grps,
["all", "change", "no_change", "reduc", "incr"])
if add_exp and exp_ind not in grp_ind:
grp_ind.extend([exp_ind])
grp_inds.append(sorted(grp_ind))
all_roi_grps = [[roi_grps[i] for i in grp_ind] for grp_ind in grp_inds]
all_grp_names = [[grp_names[i] for i in grp_ind] for grp_ind in grp_inds]
if len(grps) == 1:
all_roi_grps = all_roi_grps[0]
all_grp_names = all_grp_names[0]
return all_roi_grps, all_grp_names
def scale_data_df(data_df, datatype, interpolated="no", other_vals=[]):
"""
scale_data_df(data_df, datatype)
Returns data frame with specific column scaled using the factors
found in the dataframe.
Required args:
- data_df (pd DataFrame): dataframe containing data for each frame,
organized by:
hierarchical columns:
- datatype : type of data
- interpolated: whether data is interpolated ("yes", "no")
- scaled : whether data is scaled ("yes", "no")
hierarchical rows:
- "info" : type of information contained
("frames": values for each frame,
"factors": scaling values for
each factor)
- "specific" : specific type of information contained
(frame number, scaling factor name)
- datatype (str) : datatype to be scaled
Optional args:
- interpolated (str): whether to scale interpolated data ("yes")
or not ("no")
default: "no"
- other_vals (list) : other values in the hierarchical dataframe to
copy to new column
default: []
Returns:
- data_df (pd DataFrame): same dataframe, but with scaled
data added
"""
datatypes = data_df.columns.unique(level="datatype").tolist()
if datatype not in datatypes:
gen_util.accepted_values_error("datatype", datatype, datatypes)
other_vals = gen_util.list_if_not(other_vals)
if "yes" in data_df[datatype].columns.get_level_values(level="scaled"):
logger.info("Data already scaled.")
factor_names = data_df.loc["factors"].index.unique(
level="specific").tolist()
sub_names = list(filter(lambda x: "sub" in x, factor_names))
if len(sub_names) != 1:
raise RuntimeError("Only one factor should contain 'sub'.")
div_names = list(filter(lambda x: "div" in x, factor_names))
if len(div_names) != 1:
raise RuntimeError("Only one row should contain 'div'.")
sub = data_df.loc[("factors", sub_names[0])].values[0]
div = data_df.loc[("factors", div_names[0])].values[0]
data_df = data_df.copy(deep=True)
data_df.loc[("frames",), (datatype, interpolated, "yes", *other_vals)] = \
(data_df.loc[("frames",),
(datatype, interpolated, "no", *other_vals)].values - sub)/div
return data_df
def main(args):
"""
main(args)
Runs analyses with parser arguments.
Required args:
- args (dict): parser argument dictionary
"""
# set logger to the specified level
logger_util.set_level(level=args.log_level)
args.device = gen_util.get_device(args.cuda)
args.fontdir = DEFAULT_FONTDIR if DEFAULT_FONTDIR.is_dir() else None
if args.comp == "all":
comps = logreg.get_comps(args.stimtype, args.q1v4, args.exp_v_unexp)
else:
check_args(args.comp, args.stimtype, args.q1v4, args.exp_v_unexp)
comps = gen_util.list_if_not(args.comp)
args.output = format_output(args.output, args.runtype, args.q1v4, args.bal,
args.exp_v_unexp)
args_orig = copy.deepcopy(args)
if args.dict_path is not None:
plot_dicts.plot_from_dicts(Path(args.dict_path),
source="logreg",
plt_bkend=args.plt_bkend,
fontdir=args.fontdir,
parallel=args.parallel)
else:
for comp in comps:
args = copy.deepcopy(args_orig)
args.comp = comp
args.not_ctrl = not (set_ctrl(not (args.not_ctrl), comp=args.comp))
logger.info(
f"Task: {args.task}\nStim: {args.stimtype} "
f"\nComparison: {args.comp}\n",
extra={"spacing": "\n"})
if args.task == "run_regr":
run_regr(args)
# collates regression runs and analyses accuracy
elif args.task == "analyse":
logger.info(f"Folder: {args.output}")
logreg.run_analysis(args.output, args.stimtype, args.comp,
not (args.not_ctrl), args.CI, args.alg,
args.parallel)
elif args.task == "plot":
logreg.run_plot(args.output, args.stimtype, args.comp,
not (args.not_ctrl), args.visflow_dir,
args.fluor, not (args.no_scale), args.CI,
args.alg, args.plt_bkend, args.fontdir,
args.modif)
else:
gen_util.accepted_values_error("args.task", args.task,
["run_regr", "analyse", "plot"])
def plot_data_summ(plot_lines, data, stats, shuff_stats, title, savename,
CI=0.95, q1v4=False, evu=False, comp="unexp", modif=False,
no_save=False):
"""
plot_data_summ(plot_lines, data, stats, shuff_stats, title, savename)
Plots summary data for a specific comparison, for each line and plane and
saves figure.
Required args:
- plot_lines (pd DataFrame): DataFrame containing scores summary
for specific comparison and criteria
- data (str) : label of type of data to plot,
e.g., "epoch_n" or "test_acc_bal"
- stats (list) : list of stats to use for non shuffled
data, e.g., ["mean", "sem", "sem"]
- shuff_stats (list) : list of stats to use for shuffled
data, e.g., ["median", "p2p5", "p97p5"]
- title (str) : general plot titles (must contain "data")
- savename (str) : plot save path
Optional args:
- CI (num) : CI for shuffled data (e.g., 0.95)
default: 0.95
- q1v4 (bool) : if True, analysis is separated across first and
last quartiles
default: False
- evu (bool) : if True, the first dataset will include expected
sequences and the second will include unexpected
sequences
default: False
- comp (str) : type of comparison
default: "unexp"
- modif (bool) : if True, plots are made in a modified (simplified
way)
default: False
- no_save (bool): if True, figure is not saved
default: False
Returns:
- fig (plt Figure) : figure
"""
celltypes = [[x, y] for y in ["dend", "soma"] for x in ["L23", "L5"]]
max_sess = max(plot_lines["sess_n"].tolist())
fig, ax = init_res_fig(len(celltypes), max_sess, modif)
if modif:
fig.suptitle(title, y=1.0, weight="bold")
n_vals = 5 # (mean/med, sem/2.5p, sem/97.5p, n_rois, n_runs)
if data == "test_acc_bal":
found = False
for key in plot_lines.keys():
if data in key:
found = True
if not found:
warnings.warn("test_acc_bal was not recorded",
category=RuntimeWarning, stacklevel=1)
return
split_oris = sess_str_util.get_split_oris(comp)
data_types = gen_util.list_if_not(data)
if (q1v4 or evu or split_oris) and "test_acc" in data:
ext_test = sess_str_util.ext_test_str(q1v4, evu, comp)
n_vals = 8 # (extra mean/med, sem/2.5p, sem/97.5p for Q4)
if data == "test_acc":
data_types = ["test_acc", f"{ext_test}_acc"]
elif data == "test_acc_bal":
data_types = ["test_acc_bal", f"{ext_test}_acc_bal"]
else:
gen_util.accepted_values_error("data", data,
["test_acc", "test_acc_bal"])
for i, [line, plane] in enumerate(celltypes):
sub_ax = plot_util.get_subax(ax, i)
if not modif:
sub_ax.set_xlim(-0.5, max_sess - 0.5)
# get the right rows in dataframe
cols = ["plane"]
cri = [plane]
curr_lines = gen_util.get_df_vals(plot_lines.loc[
plot_lines["line"].str.contains(line)], cols, cri)
cri_str = ", ".join([f"{col}: {crit}" for col, crit in zip(cols, cri)])
if len(curr_lines) == 0: # no data
warnings.warn(f"No data found for {line} {plane}, {cri_str}",
category=RuntimeWarning, stacklevel=1)
continue
else: # shuffle or non shuffle missing
skip = False
for shuff in [False, True]:
if shuff not in curr_lines["shuffle"].tolist():
warnings.warn(f"No shuffle={shuff} data found for {line} "
f"{plane}, {cri_str}", category=RuntimeWarning,
stacklevel=1)
skip = True
if skip:
continue
sess_ns = gen_util.get_df_vals(curr_lines, label="sess_n", dtype=int)
mouse_ns = gen_util.get_df_vals(curr_lines, label="mouse_n",
dtype=int)
# mouse x sess x n_vals
if -1 not in mouse_ns:
raise RuntimeError("Shuffle data across mice is missing.")
mouse_ns = gen_util.remove_if(mouse_ns, -1)
data_arr = np.empty((len(mouse_ns), int(max_sess), n_vals)) * np.nan
shuff_arr = np.empty((1, int(max_sess), n_vals - 1)) * np.nan
for sess_n in sess_ns:
sess_mice = gen_util.get_df_vals(
curr_lines, "sess_n", sess_n, "mouse_n", dtype=int)
for m, mouse_n in enumerate(mouse_ns + [-1]):
if mouse_n not in sess_mice:
continue
if mouse_n == -1:
stat_types = shuff_stats
arr = shuff_arr
m = 0
else:
stat_types = stats
arr = data_arr
curr_line = gen_util.get_df_vals(curr_lines,
["sess_n", "mouse_n", "shuffle"],
[sess_n, mouse_n, mouse_n==-1])
if len(curr_line) > 1:
raise RuntimeError("Several lines correspond to criteria.")
elif len(curr_line) == 0:
continue
for st, stat in enumerate(stat_types):
for d, dat in enumerate(data_types):
i = d * 3 + st
arr[m, int(sess_n-1), i] = curr_line[f"{dat}_{stat}"]
if mouse_n != -1:
arr[m, int(sess_n-1), -2] = curr_line["n_rois"]
arr[m, int(sess_n-1), -1] = curr_line["runs_total"] - \
curr_line["runs_nan"]
summ_subplot(sub_ax, data_arr, shuff_arr, title, mouse_ns, sess_ns,
line, plane, stats[0], stats[1], CI, q1v4, evu, split_oris, modif)
if modif:
n_sess_keep = 3
ylab = ax[0, 0].get_ylabel()
ax[0, 0].set_ylabel("")
sess_plot_util.format_linpla_subaxes(ax, ylab=ylab,
xticks=np.arange(1, n_sess_keep + 1))
yticks = ax[0, 0].get_yticks()
# always set ticks (even again) before setting labels
ax[1, 0].set_yticks(yticks)
ax[1, 0].set_yticklabels([int(v) for v in yticks],
fontdict={"weight": "bold"})
if not no_save:
fig.savefig(savename)
return fig
def signif_rois_by_grp_sess(sessids,
integ_data,
permpar,
roigrppar,
qu_labs=["first quant", "last quant"],
stats="mean",
nanpol=None):
"""
signif_rois_by_grp_sess(sessids, integ_data, permpar, roigrppar)
Identifies ROIs showing significant unexpected responses in specified quantiles,
groups accordingly and retrieves statistics for each group.
Required args:
- sessids (list) : list of Session IDs
- integ_data (list) : list of 2D array of ROI activity integrated
across frames. Should only include quantiles
retained for analysis:
sess x unexp (0, 1) x quantiles x
array[ROI x sequences]
- permpar (PermPar) : named tuple containing permutation parameters
- roigrppar (RoiGrpPar): named tuple containing roi grouping parameters
Optional args:
- qu_labs (list): quantiles being compared
default: ["first Q", "last Q"]
- stats (str) : statistic parameter, i.e. "mean" or "median"
default: "mean"
- nanpol (str) : policy for NaNs, "omit" or None when taking statistics
default: None
Returns:
- all_roi_grps (list) : list of sublists per session, containing ROI
numbers included in each group, structured as
follows:
if sets of groups are passed:
session x set x roi_grp
if one group is passed:
session x roi_grp
- grp_names (list) : list of names of the ROI groups in roi grp
lists (order preserved)
"""
if len(qu_labs) != 2:
raise ValueError("Identifying significant ROIs is only implemented "
"for 2 quantiles.")
logger.info(
"Identifying ROIs showing significant unexpected responses in "
f"{qu_labs[0].capitalize()} and/or {qu_labs[1].capitalize()}.",
extra={"spacing": "\n"})
all_roi_grps = []
for sessid, sess_data in zip(sessids, integ_data):
logger.info(f"Session {sessid}", extra={"spacing": "\n"})
sess_rois = []
nrois = sess_data[0][0].shape[0]
for q, q_lab in enumerate(qu_labs):
logger.info(f"{q_lab.capitalize()}", extra={"spacing": TAB})
sign_rois = get_signif_rois([sess_data[0][q], sess_data[1][q]],
permpar, stats, roigrppar.op, nanpol)
sess_rois.append(sign_rois)
grps = gen_util.list_if_not(roigrppar.grps)
if len(grps) == 1:
roi_grps, grp_names = sep_grps(sess_rois,
nrois=nrois,
grps=roigrppar.grps,
tails=permpar.tails,
add_exp=roigrppar.add_exp)
else:
roi_grps = []
for grp_set in roigrppar.grps:
roi_grps_set, _ = sep_grps(sess_rois,
nrois=nrois,
grps=grp_set,
tails=permpar.tails,
add_exp=False)
# flat, without duplicates
flat_grp = sorted(
list(set([roi for grp in roi_grps_set for roi in grp])))
roi_grps.append(flat_grp)
grp_names = roigrppar.grps
all_roi_grps.append(roi_grps)
return all_roi_grps, grp_names
def build_stim_beh_df(sessions, analyspar, sesspar, stimpar):
"""
build_stim_beh_df(sessions, analyspar, sesspar, stimpar)
Builds a dataframe containing the stimulus and behavioural information, for
each of the specified segments.
Inputs: sessions, stimtype, segments, pre-post (pupil, running), pre-post
(ROI)
GLM inputs:
- Unexpected status (e.g., only 1 for U and the following grayscreen (G))
- Pupil dilation, if not using pilot data
- Running velocity
- ROI data (by ROI)
- Plane
- Line
- SessID
- Stimulus parameters
"""
full_df = pd.DataFrame()
sessions = gen_util.list_if_not(sessions)
pupil = (sesspar.runtype != "pilot")
for sess in sessions:
logger.info(f"Building dataframe for session {sess.sessid}",
extra={"spacing": "\n"})
stim = sess.get_stim(stimpar.stimtype)
sub_df = stim.get_stim_beh_sub_df(stimpar.pre,
stimpar.post,
analyspar.stats,
analyspar.fluor,
analyspar.rem_bad,
gabfr=stimpar.gabfr,
gabk=stimpar.gabk,
gab_ori=stimpar.gab_ori,
visflow_size=stimpar.visflow_size,
visflow_dir=stimpar.visflow_dir,
pupil=pupil,
run=True,
roi_stats=False)
full_df = full_df.append(sub_df)
# set unexpected status for any gabor frames other than D_U, G to 0.
if "gabor_frame" in full_df.columns:
full_df.loc[(~full_df["gabor_frame"].isin(["D", "U", "D_U", "G"])) &
(full_df["unexpected"] == 1), "unexpected"] = 0
# set mean orientations to the sequence's expected frame orientation for
# unexpected frames
if "gabor_mean_orientation" in full_df.columns:
unique_oris = full_df.loc[
(full_df["gabor_frame"].isin(["D", "U", "D_U", "G"])) &
(full_df["unexpected"] == 1), "gabor_mean_orientation"].unique()
for unique_ori in unique_oris:
full_df.loc[
(full_df["gabor_frame"].isin(["D", "U", "D_U", "G"])) &
(full_df["unexpected"] == 1) &
(full_df["gabor_mean_orientation"] == unique_ori),
"gabor_mean_orientation"] = sess_gen_util.get_reg_gab_ori(
unique_ori)
# drop unnecessary or redundant columns
drop = [
"stimulus_template_name", # unnecessary
"square_proportion_flipped", # redundant
"square_number", # redundant
"gabor_number", # single value, NaN for "G" frames
"gabor_locations_x", # full table / array / NaN for "G" frames
"gabor_locations_y", # full table / array / NaN for "G" frames
"gabor_sizes", # full table / array / NaN for "G" frames
"gabor_orientations" # full table / array / NaN for "G" frames
]
drop_cols = [col for col in full_df.columns if col in drop]
full_df = full_df.drop(columns=drop_cols)
full_df = full_df.reset_index(drop=True)
full_df = gen_util.drop_unique(full_df, in_place=True)
return full_df
def format_stim_criteria(stim_df,
stimtype="gabors",
unexp="any",
stim_seg="any",
gabfr="any",
gabk=None,
gab_ori=None,
visflow_size=None,
visflow_dir=None,
start2pfr="any",
end2pfr="any",
num2pfr="any"):
"""
format_stim_criteria()
Returns a list of stimulus parameters formatted correctly to use
as criteria when searching through the stim dataframe.
Will strip any criteria not related to the relevant stimulus type.
Required args:
- stim_df (pd DataFrame) : stimulus dataframe
Optional args:
- stimtype (str) : stimulus type
default: "gabors"
- unexp (str, int or list) : unexpected value(s) of interest (0, 1)
default: "any"
- stim_seg (str, int or list) : stimulus segment value(s) of interest
default: "any"
- gabfr (str, int or list) : gaborframe value(s) of interest
(0, 1, 2, 3, 4 or letters)
default: "any"
- gabk (int or list) : if not None, will overwrite
stimPar2 (4, 16, or "any")
default: None
- gab_ori (int or list) : if not None, will overwrite
stimPar1 (0, 45, 90, 135, 180, 225,
or "any")
default: None
- visflow_size (int or list) : if not None, will overwrite
stimPar1 (128, 256, or "any")
default: None
- visflow_dir (str or list) : if not None, will overwrite
stimPar2 ("right", "left", "temp",
"nasal", or "any")
default: None
- start2pfr (str or list) : 2p start frames range of interest
[min, max (excl)]
default: "any"
- end2pfr (str or list) : 2p excluded end frames range of
interest [min, max (excl)]
default: "any"
- num2pfr (str or list) : 2p num frames range of interest
[min, max (excl)]
default: "any"
Returns:
- unexp (list) : unexpected value(s) of interest (0, 1)
- stim_seg (list) : stim_seg value(s) of interest
- gabfr (list) : gaborframe value(s) of interest
- gabk (list) : gabor kappa value(s) of interest
- gab_ori (list) : gabor mean orientation value(s) of interest
- visflow_size (list): visual flow square size value(s) of interest
- visflow_dir (list) : visual flow direction value(s) of interest
- start2pfr_min (int): minimum of 2p start2pfr range of interest
- start2pfr_max (int): maximum of 2p start2pfr range of interest
(excl)
- end2pfr_min (int) : minimum of 2p end2pfr range of interest
- end2pfr_max (int) : maximum of 2p end2pfr range of interest
(excl)
- num2pfr_min (int) : minimum of num2pfr range of interest
- num2pfr_max (int) : maximum of num2pfr range of interest
(excl)
"""
# remove visual flow criteria for gabors and vv
if stimtype == "gabors":
visflow_size = None
visflow_dir = None
elif stimtype == "visflow":
gabfr = None
gabk = None
gab_ori = None
else:
gen_util.accepted_values_error("stimtype", stimtype,
["gabors", "visflow"])
# converts values to lists or gets all possible values, if "any"
unexp = gen_util.get_df_label_vals(stim_df, "unexpected", unexp)
gabk = gen_util.get_df_label_vals(stim_df, "gabor_kappa", gabk)
gabfr = gen_util.get_df_label_vals(stim_df, "gabor_frame", gabfr)
gab_ori = gen_util.get_df_label_vals(stim_df, "gabor_mean_orientation",
gab_ori)
visflow_dir = gen_util.get_df_label_vals(stim_df, "main_flow_direction",
visflow_dir)
visflow_size = gen_util.get_df_label_vals(stim_df, "square_size",
visflow_size)
if stim_seg in ["any", "all"]:
stim_seg = stim_df.index
else:
stim_seg = gen_util.list_if_not(stim_seg)
gabfr = copy.deepcopy(gabfr)
for fr in gabfr:
if str(fr) == "0":
gabfr.append("A")
elif str(fr) == "1":
gabfr.append("B")
elif str(fr) == "2":
gabfr.append("C")
elif str(fr) == "3":
gabfr.extend(["D", "U"])
elif str(fr) == "4":
gabfr.append("G")
for i in range(len(visflow_dir)):
if visflow_dir[i] in ["right", "left", "temp", "nasal"]:
visflow_dir[i] = \
sess_gen_util.get_visflow_screen_mouse_direc(
visflow_dir[i
])
if start2pfr in ["any", None]:
start2pfr_min = int(stim_df["start_frame_twop"].min())
start2pfr_max = int(stim_df["start_frame_twop"].max() + 1)
elif len(start2pfr) == 2:
start2pfr_min, start2pfr_max = start2pfr
else:
raise ValueError("'start2pfr' must be of length 2 if passed.")
if end2pfr in ["any", None]:
end2pfr_min = int(stim_df["stop_frame_twop"].min())
end2pfr_max = int(stim_df["stop_frame_twop"].max() + 1)
elif len(start2pfr) == 2:
end2pfr_min, end2pfr_max = end2pfr
else:
raise ValueError("'end2pfr' must be of length 2 if passed.")
if num2pfr in ["any", None]:
num2pfr_min = int(stim_df["num_frames_twop"].min())
num2pfr_max = int(stim_df["num_frames_twop"].max() + 1)
elif len(start2pfr) == 2:
num2pfr_min, num2pfr_max = num2pfr
else:
raise ValueError("'num2pfr' must be of length 2 if passed.")
return [
unexp, stim_seg, gabfr, gabk, gab_ori, visflow_size, visflow_dir,
start2pfr_min, start2pfr_max, end2pfr_min, end2pfr_max, num2pfr_min,
num2pfr_max
]
def gabor_example_roi_usis(sessions, analyspar, sesspar, stimpar, basepar,
idxpar, permpar, figpar, seed=None, parallel=False):
"""
gabor_example_roi_usis(sessions, analyspar, sesspar, stimpar, basepar,
idxpar, permpar, figpar)
Retrieves example ROI Gabor USI traces.
Saves results and parameters relevant to analysis in a dictionary.
Required args:
- sessions (list):
Session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
- sesspar (SessPar):
named tuple containing session parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- basepar (BasePar):
named tuple containing baseline parameters
- idxpar (IdxPar):
named tuple containing index parameters
- permpar (PermPar):
named tuple containing permutation parameters
- figpar (dict):
dictionary containing figure parameters
Optional args:
- seed (int):
seed value to use. (-1 treated as None)
default: None
- parallel (bool):
if True, some of the analysis is run in parallel across CPU cores
default: False
"""
logger.info("Compiling Gabor example ROI USI data.",
extra={"spacing": "\n"}
)
# check stimpar.pre
if (not isinstance(stimpar.pre, list)) or (len(stimpar.pre) == 1):
pre_list = gen_util.list_if_not(stimpar.pre)
stimpar = sess_ntuple_util.get_modif_ntuple(stimpar, "pre", pre_list)
elif len(stimpar.pre) != 2:
raise ValueError("Expected 2 values for stimpar.pre: one for "
"index calculation and one for traces.")
# use first stimpar.pre for idx calculation
stimpar_idx = sess_ntuple_util.get_modif_ntuple(
stimpar, "pre", stimpar.pre[0]
)
chosen_rois_df = usi_analys.get_chosen_roi_df(
sessions,
analyspar=analyspar,
stimpar=stimpar_idx,
basepar=basepar,
idxpar=idxpar,
permpar=permpar,
target_idx_vals = [0.5, 0, -0.5],
target_idx_sigs = ["sig", "not_sig", "sig"],
randst=seed,
parallel=parallel
)
# use second stimpar.pre for traces
stimpar_tr = sess_ntuple_util.get_modif_ntuple(
stimpar, "pre", stimpar.pre[1]
)
chosen_rois_df = usi_analys.add_chosen_roi_traces(
sessions,
chosen_rois_df,
analyspar=analyspar,
stimpar=stimpar_tr,
basepar=basepar,
split=idxpar.feature,
parallel=parallel
)
extrapar = {"seed": seed}
info = {
"analyspar": analyspar._asdict(),
"stimpar": stimpar._asdict(),
"sesspar": sesspar._asdict(),
"basepar": basepar._asdict(),
"idxpar": idxpar._asdict(),
"permpar": permpar._asdict(),
"extrapar": extrapar,
"chosen_rois_df": chosen_rois_df.to_dict()
}
helper_fcts.plot_save_all(info, figpar)
def main(args):
"""
main(args)
Runs analyses with parser arguments.
Required args:
- args (dict): parser argument dictionary
"""
# set logger to the specified level
logger_util.set_level(level=args.log_level)
args.fontdir = DEFAULT_FONTDIR if DEFAULT_FONTDIR.is_dir() else None
if args.dict_path is not None:
source = "modif" if args.modif else "run"
plot_dicts.plot_from_dicts(Path(args.dict_path),
source=source,
plt_bkend=args.plt_bkend,
fontdir=args.fontdir,
parallel=args.parallel,
datetime=not (args.no_datetime),
overwrite=args.overwrite)
else:
args = reformat_args(args)
if args.datadir is None:
args.datadir = DEFAULT_DATADIR
else:
args.datadir = Path(args.datadir)
mouse_df = DEFAULT_MOUSE_DF_PATH
# get numbers of sessions to analyse
if args.sess_n == "all":
all_sess_ns = sess_gen_util.get_sess_vals(mouse_df,
"sess_n",
runtype=args.runtype,
plane=args.plane,
line=args.line,
min_rois=args.min_rois,
pass_fail=args.pass_fail,
incl=args.incl,
omit_sess=args.omit_sess,
omit_mice=args.omit_mice,
sort=True)
else:
all_sess_ns = gen_util.list_if_not(args.sess_n)
# get analysis parameters for each session number
all_analys_pars = gen_util.parallel_wrap(prep_analyses,
all_sess_ns,
args_list=[args, mouse_df],
parallel=args.parallel)
# split parallel from sequential analyses
bool(args.parallel * (not args.debug))
if args.parallel:
run_seq = "" # should be run parallel within analysis
all_analyses = gen_util.remove_lett(args.analyses, run_seq)
sess_parallels = [True, False]
analyses_parallels = [False, True]
else:
all_analyses = [args.analyses]
sess_parallels, analyses_parallels = [False], [False]
for analyses, sess_parallel, analyses_parallel in zip(
all_analyses, sess_parallels, analyses_parallels):
if len(analyses) == 0:
continue
args_dict = {
"analyses": analyses,
"seed": args.seed,
"parallel": analyses_parallel,
}
# run analyses for each parameter set
gen_util.parallel_wrap(run_analyses,
all_analys_pars,
args_dict=args_dict,
parallel=sess_parallel,
mult_loop=True)
def add_stim_roi_stats(stim_stats_df,
sessions,
analyspar,
stimpar,
permpar,
comp_sess=[1, 3],
in_place=False,
randst=None):
"""
add_stim_roi_stats(stim_stats_df, sessions, analyspar, stimpar, permpar)
Adds to dataframe comparison of absolute fractional data changes
between sessions for different stimuli, calculated for individual ROIs.
Required args:
- stim_stats_df (pd.DataFrame):
dataframe with one row per line/plane, and the basic sess_df
columns, as well as stimulus columns for each comp_sess:
- {stimpar.stimtype}_s{comp_sess[0]}:
first comp_sess data for each ROI
- {stimpar.stimtype}_s{comp_sess[1]}:
second comp_sess data for each ROI
- sessions (list):
session objects
- analyspar (AnalysPar):
named tuple containing analysis parameters
- stimpar (StimPar):
named tuple containing stimulus parameters
- permpar (PermPar):
named tuple containing permutation parameters
Optional args:
- comp_sess (int):
sessions for which to obtain absolute fractional change
[x, y] => |(y - x) / x|
default: [1, 3]
- in_place (bool):
if True, targ_df is modified in place. Otherwise, a deep copy is
modified. targ_df is returned in either case.
default: False
- randst (int or np.random.RandomState):
random state or seed value to use. (-1 treated as None)
default: None
Returns:
- stim_stats_df (pd.DataFrame):
dataframe with one row per line/plane and one for all line/planes
together, and the basic sess_df columns, in addition to the input
columns, and for each stimtype:
- {stimtype} (list): absolute fractional change statistics (me, err)
- p_vals (float): p-value for data differences between stimulus
types, corrected for multiple comparisons and tails
"""
nanpol = None if analyspar.rem_bad else "omit"
if analyspar.tracked:
misc_analys.check_sessions_complete(sessions, raise_err=True)
else:
raise ValueError(
"If analysis is run for individual ROIs and not population "
"statistics, analyspar.tracked must be set to True.")
if not in_place:
stim_stats_df = stim_stats_df.copy(deep=True)
stimtypes = gen_util.list_if_not(stimpar.stimtype)
stim_stats_df = gen_util.set_object_columns(stim_stats_df,
stimtypes,
in_place=True)
# compile all data
full_data = dict()
for stimtype in stimpar.stimtype:
for n in comp_sess:
stim_col = f"{stimtype}_s{n}"
full_data[stim_col] = np.concatenate(stim_stats_df[stim_col])
row_idx = len(stim_stats_df)
for col in stim_stats_df.columns:
stim_stats_df.loc[row_idx, col] = "all"
if col in full_data.keys():
stim_stats_df.loc[row_idx, col] = full_data[col]
# take statistics
for row_idx in stim_stats_df.index:
comp_data = [None, None]
for s, stimtype in enumerate(stimpar.stimtype):
stim_data = []
for n in comp_sess:
data_col = f"{stimtype}_s{n}"
stim_data.append(stim_stats_df.loc[row_idx, data_col])
abs_fractional_diff(stim_data)
# get stats and add to dataframe
stim_stats_df.at[row_idx, stimtype] = \
math_util.get_stats(
comp_data[s], analyspar.stats, analyspar.error,
nanpol=nanpol
).tolist()
# obtain p-values
stim_stats_df.loc[row_idx, "p_vals"] = rand_util.get_op_p_val(
comp_data,
permpar.n_perms,
stats=analyspar.stats,
paired=True,
nanpol=nanpol,
randst=randst)
# remove full data columns
data_cols = []
for s, stimtype in enumerate(stimpar.stimtype):
for n in comp_sess:
data_cols.append(f"{stimtype}_s{n}")
stim_stats_df = stim_stats_df.drop(data_cols, axis=1)
return stim_stats_df
