def main(args): """ main(args) Runs analyses with parser arguments. Required args: - args (dict): parser argument dictionary """ # set logger to the specified level logger_util.set_level(level=args.log_level) # args.device = gen_util.get_device(args.cuda) args.fontdir = DEFAULT_FONTDIR if DEFAULT_FONTDIR.is_dir() else None if args.dict_path != "": plot_dicts.plot_from_dicts(Path(args.dict_path), source="glm", plt_bkend=args.plt_bkend, fontdir=args.fontdir, parallel=args.parallel, datetime=not (args.no_datetime), overwrite=args.overwrite) else: if args.datadir is None: args.datadir = DEFAULT_DATADIR else: args.datadir = Path(args.datadir) mouse_df = DEFAULT_MOUSE_DF_PATH args = reformat_args(args) # get numbers of sessions to analyse if args.sess_n == "all": all_sess_ns = sess_gen_util.get_sess_vals(mouse_df, "sess_n", runtype=args.runtype, plane=args.plane, line=args.line, min_rois=args.min_rois, pass_fail=args.pass_fail, incl=args.incl, omit_sess=args.omit_sess, omit_mice=args.omit_mice) else: all_sess_ns = gen_util.list_if_not(args.sess_n) for sess_n in all_sess_ns: analys_pars = prep_analyses(sess_n, args, mouse_df) analyses_parallel = bool(args.parallel * (not args.debug)) run_analyses(*analys_pars, analyses=args.analyses, parallel=analyses_parallel)
def main(args): """ main(args) Runs analyses with parser arguments. Required args: - args (dict): parser argument dictionary """ # set logger to the specified level logger_util.set_level(level=args.log_level) args.device = "cpu" if args.datadir is None: args.datadir = DEFAULT_DATADIR else: args.datadir = Path(args.datadir) args.mouse_df = DEFAULT_MOUSE_DF_PATH args.runtype = "prod" args.plane = "soma" args.stimtype = "gabors" args.omit_sess, args.omit_mice = sess_gen_util.all_omit( args.stimtype, args.runtype ) all_sessids = sess_gen_util.get_sess_vals( args.mouse_df, "sessid", runtype=args.runtype, sess_n=[1, 2, 3], plane=args.plane, min_rois=1, pass_fail="P", omit_sess=args.omit_sess, omit_mice=args.omit_mice) # bsizes =[1, 15, 30] #3 # outchs = [18, 9, 3] # hiddims = [100, 35, 5] # numlays = [3, 2, 1] # lr_exs = [4, 3, 5] # convs = [True, False] # args.n_epochs = 0 gen_util.parallel_wrap( run_sess_lstm, all_sessids, args_list=[args], parallel=args.parallel)
def download_dandiset_assets(dandiset_id="000037", version="draft", output=".", incl_stim_templates=False, incl_full_stacks=False, sess_ns="all", mouse_ns="all", excluded_sess=True, mouse_df=DEFAULT_MOUSE_DF_PATH, log_level="info"): logger_util.format_all(level=log_level) dandiset_id = f"{int(dandiset_id):06}" # ensure correct ID formatting asset_sessids = "all" if sess_ns not in ["all", "any"] or mouse_ns not in ["all", "any"]: if dandiset_id != "000037": raise NotImplementedError( "Selecting assets based on session and mouse numbers is only " "implemented for dandiset 000037.") sess_ns = reformat_n(sess_ns) mouse_ns = reformat_n(mouse_ns) pass_fail = "all" if excluded_sess else "P" asset_sessids = sess_gen_util.get_sess_vals(mouse_df, "dandi_session_id", mouse_n=mouse_ns, sess_n=sess_ns, runtype="prod", pass_fail=pass_fail, incl="all", sort=True) logger.info("Identifying the URLs of dandi assets to download...") dandiset_urls = get_dandiset_asset_urls( dandiset_id, version=version, asset_sessids=asset_sessids, incl_stim_templates=incl_stim_templates, incl_full_stacks=incl_full_stacks) logger.info(f"Downloading {len(dandiset_urls)} assets from " f"dandiset {dandiset_id}...") for dandiset_url in dandiset_urls: dandi_download.download(dandiset_url, output, existing="refresh")
def main(args): """ main(args) Runs analyses with parser arguments. Required args: - args (dict): parser argument dictionary """ # set logger to the specified level logger_util.set_level(level=args.log_level) args.fontdir = DEFAULT_FONTDIR if DEFAULT_FONTDIR.is_dir() else None if args.dict_path is not None: source = "modif" if args.modif else "run" plot_dicts.plot_from_dicts(Path(args.dict_path), source=source, plt_bkend=args.plt_bkend, fontdir=args.fontdir, parallel=args.parallel, datetime=not (args.no_datetime), overwrite=args.overwrite) else: args = reformat_args(args) if args.datadir is None: args.datadir = DEFAULT_DATADIR else: args.datadir = Path(args.datadir) mouse_df = DEFAULT_MOUSE_DF_PATH # get numbers of sessions to analyse if args.sess_n == "all": all_sess_ns = sess_gen_util.get_sess_vals(mouse_df, "sess_n", runtype=args.runtype, plane=args.plane, line=args.line, min_rois=args.min_rois, pass_fail=args.pass_fail, incl=args.incl, omit_sess=args.omit_sess, omit_mice=args.omit_mice, sort=True) else: all_sess_ns = gen_util.list_if_not(args.sess_n) # get analysis parameters for each session number all_analys_pars = gen_util.parallel_wrap(prep_analyses, all_sess_ns, args_list=[args, mouse_df], parallel=args.parallel) # split parallel from sequential analyses bool(args.parallel * (not args.debug)) if args.parallel: run_seq = "" # should be run parallel within analysis all_analyses = gen_util.remove_lett(args.analyses, run_seq) sess_parallels = [True, False] analyses_parallels = [False, True] else: all_analyses = [args.analyses] sess_parallels, analyses_parallels = [False], [False] for analyses, sess_parallel, analyses_parallel in zip( all_analyses, sess_parallels, analyses_parallels): if len(analyses) == 0: continue args_dict = { "analyses": analyses, "seed": args.seed, "parallel": analyses_parallel, } # run analyses for each parameter set gen_util.parallel_wrap(run_analyses, all_analys_pars, args_dict=args_dict, parallel=sess_parallel, mult_loop=True)
def run_regr(args): """ run_regr(args) Does runs of a logistic regressions on the specified comparison and range of sessions. Required args: - args (Argument parser): parser with analysis parameters as attributes: alg (str) : algorithm to use ("sklearn" or "pytorch") bal (bool) : if True, classes are balanced batchsize (int) : nbr of samples dataloader will load per batch (for "pytorch" alg) visflow_dir (str) : visual flow direction to analyse visflow_per (float) : number of seconds to include before visual flow segments visflow_size (int or list): visual flow square sizes to include comp (str) : type of comparison datadir (str) : data directory dend (str) : type of dendrites to use ("allen" or "dend") device (str) : device name (i.e., "cuda" or "cpu") ep_freq (int) : frequency at which to log loss to console error (str) : error to take, i.e., "std" (for std or quantiles) or "sem" (for SEM or MAD) fluor (str) : fluorescence trace type fontdir (str) : directory in which additional fonts are located gabfr (int) : gabor frame of reference if comparison is "unexp" gabk (int or list) : gabor kappas to include gab_ori (list or str) : gabor orientations to include incl (str or list) : sessions to include ("yes", "no", "all") lr (num) : model learning rate (for "pytorch" alg) mouse_n (int) : mouse number n_epochs (int) : number of epochs n_reg (int) : number of regular runs n_shuff (int) : number of shuffled runs scale (bool) : if True, each ROI is scaled output (str) : general directory in which to save output parallel (bool) : if True, runs are done in parallel plt_bkend (str) : pyplot backend to use q1v4 (bool) : if True, analysis is trained on first and tested on last quartiles exp_v_unexp (bool) : if True, analysis is trained on expected and tested on unexpected sequences runtype (str) : type of run ("prod" or "pilot") seed (int) : seed to seed random processes with sess_n (int) : session number stats (str) : stats to take, i.e., "mean" or "median" stimtype (str) : stim to analyse ("gabors" or "visflow") train_p (list) : proportion of dataset to allocate to training uniqueid (str or int) : unique ID for analysis wd (float) : weight decay value (for "pytorch" arg) """ args = copy.deepcopy(args) if args.datadir is None: args.datadir = DEFAULT_DATADIR else: args.datadir = Path(args.datadir) if args.uniqueid == "datetime": args.uniqueid = gen_util.create_time_str() elif args.uniqueid in ["None", "none"]: args.uniqueid = None reseed = False if args.seed in [None, "None"]: reseed = True # deal with parameters extrapar = {"uniqueid": args.uniqueid, "seed": args.seed} techpar = { "reseed": reseed, "device": args.device, "alg": args.alg, "parallel": args.parallel, "plt_bkend": args.plt_bkend, "fontdir": args.fontdir, "output": args.output, "ep_freq": args.ep_freq, "n_reg": args.n_reg, "n_shuff": args.n_shuff, } mouse_df = DEFAULT_MOUSE_DF_PATH stimpar = logreg.get_stimpar(args.comp, args.stimtype, args.visflow_dir, args.visflow_size, args.gabfr, args.gabk, gab_ori=args.gab_ori, visflow_pre=args.visflow_pre) analyspar = sess_ntuple_util.init_analyspar(args.fluor, stats=args.stats, error=args.error, scale=not (args.no_scale), dend=args.dend) if args.q1v4: quantpar = sess_ntuple_util.init_quantpar(4, [0, -1]) else: quantpar = sess_ntuple_util.init_quantpar(1, 0) logregpar = sess_ntuple_util.init_logregpar(args.comp, not (args.not_ctrl), args.q1v4, args.exp_v_unexp, args.n_epochs, args.batchsize, args.lr, args.train_p, args.wd, args.bal, args.alg) omit_sess, omit_mice = sess_gen_util.all_omit(stimpar.stimtype, args.runtype, stimpar.visflow_dir, stimpar.visflow_size, stimpar.gabk) sessids = sorted( sess_gen_util.get_sess_vals(mouse_df, "sessid", args.mouse_n, args.sess_n, args.runtype, incl=args.incl, omit_sess=omit_sess, omit_mice=omit_mice)) if len(sessids) == 0: logger.warning( f"No sessions found (mouse: {args.mouse_n}, sess: {args.sess_n}, " f"runtype: {args.runtype})") for sessid in sessids: sess = sess_gen_util.init_sessions(sessid, args.datadir, mouse_df, args.runtype, full_table=False, fluor=analyspar.fluor, dend=analyspar.dend, temp_log="warning")[0] logreg.run_regr(sess, analyspar, stimpar, logregpar, quantpar, extrapar, techpar) plot_util.cond_close_figs()
def init_sessions(analyspar, sesspar, mouse_df, datadir, sessions=None, roi=True, run=False, pupil=False, parallel=False): """ init_sessions(sesspar, mouse_df, datadir) Initializes sessions. Required args: - analyspar (AnalysPar): named tuple containing session parameters - sesspar (SessPar): named tuple containing session parameters - mouse_df (pandas df): path name of dataframe containing information on each session - datadir (Path): path to data directory Optional args: - sessions (list): preloaded sessions default: None - roi (bool): if True, ROI data is loaded default: True - run (bool): if True, running data is loaded default: False - pupil (bool): if True, pupil data is loaded default: False Returns: - sessions (list): Session objects """ sesspar_dict = sesspar._asdict() sesspar_dict.pop("closest") # identify sessions needed sessids = sorted( sess_gen_util.get_sess_vals(mouse_df, "sessid", **sesspar_dict)) if len(sessids) == 0: raise ValueError("No sessions meet the criteria.") # check for preloaded sessions, and only load new ones if sessions is not None: loaded_sessids = [session.sessid for session in sessions] ext_str = " additional" else: sessions = [] loaded_sessids = [] ext_str = "" # identify new sessions to load load_sessids = list( filter(lambda sessid: sessid not in loaded_sessids, sessids)) # remove sessions that are not needed if len(sessions): sessions = [ session for session in sessions if session.sessid in sessids ] # check that previously loaded sessions have roi/run/pupil data loaded args_list = [roi, run, pupil, analyspar.fluor, analyspar.dend] with logger_util.TempChangeLogLevel(level="warning"): sessions = gen_util.parallel_wrap(sess_gen_util.check_session, sessions, args_list=args_list, parallel=parallel) # load new sessions if len(load_sessids): logger.info(f"Loading {len(load_sessids)}{ext_str} session(s)...", extra={"spacing": "\n"}) args_dict = { "datadir": datadir, "mouse_df": mouse_df, "runtype": sesspar.runtype, "full_table": False, "fluor": analyspar.fluor, "dend": analyspar.dend, "roi": roi, "run": run, "pupil": pupil, "temp_log": "critical" # suppress almost all logs } new_sessions = gen_util.parallel_wrap(sess_gen_util.init_sessions, load_sessids, args_dict=args_dict, parallel=parallel, use_tqdm=True) # flatten list of new sessions, and add to full sessions list new_sessions = [sess for singles in new_sessions for sess in singles] sessions = sessions + new_sessions # combine session lists, and sort sorter = [sessids.index(session.sessid) for session in sessions] sessions = [sessions[i] for i in sorter] # update ROI tracking parameters for sess in sessions: sess.set_only_tracked_rois(analyspar.tracked) return sessions
def prep_analyses(sess_n, args, mouse_df, parallel=False): """ prep_analyses(sess_n, args, mouse_df) Prepares named tuples and sessions for which to run analyses, based on the arguments passed. Required args: - sess_n (int) : session number to run analyses on, or combination of session numbers to compare, e.g. "1v2" - args (Argument parser): parser containing all parameters - mouse_df (pandas df) : path name of dataframe containing information on each session Optional args: - parallel (bool): if True, sessions are initialized in parallel across CPU cores default: False Returns: - sessions (list) : list of sessions, or nested list per mouse if sess_n is a combination - analysis_dict (dict): dictionary of analysis parameters (see init_param_cont()) """ args = copy.deepcopy(args) args.sess_n = sess_n analysis_dict = init_param_cont(args) analyspar, sesspar, stimpar = [ analysis_dict[key] for key in ["analyspar", "sesspar", "stimpar"] ] roi = (args.datatype == "roi") run = (args.datatype == "run") sesspar_dict = sesspar._asdict() _ = sesspar_dict.pop("closest") [all_mouse_ns, all_sess_ns] = sess_gen_util.get_sess_vals(mouse_df, ["mouse_n", "sess_n"], omit_sess=args.omit_sess, omit_mice=args.omit_mice, unique=False, **sesspar_dict) if args.sess_n in ["any", "all"]: all_sess_ns = [n + 1 for n in range(max(all_sess_ns))] else: all_sess_ns = args.sess_n # get session IDs and create Sessions all_mouse_ns = sorted(set(all_mouse_ns)) logger.info(f"Loading sessions for {len(all_mouse_ns)} mice...", extra={"spacing": "\n"}) args_list = [ all_sess_ns, sesspar, mouse_df, args.datadir, args.omit_sess, analyspar.fluor, analyspar.dend, roi, run ] sessions = gen_util.parallel_wrap(init_mouse_sess, all_mouse_ns, args_list=args_list, parallel=parallel, use_tqdm=True) check_all = set([sess for m_sess in sessions for sess in m_sess]) if len(sessions) == 0 or check_all == {None}: raise RuntimeError("No sessions meet the criteria.") runtype_str = "" if sesspar.runtype != "prod": runtype_str = f" ({sesspar.runtype} data)" stim_str = stimpar.stimtype if stimpar.stimtype == "gabors": stim_str = "gabor" elif stimpar.stimtype == "visflow": stim_str = "visual flow" logger.info( f"Analysis of {sesspar.plane} responses to {stim_str} " f"stimuli{runtype_str}.\nSession {sesspar.sess_n}", extra={"spacing": "\n"}) return sessions, analysis_dict
def init_mouse_sess(mouse_n, all_sess_ns, sesspar, mouse_df, datadir, omit_sess=[], fluor="dff", dend="extr", roi=True, run=False, pupil=False): """ init_mouse_sess(mouse_n, all_sess_ns, sesspar, mouse_df, datadir) Initializes the sessions for the specified mouse. Required args: - mouse_n (int) : mouse number - all_sess_ns (list) : list of all sessions to include - sesspar (SessPar) : named tuple containing session parameters - mouse_df (pandas df): path name of dataframe containing information on each session - datadir (str) : path to data directory Optional args: - omit_sess (list): list of sessions to omit - dend (str) : type of dendrites to use ("allen" or "dend") - fluor (str) : if "raw", raw ROI traces are used. If "dff", dF/F ROI traces are used. - dend (str) : type of dendrites to use ("allen" or "dend") - roi (bool) : if True, ROI data is loaded default: True - run (bool) : if True, running data is loaded default: False - pupil (bool) : if True, pupil data is loaded default: False Returns: - mouse_sesses (list): list of Session objects for the specified mouse, with None in the position of missing sessions """ sesspar_dict = sesspar._asdict() sesspar_dict.pop("closest") mouse_sesses = [] for sess_n in all_sess_ns: sesspar_dict["sess_n"] = sess_n sesspar_dict["mouse_n"] = mouse_n sessid = sess_gen_util.get_sess_vals(mouse_df, "sessid", omit_sess=omit_sess, **sesspar_dict) if len(sessid) == 0: sess = [None] elif len(sessid) > 1: raise RuntimeError( "Expected no more than 1 session per mouse/session number.") else: sess = sess_gen_util.init_sessions(sessid[0], datadir, mouse_df, sesspar.runtype, full_table=False, fluor=fluor, dend=dend, omit=roi, roi=roi, run=run, pupil=pupil, temp_log="warning") if len(sess) == 0: sess = [None] mouse_sesses.append(sess[0]) return mouse_sesses
def prep_analyses(sess_n, args, mouse_df): """ prep_analyses(sess_n, args, mouse_df) Prepares named tuples and sessions for which to run analyses, based on the arguments passed. Required args: - sess_n (int) : session number to run analyses on - args (Argument parser): parser containing all parameters - mouse_df (pandas df) : path name of dataframe containing information on each session Returns: - sessions (list) : list of sessions, or nested list per mouse if sess_n is a combination to compare - analysis_dict (dict): dictionary of analysis parameters (see init_param_cont()) """ args = copy.deepcopy(args) args.sess_n = sess_n analysis_dict = init_param_cont(args) analyspar, sesspar, stimpar = [ analysis_dict[key] for key in ["analyspar", "sesspar", "stimpar"] ] # get session IDs and create Sessions sesspar_dict = sesspar._asdict() _ = sesspar_dict.pop("closest") sessids = sess_gen_util.get_sess_vals(mouse_df, "sessid", omit_sess=args.omit_sess, omit_mice=args.omit_mice, **sesspar_dict) logger.info(f"Loading {len(sessids)} session(s)...", extra={"spacing": "\n"}) args_dict = { "datadir": args.datadir, "mouse_df": mouse_df, "runtype": sesspar.runtype, "full_table": False, "fluor": analyspar.fluor, "dend": analyspar.dend, "run": True, "pupil": (sesspar.runtype != "pilot"), "temp_log": "warning", } sessions = gen_util.parallel_wrap(sess_gen_util.init_sessions, sessids, args_dict=args_dict, parallel=args.parallel, use_tqdm=True) # flatten list of sessions sessions = [sess for singles in sessions for sess in singles] # sort by mouse number sort_order = np.argsort([sess.mouse_n for sess in sessions]) sessions = [sessions[s] for s in sort_order] runtype_str = "" if sesspar.runtype != "prod": runtype_str = f" ({sesspar.runtype} data)" stim_str = stimpar.stimtype if stimpar.stimtype == "gabors": stim_str = "gabor" elif stimpar.stimtype == "visflow": stim_str = "visual flow" logger.info( f"Analysis of {sesspar.plane} responses to {stim_str} " f"stimuli{runtype_str}.\nSession {sesspar.sess_n}", extra={"spacing": "\n"}) return sessions, analysis_dict