def organizer_decode_day(condition, decodeConfig, data_path, save_path): #TODO: obsolete right now. need to fix before using ''' Run decoding experiments with labels based on the day of presentation. Data is contained :param condition: experimental condition. For example, condition OFC. Must contain fields: name, paths, odors, csp :param decodeConfig: class config, contains as fields relevant parameters to run decoding experiment :param data_path: :param save_path: :return: ''' data_pathnames = sorted( glob.glob(os.path.join(data_path, '*' + Config.mat_ext))) config_pathnames = sorted( glob.glob(os.path.join(data_path, '*' + Config.cons_ext))) list_of_all_data = np.array([Config.load_mat_f(d) for d in data_pathnames]) list_of_all_cons = np.array( [Config.load_cons_f(d) for d in config_pathnames]) mouse_names_per_file = np.array( [cons.NAME_MOUSE for cons in list_of_all_cons]) mouse_names, list_of_mouse_ix = np.unique(mouse_names_per_file, return_inverse=True) if mouse_names.size != len(condition.paths): raise ValueError( "res has {0:d} mice, but filter has {1:d} mice".format( mouse_names.size, len(condition.paths))) for i, mouse_name in enumerate(mouse_names): start_time = time.time() ix = mouse_name == mouse_names_per_file list_of_cons = list_of_all_cons[ix] list_of_data = list_of_all_data[ix] for cons in list_of_cons: assert cons.NAME_MOUSE == mouse_name, 'Wrong mouse file!' cons = list_of_cons[0] cons_dict = cons.__dict__ for key, value in cons_dict.items(): if isinstance(value, list) or isinstance(value, np.ndarray): pass else: setattr(decodeConfig, key, value) odor = condition.odors[i] if decodeConfig.decode_style == 'identity': csp = None else: csp = condition.csp[i] scores = decoding.decode_day_labels(list_of_cons, list_of_data, odor, csp, decodeConfig) name = cons.NAME_MOUSE fio.save_json(save_path=save_path, save_name=name, config=decodeConfig) fio.save_numpy(save_path=save_path, save_name=name, data=scores) print("Analyzed: {0:s} in {1:.2f} seconds".format( name, time.time() - start_time))
def load_behavior_folders_from_matlab(path, name, timing_override = False): date_dirs = [os.path.join(path,x) for x in os.listdir(path) if os.path.isdir(os.path.join(path, x))] for date_dir in date_dirs: start_time = time.time() dirs = [os.path.join(date_dir, x) for x in os.listdir(date_dir) if 'cycle' not in x] for dir in dirs: cons = Cons(dir, timing_override) print('[***] LOADED {0:<50s} in: {1:3.3f} seconds'.format(dir, time.time() - start_time)) save_path = os.path.join(Config.LOCAL_DATA_PATH, Config.LOCAL_DATA_BEHAVIOR_FOLDER, name) save_name = cons.NAME_MOUSE + '__' + cons.NAME_DATE + '__' + cons.NAME_PLANE Config.save_cons_f(save_path, save_name, data=cons)
def do_PCA(condition, PCAConfig, data_path, save_path): ''' Run PCA experiment. For each mouse, data should be in format of (trials * time) X neurons, stacked for all odor conditions :param condition: :param PCAConfig: :param data_path: :param save_path: :return: ''' data_pathnames = sorted( glob.glob(os.path.join(data_path, '*' + Config.mat_ext))) config_pathnames = sorted( glob.glob(os.path.join(data_path, '*' + Config.cons_ext))) list_of_all_data = np.array([Config.load_mat_f(d) for d in data_pathnames]) list_of_all_cons = np.array( [Config.load_cons_f(d) for d in config_pathnames]) mouse_names_per_file = np.array( [cons.NAME_MOUSE for cons in list_of_all_cons]) mouse_names, list_of_mouse_ix = np.unique(mouse_names_per_file, return_inverse=True) if mouse_names.size != len(condition.paths): raise ValueError( "res has {0:d} mice, but filter has {1:d} mice".format( mouse_names.size, len(condition.paths))) for i, mouse_name in enumerate(mouse_names): ix = mouse_name == mouse_names_per_file start_day = 0 list_of_cons = list_of_all_cons[ix][start_day:] list_of_data = list_of_all_data[ix][start_day:] for cons in list_of_cons: assert cons.NAME_MOUSE == mouse_name, 'Wrong mouse file!' if hasattr(condition, 'odors'): odor = condition.odors[i] csp = condition.csp[i] else: odor = condition.dt_odors[i] + condition.pt_csp[i] csp = condition.dt_csp[i] + condition.pt_csp[i] name = list_of_cons[0].NAME_MOUSE out, variance_explained = PCA(list_of_cons, list_of_data, odor, csp, PCAConfig) fio.save_pickle(save_path=save_path, save_name=name, data=out) print("Analyzed: {}. Variance explained: {}".format( name, variance_explained))
def load_single_from_matlab(data_path, save = True): start_time = time.time() eng = matlab.engine.start_matlab() cons = Cons(data_path) mat, _ = load_calcium_traces_from_matlab(cons.STORAGE_DATA, eng) print('Time taken to load matfiles: {0:3.3f}'.format(time.time() - start_time)) print(mat.shape) if save == True: mouse = cons.NAME_MOUSE date = cons.NAME_DATE plane = cons.NAME_PLANE save_path = os.path.join(Config.LOCAL_DATA_PATH, Config.LOCAL_DATA_SINGLE_FOLDER, mouse) save_name = date + '_' + plane Config.save_mat_f(save_path, save_name, data=mat) Config.save_cons_f(save_path, save_name, data=cons) return mat, cons
def load_timepoint_from_matlab(path, name, timing_override = False): eng = matlab.engine.start_matlab() data_path = os.path.join(path, 'data') data_wildcard = os.path.join(data_path, '*.mat') matfile_paths = glob.glob(data_wildcard) for p in matfile_paths: start_time = time.time() mat, obj_name = load_calcium_traces_from_matlab(p, eng) dir = eng.eval(obj_name + ".constants.DIR") dir = 'I' + dir[1:] cons = Cons(dir, timing_override) print('[***] LOADED {0:<50s} in: {1:3.3f} seconds'.format(p, time.time() - start_time)) save_path = os.path.join(Config.LOCAL_DATA_PATH, Config.LOCAL_DATA_TIMEPOINT_FOLDER, name) save_name = cons.NAME_MOUSE + '__' + cons.NAME_DATE + '__' + cons.NAME_PLANE Config.save_mat_f(save_path, save_name, data=mat) Config.save_cons_f(save_path, save_name, data=cons)
def load_data(data_path): res = load_all_cons(data_path) data_pathnames = sorted( glob.glob(os.path.join(data_path, '*' + Config.mat_ext))) list_of_all_data = np.zeros(len(data_pathnames), dtype='object') for i in range(len(data_pathnames)): list_of_all_data[i] = Config.load_mat_f(data_pathnames[i]) res['data'] = list_of_all_data assert len(list_of_all_data) == len( res['TRIAL_FRAMES'] ), 'number of data files does not equal number of cons files' return res
def load_all_cons(data_path): res = defaultdict(list) config_pathnames = sorted( glob.glob(os.path.join(data_path, '*' + Config.cons_ext))) for i, config_pn in enumerate(config_pathnames): cons = Config.load_cons_f(config_pn) for key, val in cons.__dict__.items(): if isinstance(val, list): res[key].append(np.array(val)) else: res[key].append(val) for key, val in res.items(): if key == 'DAQ_DATA': arr = np.empty(len(val), dtype='object') for i, v in enumerate(val): arr[i] = v res[key] = arr else: res[key] = np.array(val) return res
mpl.rcParams['font.family'] = 'arial' ax_args_copy = ax_args.copy() ax_args_copy.update({'ylim': [-5, 65], 'yticks': [0, 30, 60]}) bool_ax_args_copy = ax_args.copy() bool_ax_args_copy.update({'ylim': [-5, 105], 'yticks': [0, 50, 100]}) class OFC_DT_Config(): path = 'I:\MANUSCRIPT_DATA\FREELY MOVING\OFC_Discrimination_with_lick_data' name = 'OFC_DT' indices = analysis.Indices() constants = analysis.Constants() config = Config() experiments = [OFC_DT_Config] names = ','.join([x.name for x in experiments]) save_path = os.path.join(Config.LOCAL_FIGURE_PATH, 'BEHAVIOR_CRISTIAN', 'LICKING_CORRESPONDENCE', names) directories = [ constants.pretraining_directory, constants.discrimination_directory ] color_dict = { 'Pretraining_CS+': 'C1', 'Discrimination_CS+': 'green', 'Discrimination_CS-': 'red' }
# load images d = os.path.join('I:\IMPORTANT DATA\DATA_X', mouse, date) start_time = time.time() raw_dir = glob.glob(os.path.join(d, '*z')) d_raw = glob.glob(os.path.join(raw_dir[0], '*.tif'))[0] raw = io.imread(d_raw)[start_frame:end_frame] d_bg = os.path.join(d, 'bg.tif') bg = io.imread(d_bg)[start_frame:end_frame] d_residual = os.path.join(d, 'res.tif') residual = io.imread(d_residual)[start_frame:end_frame] d_denoised = os.path.join(d, 'den_bg.tif') denoised = io.imread(d_denoised)[start_frame:end_frame] print('finished reading : {} s'.format(time.time() - start_time)) # load ROIs data_directory = Config().LOCAL_DATA_PATH data_path = os.path.join(data_directory, 'registration','CNMF', mouse, date) if not reload: import matlab.engine eng = matlab.engine.start_matlab() ds = glob.glob(os.path.join('I:\IMPORTANT DATA\STORAGE_DATA', mouse + '_' + date + '*.mat')) assert len(ds) == 1 temp = eng.load(ds[0]) obj = list(temp.values())[0] obj_name = "obj" eng.workspace[obj_name] = obj signal = eng.eval(obj_name + ".roiCell('norm')") signal = np.asarray(signal).squeeze() roi = eng.eval(obj_name + ".roi") roi = np.asarray(roi).squeeze()
# condition = experimental_conditions.BEHAVIOR_OFC_MUSH_HALO # data_path = os.path.join(Config.LOCAL_DATA_PATH, Config.LOCAL_DATA_TIMEPOINT_FOLDER, condition.name) condition = experimental_conditions.BEHAVIOR_OFC_HALO_PRETRAINING data_path = os.path.join(Config.LOCAL_DATA_PATH, Config.LOCAL_DATA_BEHAVIOR_FOLDER, condition.name) # load cons # paths = condition.paths # import matlab.engine # eng = matlab.engine.start_matlab() # for path in paths: # data_path = os.path.join(path, 'data') # data_wildcard = os.path.join(data_path, '*.mat') # matfile_paths = glob.glob(data_wildcard) # list_of_mats, list_of_cons = [],[] # for i, p in enumerate(matfile_paths): # mat, obj_name = init.load_matlab.load_calcium_traces_from_matlab(p, eng) # dir = eng.eval(obj_name + ".constants.DIR") # cons = Cons(dir, condition.timing_override[i]) # _look_at_timing(cons) #look at cons once loaded config_pathnames = glob.glob(os.path.join(data_path, '*' + Config.cons_ext)) for i, config_pn in enumerate(config_pathnames): cons = Config.load_cons_f(config_pn) _look_at_timing(cons) # print(cons.DIR + '__' + str(cons.TRIAL_FRAMES)) # print(cons.DIR + '__' + str(cons.ODOR_TRIALS))
def organizer_test_split(condition, decodeConfig, data_path, save_path): ''' :param condition: :param decodeConfig: :param data_path: :param save_path: :return: ''' data_pathnames = sorted( glob.glob(os.path.join(data_path, '*' + Config.mat_ext))) config_pathnames = sorted( glob.glob(os.path.join(data_path, '*' + Config.cons_ext))) list_of_all_data = np.array([Config.load_mat_f(d) for d in data_pathnames]) list_of_all_cons = np.array( [Config.load_cons_f(d) for d in config_pathnames]) mouse_names_per_file = np.array( [cons.NAME_MOUSE for cons in list_of_all_cons]) mouse_names, list_of_mouse_ix = np.unique(mouse_names_per_file, return_inverse=True) if mouse_names.size != len(condition.paths): raise ValueError( "res has {0:d} mice, but filter has {1:d} mice".format( mouse_names.size, len(condition.paths))) start_days = decodeConfig.start_day end_days = decodeConfig.end_day days_per_mouse = [] for x, y in zip(start_days, end_days): days_per_mouse.append(np.arange(x, y + 1)) print(days_per_mouse) for i, mouse_name in enumerate(mouse_names): start_time = time.time() ix = mouse_name == mouse_names_per_file list_of_cons_ = list_of_all_cons[ix] list_of_data_ = list_of_all_data[ix] for cons in list_of_cons_: assert cons.NAME_MOUSE == mouse_name, 'Wrong mouse file!' if len(days_per_mouse[i]): list_of_cons = list_of_cons_[days_per_mouse[i]] list_of_data = list_of_data_[days_per_mouse[i]] cons = list_of_cons[0] cons_dict = cons.__dict__ for key, value in cons_dict.items(): if isinstance(value, list) or isinstance(value, np.ndarray): pass else: setattr(decodeConfig, key, value) if hasattr(condition, 'odors'): odor = condition.odors[i] csp = condition.csp[i] else: odor = condition.dt_odors[i] csp = condition.dt_csp[i] cons_odors = [cons.ODOR_UNIQUE for cons in list_of_cons] ix = [ i for i, unique_odors in enumerate(cons_odors) if odor[0] in unique_odors ] list_of_cons = [list_of_cons[i] for i in ix] list_of_data = [list_of_data[i] for i in ix] scores_res = decoding.test_split(list_of_cons, list_of_data, odor, csp, decodeConfig) name = cons.NAME_MOUSE fio.save_json(save_path=save_path, save_name=name, config=decodeConfig) fio.save_pickle(save_path=save_path, save_name=name, data=scores_res) print("Analyzed: {0:s} in {1:.2f} seconds".format( name, time.time() - start_time))
def organizer_decode_odor_within_day(condition, decodeConfig, data_path, save_path): ''' Run decoding experiments with labels based on the odor that was presented. Data is contained :param condition: experimental condition. For example, condition OFC. Must contain fields: name, paths, odors, csp :param decodeConfig: class config, contains as fields relevant parameters to run decoding experiment :param data_path: :param save_path: :return: ''' #TODO: see if it works when odors are different for each animal data_pathnames = sorted( glob.glob(os.path.join(data_path, '*' + Config.mat_ext))) config_pathnames = sorted( glob.glob(os.path.join(data_path, '*' + Config.cons_ext))) mouse_names_per_file = [ Config.load_cons_f(d).NAME_MOUSE for d in config_pathnames ] mouse_names, list_of_mouse_ix = np.unique(mouse_names_per_file, return_inverse=True) if mouse_names.size != len(condition.paths): raise ValueError( "res has {0:d} mice, but filter has {1:d} mice".format( mouse_names.size, len(condition.paths))) for i, data_pn in enumerate(data_pathnames): start_time = time.time() cons = Config.load_cons_f(config_pathnames[i]) data = Config.load_mat_f(data_pn) cons_dict = cons.__dict__ for key, value in cons_dict.items(): if isinstance(value, list) or isinstance(value, np.ndarray): pass else: setattr(decodeConfig, key, value) if hasattr(condition, 'odors'): odor = condition.odors[list_of_mouse_ix[i]] else: odor = condition.dt_odors[list_of_mouse_ix[i]] if decodeConfig.decode_style == 'identity': csp = None else: if hasattr(condition, 'odors'): csp = condition.csp[list_of_mouse_ix[i]] else: csp = condition.dt_csp[list_of_mouse_ix[i]] if odor[0] in cons.ODOR_UNIQUE: scores = decoding.decode_odor_labels(cons, data, odor, csp, decodeConfig) mouse = cons.NAME_MOUSE date = cons.NAME_DATE plane = cons.NAME_PLANE name = mouse + '__' + date + '__' + plane fio.save_json(save_path=save_path, save_name=name, config=decodeConfig) fio.save_numpy(save_path=save_path, save_name=name, data=scores) print("Analyzed: {0:s} in {1:.2f} seconds".format( data_pn, time.time() - start_time))