def organizer_decode_day(condition, decodeConfig, data_path, save_path):
#TODO: obsolete right now. need to fix before using
'''
Run decoding experiments with labels based on the day of presentation. Data is contained
:param condition: experimental condition. For example, condition OFC.
Must contain fields: name, paths, odors, csp
:param decodeConfig: class config, contains as fields relevant parameters to run decoding experiment
:param data_path:
:param save_path:
:return:
'''
data_pathnames = sorted(
glob.glob(os.path.join(data_path, '*' + Config.mat_ext)))
config_pathnames = sorted(
glob.glob(os.path.join(data_path, '*' + Config.cons_ext)))
list_of_all_data = np.array([Config.load_mat_f(d) for d in data_pathnames])
list_of_all_cons = np.array(
[Config.load_cons_f(d) for d in config_pathnames])
mouse_names_per_file = np.array(
[cons.NAME_MOUSE for cons in list_of_all_cons])
mouse_names, list_of_mouse_ix = np.unique(mouse_names_per_file,
return_inverse=True)
if mouse_names.size != len(condition.paths):
raise ValueError(
"res has {0:d} mice, but filter has {1:d} mice".format(
mouse_names.size, len(condition.paths)))
for i, mouse_name in enumerate(mouse_names):
start_time = time.time()
ix = mouse_name == mouse_names_per_file
list_of_cons = list_of_all_cons[ix]
list_of_data = list_of_all_data[ix]
for cons in list_of_cons:
assert cons.NAME_MOUSE == mouse_name, 'Wrong mouse file!'
cons = list_of_cons[0]
cons_dict = cons.__dict__
for key, value in cons_dict.items():
if isinstance(value, list) or isinstance(value, np.ndarray):
pass
else:
setattr(decodeConfig, key, value)
odor = condition.odors[i]
if decodeConfig.decode_style == 'identity':
csp = None
else:
csp = condition.csp[i]
scores = decoding.decode_day_labels(list_of_cons, list_of_data, odor,
csp, decodeConfig)
name = cons.NAME_MOUSE
fio.save_json(save_path=save_path, save_name=name, config=decodeConfig)
fio.save_numpy(save_path=save_path, save_name=name, data=scores)
print("Analyzed: {0:s} in {1:.2f} seconds".format(
name,
time.time() - start_time))
def load_behavior_folders_from_matlab(path, name, timing_override = False):
date_dirs = [os.path.join(path,x) for x in os.listdir(path) if os.path.isdir(os.path.join(path, x))]
for date_dir in date_dirs:
start_time = time.time()
dirs = [os.path.join(date_dir, x) for x in os.listdir(date_dir) if 'cycle' not in x]
for dir in dirs:
cons = Cons(dir, timing_override)
print('[***] LOADED {0:<50s} in: {1:3.3f} seconds'.format(dir, time.time() - start_time))
save_path = os.path.join(Config.LOCAL_DATA_PATH, Config.LOCAL_DATA_BEHAVIOR_FOLDER,
name)
save_name = cons.NAME_MOUSE + '__' + cons.NAME_DATE + '__' + cons.NAME_PLANE
Config.save_cons_f(save_path, save_name, data=cons)
def do_PCA(condition, PCAConfig, data_path, save_path):
'''
Run PCA experiment. For each mouse, data should be in format of (trials * time) X neurons,
stacked for all odor conditions
:param condition:
:param PCAConfig:
:param data_path:
:param save_path:
:return:
'''
data_pathnames = sorted(
glob.glob(os.path.join(data_path, '*' + Config.mat_ext)))
config_pathnames = sorted(
glob.glob(os.path.join(data_path, '*' + Config.cons_ext)))
list_of_all_data = np.array([Config.load_mat_f(d) for d in data_pathnames])
list_of_all_cons = np.array(
[Config.load_cons_f(d) for d in config_pathnames])
mouse_names_per_file = np.array(
[cons.NAME_MOUSE for cons in list_of_all_cons])
mouse_names, list_of_mouse_ix = np.unique(mouse_names_per_file,
return_inverse=True)
if mouse_names.size != len(condition.paths):
raise ValueError(
"res has {0:d} mice, but filter has {1:d} mice".format(
mouse_names.size, len(condition.paths)))
for i, mouse_name in enumerate(mouse_names):
ix = mouse_name == mouse_names_per_file
start_day = 0
list_of_cons = list_of_all_cons[ix][start_day:]
list_of_data = list_of_all_data[ix][start_day:]
for cons in list_of_cons:
assert cons.NAME_MOUSE == mouse_name, 'Wrong mouse file!'
if hasattr(condition, 'odors'):
odor = condition.odors[i]
csp = condition.csp[i]
else:
odor = condition.dt_odors[i] + condition.pt_csp[i]
csp = condition.dt_csp[i] + condition.pt_csp[i]
name = list_of_cons[0].NAME_MOUSE
out, variance_explained = PCA(list_of_cons, list_of_data, odor, csp,
PCAConfig)
fio.save_pickle(save_path=save_path, save_name=name, data=out)
print("Analyzed: {}. Variance explained: {}".format(
name, variance_explained))
def load_single_from_matlab(data_path, save = True):
start_time = time.time()
eng = matlab.engine.start_matlab()
cons = Cons(data_path)
mat, _ = load_calcium_traces_from_matlab(cons.STORAGE_DATA, eng)
print('Time taken to load matfiles: {0:3.3f}'.format(time.time() - start_time))
print(mat.shape)
if save == True:
mouse = cons.NAME_MOUSE
date = cons.NAME_DATE
plane = cons.NAME_PLANE
save_path = os.path.join(Config.LOCAL_DATA_PATH, Config.LOCAL_DATA_SINGLE_FOLDER, mouse)
save_name = date + '_' + plane
Config.save_mat_f(save_path, save_name, data=mat)
Config.save_cons_f(save_path, save_name, data=cons)
return mat, cons
def load_timepoint_from_matlab(path, name, timing_override = False):
eng = matlab.engine.start_matlab()
data_path = os.path.join(path, 'data')
data_wildcard = os.path.join(data_path, '*.mat')
matfile_paths = glob.glob(data_wildcard)
for p in matfile_paths:
start_time = time.time()
mat, obj_name = load_calcium_traces_from_matlab(p, eng)
dir = eng.eval(obj_name + ".constants.DIR")
dir = 'I' + dir[1:]
cons = Cons(dir, timing_override)
print('[***] LOADED {0:<50s} in: {1:3.3f} seconds'.format(p, time.time() - start_time))
save_path = os.path.join(Config.LOCAL_DATA_PATH, Config.LOCAL_DATA_TIMEPOINT_FOLDER,
name)
save_name = cons.NAME_MOUSE + '__' + cons.NAME_DATE + '__' + cons.NAME_PLANE
Config.save_mat_f(save_path, save_name, data=mat)
Config.save_cons_f(save_path, save_name, data=cons)
def load_data(data_path):
res = load_all_cons(data_path)
data_pathnames = sorted(
glob.glob(os.path.join(data_path, '*' + Config.mat_ext)))
list_of_all_data = np.zeros(len(data_pathnames), dtype='object')
for i in range(len(data_pathnames)):
list_of_all_data[i] = Config.load_mat_f(data_pathnames[i])
res['data'] = list_of_all_data
assert len(list_of_all_data) == len(
res['TRIAL_FRAMES']
), 'number of data files does not equal number of cons files'
return res
def load_all_cons(data_path):
res = defaultdict(list)
config_pathnames = sorted(
glob.glob(os.path.join(data_path, '*' + Config.cons_ext)))
for i, config_pn in enumerate(config_pathnames):
cons = Config.load_cons_f(config_pn)
for key, val in cons.__dict__.items():
if isinstance(val, list):
res[key].append(np.array(val))
else:
res[key].append(val)
for key, val in res.items():
if key == 'DAQ_DATA':
arr = np.empty(len(val), dtype='object')
for i, v in enumerate(val):
arr[i] = v
res[key] = arr
else:
res[key] = np.array(val)
return res
mpl.rcParams['font.family'] = 'arial'
ax_args_copy = ax_args.copy()
ax_args_copy.update({'ylim': [-5, 65], 'yticks': [0, 30, 60]})
bool_ax_args_copy = ax_args.copy()
bool_ax_args_copy.update({'ylim': [-5, 105], 'yticks': [0, 50, 100]})
class OFC_DT_Config():
path = 'I:\MANUSCRIPT_DATA\FREELY MOVING\OFC_Discrimination_with_lick_data'
name = 'OFC_DT'
indices = analysis.Indices()
constants = analysis.Constants()
config = Config()
experiments = [OFC_DT_Config]
names = ','.join([x.name for x in experiments])
save_path = os.path.join(Config.LOCAL_FIGURE_PATH, 'BEHAVIOR_CRISTIAN',
'LICKING_CORRESPONDENCE', names)
directories = [
constants.pretraining_directory, constants.discrimination_directory
]
color_dict = {
'Pretraining_CS+': 'C1',
'Discrimination_CS+': 'green',
'Discrimination_CS-': 'red'
}
# load images
d = os.path.join('I:\IMPORTANT DATA\DATA_X', mouse, date)
start_time = time.time()
raw_dir = glob.glob(os.path.join(d, '*z'))
d_raw = glob.glob(os.path.join(raw_dir[0], '*.tif'))[0]
raw = io.imread(d_raw)[start_frame:end_frame]
d_bg = os.path.join(d, 'bg.tif')
bg = io.imread(d_bg)[start_frame:end_frame]
d_residual = os.path.join(d, 'res.tif')
residual = io.imread(d_residual)[start_frame:end_frame]
d_denoised = os.path.join(d, 'den_bg.tif')
denoised = io.imread(d_denoised)[start_frame:end_frame]
print('finished reading : {} s'.format(time.time() - start_time))
# load ROIs
data_directory = Config().LOCAL_DATA_PATH
data_path = os.path.join(data_directory, 'registration','CNMF', mouse, date)
if not reload:
import matlab.engine
eng = matlab.engine.start_matlab()
ds = glob.glob(os.path.join('I:\IMPORTANT DATA\STORAGE_DATA', mouse + '_' + date + '*.mat'))
assert len(ds) == 1
temp = eng.load(ds[0])
obj = list(temp.values())[0]
obj_name = "obj"
eng.workspace[obj_name] = obj
signal = eng.eval(obj_name + ".roiCell('norm')")
signal = np.asarray(signal).squeeze()
roi = eng.eval(obj_name + ".roi")
roi = np.asarray(roi).squeeze()
# condition = experimental_conditions.BEHAVIOR_OFC_MUSH_HALO
# data_path = os.path.join(Config.LOCAL_DATA_PATH, Config.LOCAL_DATA_TIMEPOINT_FOLDER, condition.name)
condition = experimental_conditions.BEHAVIOR_OFC_HALO_PRETRAINING
data_path = os.path.join(Config.LOCAL_DATA_PATH,
Config.LOCAL_DATA_BEHAVIOR_FOLDER, condition.name)
# load cons
# paths = condition.paths
# import matlab.engine
# eng = matlab.engine.start_matlab()
# for path in paths:
# data_path = os.path.join(path, 'data')
# data_wildcard = os.path.join(data_path, '*.mat')
# matfile_paths = glob.glob(data_wildcard)
# list_of_mats, list_of_cons = [],[]
# for i, p in enumerate(matfile_paths):
# mat, obj_name = init.load_matlab.load_calcium_traces_from_matlab(p, eng)
# dir = eng.eval(obj_name + ".constants.DIR")
# cons = Cons(dir, condition.timing_override[i])
# _look_at_timing(cons)
#look at cons once loaded
config_pathnames = glob.glob(os.path.join(data_path, '*' + Config.cons_ext))
for i, config_pn in enumerate(config_pathnames):
cons = Config.load_cons_f(config_pn)
_look_at_timing(cons)
# print(cons.DIR + '__' + str(cons.TRIAL_FRAMES))
# print(cons.DIR + '__' + str(cons.ODOR_TRIALS))
def organizer_test_split(condition, decodeConfig, data_path, save_path):
'''
:param condition:
:param decodeConfig:
:param data_path:
:param save_path:
:return:
'''
data_pathnames = sorted(
glob.glob(os.path.join(data_path, '*' + Config.mat_ext)))
config_pathnames = sorted(
glob.glob(os.path.join(data_path, '*' + Config.cons_ext)))
list_of_all_data = np.array([Config.load_mat_f(d) for d in data_pathnames])
list_of_all_cons = np.array(
[Config.load_cons_f(d) for d in config_pathnames])
mouse_names_per_file = np.array(
[cons.NAME_MOUSE for cons in list_of_all_cons])
mouse_names, list_of_mouse_ix = np.unique(mouse_names_per_file,
return_inverse=True)
if mouse_names.size != len(condition.paths):
raise ValueError(
"res has {0:d} mice, but filter has {1:d} mice".format(
mouse_names.size, len(condition.paths)))
start_days = decodeConfig.start_day
end_days = decodeConfig.end_day
days_per_mouse = []
for x, y in zip(start_days, end_days):
days_per_mouse.append(np.arange(x, y + 1))
print(days_per_mouse)
for i, mouse_name in enumerate(mouse_names):
start_time = time.time()
ix = mouse_name == mouse_names_per_file
list_of_cons_ = list_of_all_cons[ix]
list_of_data_ = list_of_all_data[ix]
for cons in list_of_cons_:
assert cons.NAME_MOUSE == mouse_name, 'Wrong mouse file!'
if len(days_per_mouse[i]):
list_of_cons = list_of_cons_[days_per_mouse[i]]
list_of_data = list_of_data_[days_per_mouse[i]]
cons = list_of_cons[0]
cons_dict = cons.__dict__
for key, value in cons_dict.items():
if isinstance(value, list) or isinstance(value, np.ndarray):
pass
else:
setattr(decodeConfig, key, value)
if hasattr(condition, 'odors'):
odor = condition.odors[i]
csp = condition.csp[i]
else:
odor = condition.dt_odors[i]
csp = condition.dt_csp[i]
cons_odors = [cons.ODOR_UNIQUE for cons in list_of_cons]
ix = [
i for i, unique_odors in enumerate(cons_odors)
if odor[0] in unique_odors
]
list_of_cons = [list_of_cons[i] for i in ix]
list_of_data = [list_of_data[i] for i in ix]
scores_res = decoding.test_split(list_of_cons, list_of_data, odor,
csp, decodeConfig)
name = cons.NAME_MOUSE
fio.save_json(save_path=save_path,
save_name=name,
config=decodeConfig)
fio.save_pickle(save_path=save_path,
save_name=name,
data=scores_res)
print("Analyzed: {0:s} in {1:.2f} seconds".format(
name,
time.time() - start_time))
def organizer_decode_odor_within_day(condition, decodeConfig, data_path,
save_path):
'''
Run decoding experiments with labels based on the odor that was presented. Data is contained
:param condition: experimental condition. For example, condition OFC.
Must contain fields: name, paths, odors, csp
:param decodeConfig: class config, contains as fields relevant parameters to run decoding experiment
:param data_path:
:param save_path:
:return:
'''
#TODO: see if it works when odors are different for each animal
data_pathnames = sorted(
glob.glob(os.path.join(data_path, '*' + Config.mat_ext)))
config_pathnames = sorted(
glob.glob(os.path.join(data_path, '*' + Config.cons_ext)))
mouse_names_per_file = [
Config.load_cons_f(d).NAME_MOUSE for d in config_pathnames
]
mouse_names, list_of_mouse_ix = np.unique(mouse_names_per_file,
return_inverse=True)
if mouse_names.size != len(condition.paths):
raise ValueError(
"res has {0:d} mice, but filter has {1:d} mice".format(
mouse_names.size, len(condition.paths)))
for i, data_pn in enumerate(data_pathnames):
start_time = time.time()
cons = Config.load_cons_f(config_pathnames[i])
data = Config.load_mat_f(data_pn)
cons_dict = cons.__dict__
for key, value in cons_dict.items():
if isinstance(value, list) or isinstance(value, np.ndarray):
pass
else:
setattr(decodeConfig, key, value)
if hasattr(condition, 'odors'):
odor = condition.odors[list_of_mouse_ix[i]]
else:
odor = condition.dt_odors[list_of_mouse_ix[i]]
if decodeConfig.decode_style == 'identity':
csp = None
else:
if hasattr(condition, 'odors'):
csp = condition.csp[list_of_mouse_ix[i]]
else:
csp = condition.dt_csp[list_of_mouse_ix[i]]
if odor[0] in cons.ODOR_UNIQUE:
scores = decoding.decode_odor_labels(cons, data, odor, csp,
decodeConfig)
mouse = cons.NAME_MOUSE
date = cons.NAME_DATE
plane = cons.NAME_PLANE
name = mouse + '__' + date + '__' + plane
fio.save_json(save_path=save_path,
save_name=name,
config=decodeConfig)
fio.save_numpy(save_path=save_path, save_name=name, data=scores)
print("Analyzed: {0:s} in {1:.2f} seconds".format(
data_pn,
time.time() - start_time))