def _helper(cons, traces):
list_of_masks = _get_odor_masks(cons, chosen_odors, csp_odors, decode_style)
data_trial_cell_time = utils.reshape_data(traces, trial_axis=0, cell_axis=1, time_axis=2,
nFrames=cons.TRIAL_FRAMES)
if average_time:
on = cons.DAQ_O_ON_F
off = cons.DAQ_W_ON_F
if decode_config.no_end_time:
data_trial_cell_time = np.max(data_trial_cell_time[:, :, on:], axis=2, keepdims=True)
else:
data_trial_cell_time = np.max(data_trial_cell_time[:,:,on:off], axis=2, keepdims=True)
licks = cons.DAQ_DATA[:,cons.DAQ_L,:] > 0.5
start = np.round(cons.DAQ_O_ON * cons.DAQ_SAMP).astype(int)
end = np.round(cons.DAQ_W_ON * cons.DAQ_SAMP).astype(int)
lick_boolean = np.sum(licks[start:end], axis=0) > 0
csp_lick = np.bitwise_and(list_of_masks[0], lick_boolean)
csp_nolick = np.bitwise_and(list_of_masks[0], np.invert(lick_boolean))
csm_lick = np.bitwise_and(list_of_masks[1], lick_boolean)
csm_nolick = np.bitwise_and(list_of_masks[1], np.invert(lick_boolean))
train_trials = np.bitwise_or(csp_lick, csm_nolick)
train_labels = _assign_odor_labels([csp_lick, csm_nolick])[train_trials]
train_data = data_trial_cell_time[train_trials]
test_trials = np.bitwise_or(csp_nolick, csm_lick)
test_labels = _assign_odor_labels([csp_nolick, csm_lick])[test_trials]
test_data = data_trial_cell_time[test_trials]
return train_data, train_labels, test_data, test_labels
def _helper(day):
list_of_masks = _get_odor_masks(list_of_cons[day], chosen_odors, csp_odors, decode_style)
data_trial_cell_time = utils.reshape_data(list_of_data[day], trial_axis=0, cell_axis=1, time_axis=2,
nFrames=list_of_cons[day].TRIAL_FRAMES)
good_trials = np.any(list_of_masks, axis=0)
labels = _assign_odor_labels(list_of_masks)[good_trials]
data = data_trial_cell_time[good_trials]
if average_time:
on = list_of_cons[day].DAQ_O_ON_F
off = list_of_cons[day].DAQ_W_ON_F
if decode_config.no_end_time:
data = np.max(data[:, :, on:], axis=2, keepdims=True)
else:
data = np.max(data[:,:,on:off], axis=2, keepdims=True)
return data, labels
def _helper(cons, traces):
list_of_masks = _get_odor_masks(cons, chosen_odors, csp_odors, decode_style)
data_trial_cell_time = utils.reshape_data(traces, trial_axis=0, cell_axis=1, time_axis=2,
nFrames=cons.TRIAL_FRAMES)
if average_time:
on = cons.DAQ_O_ON_F
off = cons.DAQ_W_ON_F
if decode_config.no_end_time:
data_trial_cell_time = np.max(data_trial_cell_time[:, :, on:], axis=2, keepdims=True)
else:
data_trial_cell_time = np.max(data_trial_cell_time[:,:,on:off], axis=2, keepdims=True)
start = np.round(cons.DAQ_O_ON * cons.DAQ_SAMP).astype(int)
end = np.round(cons.DAQ_W_ON * cons.DAQ_SAMP).astype(int)
licks = cons.DAQ_DATA[start:end,cons.DAQ_L,:] > 0.5
licks = np.transpose(licks)
if decode_config.split_style == 'onset':
factor = []
for l in licks:
if np.any(l):
loc = np.where(l)[0][0]
else:
loc = 1000
factor.append(loc)
factor = np.array(factor)
elif decode_config.split_style == 'time':
factor = np.arange(len(licks))
elif decode_config.split_style == 'magnitude':
on_off = np.diff(licks, n=1, axis=1)
factor = np.sum(on_off > 0, axis=1)
else:
raise ValueError('split style {} not recognized'.format(decode_config.split_style))
data = []
labels = []
for mask in list_of_masks:
factor_current_odor = factor[mask]
ixs = np.argsort(factor_current_odor)
cutoff = len(ixs)//2
current_data = data_trial_cell_time[mask][ixs]
current_labels = np.ones_like(ixs)
current_labels[cutoff:] = 2
data.append(current_data)
labels.append(current_labels)
return data, labels
def decode_day_labels(list_of_cons, list_of_data, chosen_odors, csp_odors, decode_config):
#TODO: obsolete, need to fix before using
'''
:param cons:
:param data:
:param chosen_odors:
:param csp_odors:
:param decode_config:
:return: decoding: returns CV scores in dimensions of Time X Cross-Folds X Repeats
'''
decode_style = decode_config.decode_style
decode_neurons = decode_config.neurons
decode_shuffle = decode_config.shuffle
decode_repeat = decode_config.repeat
day_odor_masks = []
for i, cons in enumerate(list_of_cons):
list_of_masks = _get_odor_masks(cons, chosen_odors, csp_odors, decode_style)
day_odor_masks.append(list_of_masks)
odor_day_masks = np.swapaxes(np.array(day_odor_masks), 0, 1)
list_of_reshaped_data = []
for data, cons in zip(list_of_data, list_of_cons):
data_trial_cell_time = utils.reshape_data(data, trial_axis=0, cell_axis=1, time_axis=2,
nFrames=cons.TRIAL_FRAMES)
list_of_reshaped_data.append(data_trial_cell_time)
list_of_scores = []
for i, day_masks in enumerate(odor_day_masks):
cur_data = []
cur_label = []
for j, day_mask in enumerate(day_masks):
cur_data.append(list_of_reshaped_data[j][day_mask])
cur_label.append(np.ones(np.sum(day_mask)) * j)
cur_data = np.concatenate(cur_data, axis=0)
cur_label = np.concatenate(cur_label, axis=0)
for _ in range(decode_repeat):
scores = decode_odors_time_bin(cur_data, cur_label,
number_of_cells=decode_neurons,
shuffle=decode_shuffle,
cv=5)
list_of_scores.append(scores)
out = np.stack(list_of_scores, axis=2)
return out
def decode_odor_labels(cons, data, chosen_odors, csp_odors, decode_config):
'''
:param cons:
:param data:
:param chosen_odors:
:param csp_odors:
:param decode_config:
:return: decoding: returns CV scores in dimensions of Time X Cross-Folds X Repeats
'''
decode_style = decode_config.decode_style
decode_neurons = decode_config.neurons
decode_shuffle = decode_config.shuffle
decode_repeat = decode_config.repeat
decode_average_time = decode_config.average_time
data_trial_cell_time = utils.reshape_data(data, trial_axis=0, cell_axis=1, time_axis=2,
nFrames=cons.TRIAL_FRAMES)
list_of_masks= _get_odor_masks(cons, chosen_odors, csp_odors, decode_style)
good_trials = np.any(list_of_masks, axis=0)
decode_labels = _assign_odor_labels(list_of_masks)[good_trials]
input = data_trial_cell_time[good_trials]
if decode_average_time:
on = cons.DAQ_O_ON_F
if decode_config.no_end_time:
input = np.max(input[:, :, on:], axis=2, keepdims=True)
else:
off = cons.DAQ_W_ON_F
input = np.max(input[:, :, on:off], axis=2, keepdims=True)
list_of_scores = []
for _ in range(decode_repeat):
scores = decode_odors_time_bin(input, decode_labels,
number_of_cells=decode_neurons,
shuffle=decode_shuffle,
cv=5)
list_of_scores.append(scores)
out = np.stack(list_of_scores, axis=2)
return out
def convert(res, condition_config):
d = defaultdict(list)
toConvert = [
'day', 'mouse', 'DAQ_O_ON_F', 'DAQ_O_OFF_F', 'DAQ_W_ON_F',
'TRIAL_FRAMES'
]
for i in range(len(res['mouse'])):
data = res['data'][i]
data = utils.reshape_data(data,
nFrames=res['TRIAL_FRAMES'][i],
cell_axis=0,
trial_axis=1,
time_axis=2)
odor_trials = res['ODOR_TRIALS'][i]
assert data.shape[1] == len(
odor_trials
), 'number of trials in cons does not match trials in data'
mouse = res['mouse'][i]
condition = condition_config.condition
if hasattr(condition, 'odors'):
relevant_odors = copy.copy(condition.odors[mouse])
else:
relevant_odors = copy.copy(condition.pt_odors[mouse] +
condition.dt_odors[mouse])
if condition_config.include_water:
relevant_odors.append('water')
for odor in relevant_odors:
ix = odor == odor_trials
if np.any(ix):
current_data = data[:, ix, :]
d['data'].append(current_data)
d['odor'].append(odor)
for names in toConvert:
d[names].append(res[names][i])
for key, val in d.items():
d[key] = np.array(val)
return d
def helper(res, mouse, days, condition_config):
res_mouse = filter.filter(res, filter_dict={'mouse': mouse, 'day': days})
if hasattr(condition, 'odors'):
odors = condition.odors[mouse]
elif condition_config.period == 'pt':
odors = condition.pt_csp[mouse]
elif condition_config.period == 'dt':
odors = condition.dt_odors[mouse]
else:
raise ValueError('odor condition not recognized')
# sorting step
if not condition_config.independent_sort and condition_config.sort_individually:
odors_copy = copy.copy(odors)
i = condition_config.sort_day_ix
if condition_config.include_water and days[
i] >= condition.training_start_day[mouse]:
odors_copy.append('water')
odor_on = res_mouse['DAQ_O_ON_F'][i]
odor_off = res_mouse['DAQ_O_OFF_F'][i]
water_on = res_mouse['DAQ_W_ON_F'][i]
odor_trials = res_mouse['ODOR_TRIALS'][i]
frames_per_trial = res_mouse['TRIAL_FRAMES'][i]
data = utils.reshape_data(res_mouse['data'][i],
nFrames=frames_per_trial,
cell_axis=0,
trial_axis=1,
time_axis=2)
list_of_psths = []
for j, odor in enumerate(odors):
ix = odor == odor_trials
cur_data = data[:, ix, :]
for k, cell in enumerate(cur_data):
cur_data[k, :, :] = subtract_baseline(
cell, config.baseline_start, odor_on - config.baseline_end)
mean = np.mean(cur_data, axis=1)
list_of_psths.append(mean)
if condition_config.sort_method == 'selectivity':
ixs = sort.sort_by_selectivity(
list_of_psths,
odor_on,
water_on,
condition_config,
delete_nonselective=condition_config.delete_nonselective)
elif condition_config.sort_method == 'onset':
ixs = sort.sort_by_onset(list_of_psths, odor_on, water_on,
condition_config)
elif condition_config.sort_method == 'plus_minus':
ixs = sort.sort_by_plus_minus(list_of_psths, odor_on, water_on,
condition_config)
else:
print('sorting method is not recognized')
# plotting step
images = []
odor_on_times = []
odor_off_times = []
water_on_times = []
list_of_odor_names = []
for i, _ in enumerate(days):
odors_copy = copy.copy(odors)
if condition_config.include_water and days[
i] >= condition.training_start_day[mouse]:
odors_copy.append('water')
odor_on = res_mouse['DAQ_O_ON_F'][i]
water_on = res_mouse['DAQ_W_ON_F'][i]
odor_trials = res_mouse['ODOR_TRIALS'][i]
frames_per_trial = res_mouse['TRIAL_FRAMES'][i]
data = utils.reshape_data(res_mouse['data'][i],
nFrames=frames_per_trial,
cell_axis=0,
trial_axis=1,
time_axis=2)
list_of_psths = []
for odor in odors_copy:
ix = odor == odor_trials
cur_data = data[:, ix, :]
for k, cell in enumerate(cur_data):
cur_data[k, :, :] = subtract_baseline(
cell, config.baseline_start, odor_on - config.baseline_end)
mean = np.mean(cur_data, axis=1)
list_of_psths.append(mean)
psth = np.concatenate(list_of_psths, axis=1)
if condition_config.independent_sort and condition_config.sort_individually:
if condition_config.sort_method == 'selectivity':
ixs = sort.sort_by_selectivity(
list_of_psths[:4],
odor_on,
water_on,
condition_config,
delete_nonselective=condition_config.delete_nonselective)
elif condition_config.sort_method == 'onset':
ixs = sort.sort_by_onset(list_of_psths, odor_on, water_on,
condition_config)
elif condition_config.sort_method == 'plus_minus':
ixs = sort.sort_by_plus_minus(list_of_psths, odor_on, water_on,
condition_config)
else:
raise ValueError('sorting method is not recognized')
sorted_list_of_psths = [x[ixs, :] for x in list_of_psths]
psth = psth[ixs, :]
else:
sorted_list_of_psths = list_of_psths
images.append(psth)
odor_on_times.append(odor_on)
water_on_times.append(water_on)
list_of_odor_names.append(odors_copy)
return images, odor_on_times, water_on_times, list_of_odor_names, sorted_list_of_psths
means = []
for m in mouse:
odor = condition.odors[m][o]
res_mouse = filter.filter(res, filter_dict={'mouse': m, 'day': days})
odor_on = res_mouse['DAQ_O_ON_F'][i]
odor_off = res_mouse['DAQ_O_OFF_F'][i]
water_on = res_mouse['DAQ_W_ON_F'][i]
odor_trials = res_mouse['ODOR_TRIALS'][i]
time_odor_on = 0
time_odor_off = res_mouse['DAQ_O_OFF'][i] - res['DAQ_O_ON'][i]
time_water_on = res_mouse['DAQ_W_ON'][i] - res['DAQ_O_ON'][i]
time = res_mouse['time'][i]
frames_per_trial = res_mouse['TRIAL_FRAMES'][i]
data = utils.reshape_data(res_mouse['data'][i], nFrames= frames_per_trial,
cell_axis=0, trial_axis=1, time_axis=2)
ix = odor == odor_trials
cur_data = data[:, ix, :]
for k, cell in enumerate(cur_data):
cur_data[k,:,:] = subtract_baseline(cell, config.baseline_start, odor_on - config.baseline_end)
mean = np.mean(cur_data, axis=1)
means.append(mean)
list_of_psths.append(np.concatenate(means,axis=0))
ixs = sort_by_selectivity([list_of_psths[-1]], odor_on, water_on, condition_config)
psth = np.concatenate(list_of_psths, axis=1)
psth = psth[ixs,:]
fig = plt.figure(figsize=(1.5, 3))
rect = [.1, .1, .65, .7]
rect_cb = [0.76, 0.1, 0.02, 0.7]
analysis.add_indices(res)
analysis.add_time(res)
odors = condition.odors[mouse]
if condition_config.include_water:
odors.append('water')
for j, cell in enumerate(cells):
res_mouse = filter.filter(res,
filter_dict={
'mouse': mouse,
'day': days[j]
})
for i in range(len(res_mouse['day'])):
data = utils.reshape_data(res_mouse['data'][i],
nFrames=res_mouse['TRIAL_FRAMES'][i],
cell_axis=0,
trial_axis=1,
time_axis=2)
odor_on = res_mouse['DAQ_O_ON_F'][i]
odor_trials = res_mouse['ODOR_TRIALS'][i]
time_odor_on = 0
time_odor_off = res_mouse['DAQ_O_OFF'][i] - res['DAQ_O_ON'][i]
time_water_on = res_mouse['DAQ_W_ON'][i] - res['DAQ_O_ON'][i]
time = res_mouse['time'][i]
fig = plt.figure(figsize=(2, 1.5))
ax = fig.add_axes([.2, .25, .7, .65])
for j, odor in enumerate(odors):
ix = odor == odor_trials
cur_data = data[cell, ix, :]
pretraining_odors = ['oct']
discrimination_odors = ['pin', 'msy', 'euy', 'lim']
csp = ['oct', 'pin', 'msy']
condition = OFC_SINGLE_PHASE() #input your condition
res = analysis.load_data(condition.path)
analysis.add_indices(res)
analysis.add_time(res)
for i, data in enumerate(res['data']):
frames_per_trial = res['TRIAL_FRAMES'][i]
data_r = utils.reshape_data(data,
nFrames=frames_per_trial,
cell_axis=0,
trial_axis=1,
time_axis=2)
res['data'][i] = data_r
print(res.keys())
print(res['day'])
day = 1
print('Imaging day = {}'.format(res['day'][day]))
print('corresponding folder name = {}'.format(res['DIR'][day]))
print('raw data. dimensions are cells X trials X frames = {}'.format(
res['data'][day].shape))
print('frame number for when odor is on. not applicable for water trials = {}'.
format(res['DAQ_O_ON_F'][day]))
print(
'frame number for when odor is off. not applicable for water trials = {}'.
def helper(res, mouse, day, condition_config):
def _pad(data, diff):
newp = np.zeros_like(data)
if diff > 0:
newp[:, :diff] = np.repeat(data[:, -1].reshape(-1, 1),
diff,
axis=1)
newp[:, diff:] = data[:, :-diff]
print('early odor time. mouse: {}, day: {}'.format(
res_mouse['mouse'][0], res_mouse['day'][0]))
else:
newp[:, :diff] = data[:, -diff:]
newp[:, diff:] = np.repeat(data[:, -1].reshape(-1, 1),
-diff,
axis=1)
print('late odor time. mouse: {}, day: {}'.format(
res_mouse['mouse'][0], res_mouse['day'][0]))
return newp
def _align(data, diff):
newp = np.zeros([data.shape[0], data.shape[1] + diff])
newp[:, :data.shape[1]] = data
newp[:, data.shape[1]:] = np.repeat(data[:, -1].reshape(-1, 1),
diff,
axis=1)
print('pad frames. mouse: {}, day: {}'.format(res_mouse['mouse'][0],
res_mouse['day'][0]))
return newp
right_frame = np.max(res['TRIAL_FRAMES'])
right_on = np.median(res['DAQ_O_ON_F'])
res_mouse = filter.filter(res, filter_dict={'mouse': mouse, 'day': day})
if hasattr(condition, 'odors'):
odors = condition.odors[mouse]
elif condition_config.period == 'pt':
odors = ['naive'] + condition.pt_csp[mouse]
elif condition_config.period == 'dt':
odors = condition.dt_odors[mouse]
elif condition_config.period == 'ptdt':
odors = ['naive'] + condition.pt_csp[mouse] + condition.dt_odors[mouse]
else:
raise ValueError('odor condition not recognized')
odors_copy = copy.copy(odors)
if condition_config.include_water and day >= condition.training_start_day[
mouse]:
odors_copy.append('water')
odor_on = res_mouse['DAQ_O_ON_F'][0]
water_on = res_mouse['DAQ_W_ON_F'][0]
odor_trials = res_mouse['ODOR_TRIALS'][0]
frames_per_trial = res_mouse['TRIAL_FRAMES'][0]
data = utils.reshape_data(res_mouse['data'][0],
nFrames=frames_per_trial,
cell_axis=0,
trial_axis=1,
time_axis=2)
list_of_psth = []
odors_out = []
for odor in odors_copy:
ix = odor == odor_trials
if np.any(ix):
cur_data = data[:, ix, :]
for k, cell in enumerate(cur_data):
cur_data[k, :, :] = subtract_baseline(
cell, config.baseline_start, odor_on - config.baseline_end)
mean = np.mean(cur_data, axis=1)
if np.abs(odor_on - right_on) > 2:
diff = (right_on - odor_on).astype(int)
mean = _pad(mean, diff)
odor_on = odor_on + diff
water_on = water_on + diff
if frames_per_trial < right_frame:
diff = right_frame - frames_per_trial
mean = _align(mean, diff)
list_of_psth.append(mean)
odors_out.append(odor)
# if 'naive' in odors_out:
# ix = odors_out.index('oct')
# odors_out.pop(ix)
# list_of_psth.pop(ix)
return list_of_psth, odor_on, water_on, odors_out
def PCA(list_of_cons, list_of_data, odor, csp, pca_config):
def _get_trial_index(cons, pca_style, odor, csp):
if pca_style == 'all':
chosen_odors = odor
elif pca_style == 'csp':
chosen_odors = csp
elif pca_style == 'csm':
chosen_odors = [x for x in odor if not np.isin(x, csp)]
else:
raise ValueError(
'pca style is not recognized {}'.format(pca_style))
labels = cons.ODOR_TRIALS
list_of_ixs = []
for o in chosen_odors:
trial_ixs = np.isin(labels, o)
list_of_ixs.append(trial_ixs)
return list_of_ixs, chosen_odors
pca_nPCs = pca_config.n_principal_components
pca_style = pca_config.style
pca_average = pca_config.average
list_of_reshaped_data = []
for cons, data in zip(list_of_cons, list_of_data):
list_of_ixs, chosen_odors = _get_trial_index(cons, pca_style, odor,
csp)
data_cell_trial_time = utils.reshape_data(data,
trial_axis=1,
cell_axis=0,
time_axis=2,
nFrames=cons.TRIAL_FRAMES)
for ix in list_of_ixs:
cur = data_cell_trial_time[:, ix, :]
if pca_average:
cur = np.mean(cur, axis=1, keepdims=True)
cur_reshaped = cur.reshape(
[cur.shape[0], cur.shape[1] * cur.shape[2]])
list_of_reshaped_data.append(cur_reshaped.transpose())
pca_data = np.concatenate(list_of_reshaped_data, axis=0)
pca = skdecomp.PCA(n_components=pca_nPCs)
pca.fit(pca_data)
variance_explained = pca.explained_variance_ratio_
components = pca.components_
variance = pca.explained_variance_
res = defaultdict(list)
for cons, data in zip(list_of_cons, list_of_data):
list_of_ixs, chosen_odors = _get_trial_index(cons, pca_style, odor,
csp)
data_trial_time_cell = utils.reshape_data(data,
trial_axis=0,
cell_axis=2,
time_axis=1,
nFrames=cons.TRIAL_FRAMES)
for i, ix in enumerate(list_of_ixs):
if any(ix):
o = chosen_odors[i]
cur = data_trial_time_cell[ix]
transformed_data = []
for trial in range(cur.shape[0]):
transformed_data.append(pca.transform(cur[trial]))
transformed_data = np.stack(transformed_data, axis=0)
# data is in format of trials X time X cells
res['data'].append(transformed_data)
res['odor'].append(o)
res['variance_explained'].append(variance_explained)
# res['components'].append(components)
# res['variance'].append(variance)
for key, val in pca_config.__dict__.items():
res[key].append(val)
for key, val in cons.__dict__.items():
if type(val) != list and type(val) != np.ndarray:
res[key].append(val)
for key, val in res.items():
res[key] = np.array(val)
return res, variance_explained