def create_raw_data_for_blender(subject, edf_name, conds, bipolar=False, norm_by_percentile=True,
norm_percs=(3, 97), threshold=0, cm_big='YlOrRd', cm_small='PuBu', flip_cm_big=False,
flip_cm_small=True, moving_average_win_size=0, stat = STAT_DIFF, do_plot=False):
import mne.io
import numpy.matlib as npm
edf_fname = op.join(ELECTRODES_DIR, subject, edf_name)
if not op.isfile(edf_fname):
raise Exception('The EDF file cannot be found in {}!'.format(edf_fname))
edf_raw = mne.io.read_raw_edf(edf_fname, preload=True)
edf_raw.notch_filter(np.arange(60, 241, 60))
dt = (edf_raw.times[1] - edf_raw.times[0])
hz = int(1/ dt)
# T = edf_raw.times[-1] # sec
data_channels, _ = read_electrodes_file(subject, bipolar)
# live_channels = find_live_channels(edf_raw, hz)
# no_stim_channels = get_data_channels(edf_raw)
channels_tup = [(ind, ch) for ind, ch in enumerate(edf_raw.ch_names) if ch in data_channels]
channels_indices = [c[0] for c in channels_tup]
labels = [c[1] for c in channels_tup]
for cond_id, cond in enumerate(conds):
cond_data, times = edf_raw[channels_indices, int(cond['from_t']*hz):int(cond['to_t']*hz)]
C, T = cond_data.shape
if cond_id == 0:
data = np.zeros((C, T, len(conds)))
if cond['name'] == 'baseline':
baseline = np.mean(cond_data, 1)
data[:, :, cond_id] = cond_data
# x_rdp_2 = rdp.rdp(list(zip(t, x)), epsilon=0.2)
if args.ref_elec != '':
ref_ind = labels.index(args.ref_elec)
data -= np.tile(data[ref_ind], (C, 1, 1))
data[:, :, 0] -= np.tile(baseline, (T, 1)).T
data[:, :, 1] -= np.tile(baseline, (T, 1)).T
conditions = [c['name'] for c in conds]
data = utils.normalize_data(data, norm_by_percentile=False)
stat_data = calc_stat_data(data, STAT_DIFF)
fol = op.join(MMVT_DIR, subject, 'electrodes')
output_fname = op.join(fol, 'electrodes{}_data_{}.npz'.format('_bipolar' if bipolar else '', STAT_NAME[stat]))
calc_colors = partial(
utils.mat_to_colors_two_colors_maps, threshold=threshold, cm_big=cm_big, cm_small=cm_small, default_val=1,
flip_cm_big=flip_cm_big, flip_cm_small=flip_cm_small, min_is_abs_max=True, norm_percs=norm_percs)
if moving_average_win_size > 0:
stat_data_mv = utils.moving_avg(stat_data, moving_average_win_size)
colors_mv = calc_colors(stat_data_mv)
np.savez(output_fname, data=data, stat=stat_data_mv, names=labels, conditions=conditions, colors=colors_mv,
times=times)
else:
colors = calc_colors(stat_data)
# try:
# np.savez(output_fname, data=data, names=labels, conditions=conditions, colors=colors, times=times)
# except:
np.savez(op.join(fol, 'electrodes{}_data_{}_meta.npz'.format('_bipolar' if bipolar else '', STAT_NAME[stat])),
names=labels, conditions=conditions, times=times)
np.save(op.join(fol, 'electrodes{}_data_{}_data.npy'.format('_bipolar' if bipolar else '', STAT_NAME[stat])), data)
np.save(op.join(fol, 'electrodes{}_data_{}_colors.npy'.format('_bipolar' if bipolar else '', STAT_NAME[stat])), colors)
def average_all_evoked_responses(root_fol, moving_average_win_size=100, do_plot=False):
import matplotlib.pyplot as plt
for hemi in utils.HEMIS:
print(hemi)
if do_plot:
plt.figure()
evoked_files = glob.glob(op.join(root_fol, 'hc*labels_data_{}.npz'.format(hemi)))
all_data = None
all_data_win = None
for evoked_ind, evoked_fname in enumerate(evoked_files):
print(evoked_fname)
f = np.load(evoked_fname)
data = f['data'] # labels x time x conditions
print(data.shape)
if all_data is None:
all_data = np.zeros((*data.shape, len(evoked_files)))
for cond_ind in range(data.shape[2]):
print(cond_ind)
for label_ind in range(data.shape[0]):
x = data[label_ind, :, cond_ind]
x *= np.sign(x[np.argmax(np.abs(x))])
all_data[label_ind, :, cond_ind, evoked_ind] = x
win_avg = utils.moving_avg(all_data[:, :, cond_ind, evoked_ind], moving_average_win_size)
win_avg = win_avg[:, :-moving_average_win_size]
if do_plot:
for label_data in win_avg:
plt.plot(label_data)
if all_data_win is None:
all_data_win = np.zeros((*win_avg.shape, data.shape[2], len(evoked_files)))
all_data_win[:, :, cond_ind, evoked_ind] = win_avg
if do_plot:
plt.savefig(op.join(root_fol, 'evoked_win_{}_{}.jpg'.format(hemi, moving_average_win_size, hemi)))
plt.close()
mean_evoked = np.mean(all_data, 3)
mean_win_evoked = np.mean(all_data_win, 3)
np.savez(op.join(root_fol, 'healthy_labels_data_{}.npz'.format(hemi)),
data=mean_evoked, names=f['names'], conditions=f['conditions'])
np.savez(op.join(root_fol, 'healthy_labels_data_win_{}_{}.npz'.format(moving_average_win_size, hemi)),
data=mean_win_evoked, names=f['names'], conditions=f['conditions'])
np.savez(op.join(root_fol, 'healthy_labels_all_data_win_{}_{}.npz'.format(moving_average_win_size, hemi)),
data=all_data_win, names=f['names'], conditions=f['conditions'])
shutil.copy(op.join(root_fol, 'healthy_labels_data_win_{}_{}.npz'.format(moving_average_win_size, hemi)),
op.join(root_fol, 'healthy_labels_data_{}.npz'.format(hemi)))
if do_plot:
plt.figure()
for label_data in mean_win_evoked:
plt.plot(label_data)
plt.savefig(op.join(root_fol, 'avg_evoked_win_{}_{}.jpg'.format(hemi, moving_average_win_size, hemi)))
plt.close()
def average_all_evoked_responses(root_fol, moving_average_win_size=100, do_plot=False):
import matplotlib.pyplot as plt
for hemi in utils.HEMIS:
print(hemi)
if do_plot:
plt.figure()
evoked_files = glob.glob(op.join(root_fol, 'hc*labels_data_{}.npz'.format(hemi)))
all_data = None
all_data_win = None
for evoked_ind, evoked_fname in enumerate(evoked_files):
print(evoked_fname)
f = np.load(evoked_fname)
data = f['data'] # labels x time x conditions
print(data.shape)
if all_data is None:
all_data = np.zeros((*data.shape, len(evoked_files)))
for cond_ind in range(data.shape[2]):
print(cond_ind)
for label_ind in range(data.shape[0]):
x = data[label_ind, :, cond_ind]
x *= np.sign(x[np.argmax(np.abs(x))])
all_data[label_ind, :, cond_ind, evoked_ind] = x
win_avg = utils.moving_avg(all_data[:, :, cond_ind, evoked_ind], moving_average_win_size)
win_avg = win_avg[:, :-moving_average_win_size]
if do_plot:
for label_data in win_avg:
plt.plot(label_data)
if all_data_win is None:
all_data_win = np.zeros((*win_avg.shape, data.shape[2], len(evoked_files)))
all_data_win[:, :, cond_ind, evoked_ind] = win_avg
if do_plot:
plt.savefig(op.join(root_fol, 'evoked_win_{}_{}.jpg'.format(hemi, moving_average_win_size, hemi)))
plt.close()
mean_evoked = np.mean(all_data, 3)
mean_win_evoked = np.mean(all_data_win, 3)
np.savez(op.join(root_fol, 'healthy_labels_data_{}.npz'.format(hemi)),
data=mean_evoked, names=f['names'], conditions=f['conditions'])
np.savez(op.join(root_fol, 'healthy_labels_data_win_{}_{}.npz'.format(moving_average_win_size, hemi)),
data=mean_win_evoked, names=f['names'], conditions=f['conditions'])
np.savez(op.join(root_fol, 'healthy_labels_all_data_win_{}_{}.npz'.format(moving_average_win_size, hemi)),
data=all_data_win, names=f['names'], conditions=f['conditions'])
utils.copy_file(op.join(root_fol, 'healthy_labels_data_win_{}_{}.npz'.format(moving_average_win_size, hemi)),
op.join(root_fol, 'healthy_labels_data_{}.npz'.format(hemi)))
if do_plot:
plt.figure()
for label_data in mean_win_evoked:
plt.plot(label_data)
plt.savefig(op.join(root_fol, 'avg_evoked_win_{}_{}.jpg'.format(hemi, moving_average_win_size, hemi)))
plt.close()
def create_raw_data_for_blender(subject, edf_name, conds, bipolar=False, norm_by_percentile=True,
norm_percs=(3, 97), threshold=0, cm_big='YlOrRd', cm_small='PuBu', flip_cm_big=False,
flip_cm_small=True, moving_average_win_size=0, stat = STAT_DIFF, do_plot=False):
import mne.io
edf_fname = op.join(SUBJECTS_DIR, subject, 'electrodes', edf_name)
edf_raw = mne.io.read_raw_edf(edf_fname, preload=True)
edf_raw.notch_filter(np.arange(60, 241, 60))
dt = (edf_raw.times[1] - edf_raw.times[0])
hz = int(1/ dt)
# T = edf_raw.times[-1] # sec
data_channels, _ = read_electrodes_file(subject, bipolar)
# live_channels = find_live_channels(edf_raw, hz)
# no_stim_channels = get_data_channels(edf_raw)
channels_tup = [(ind, ch) for ind, ch in enumerate(edf_raw.ch_names) if ch in data_channels]
channels_indices = [c[0] for c in channels_tup]
labels = [c[1] for c in channels_tup]
for cond_id, cond in enumerate(conds):
cond_data, times = edf_raw[channels_indices, int(cond['from_t']*hz):int(cond['to_t']*hz)]
if cond_id == 0:
data = np.zeros((cond_data.shape[0], cond_data.shape[1], len(conds)))
data[:, :, cond_id] = cond_data
conditions = [c['name'] for c in conds]
data = utils.normalize_data(data, norm_by_percentile=False)
stat_data = calc_stat_data(data, STAT_DIFF)
output_fname = op.join(BLENDER_ROOT_DIR, subject, 'electrodes', 'electrodes{}_data_{}.npz'.format(
'_bipolar' if bipolar else '', STAT_NAME[stat]))
calc_colors = partial(
utils.mat_to_colors_two_colors_maps, threshold=threshold, cm_big=cm_big, cm_small=cm_small, default_val=1,
flip_cm_big=flip_cm_big, flip_cm_small=flip_cm_small, min_is_abs_max=True, norm_percs=norm_percs)
if moving_average_win_size > 0:
stat_data_mv = utils.moving_avg(stat_data, moving_average_win_size)
colors_mv = calc_colors(stat_data_mv)
np.savez(output_fname, data=data, stat=stat_data_mv, names=labels, conditions=conditions, colors=colors_mv,
times=times)
else:
colors = calc_colors(stat_data)
np.savez(output_fname, data=data, names=labels, conditions=conditions, colors=colors, times=times)
def read_electrodes_data_one_mat(subject, mat_file, conditions, stat, output_file_name, bipolar, electrodes_names_field,
field_cond_template, from_t=0, to_t=None, norm_by_percentile=True, norm_percs=(3, 97), threshold=0,
color_map='jet', cm_big='YlOrRd', cm_small='PuBu', flip_cm_big=False, flip_cm_small=True,
moving_average_win_size=0, downsample=2):
# load the matlab file
d = sio.loadmat(mat_file)
# get the labels names
if electrodes_names_field in d:
labels = d[electrodes_names_field]
labels = mu.matlab_cell_str_to_list(labels)
if bipolar and '-' in labels[0]:
labels = fix_bipolar_labels(labels)
# else:
# #todo: change that!!!
# if len(labels) == 1:
# labels = [str(l[0]) for l in labels[0]]
# else:
# labels = [str(l[0][0]) for l in labels]
else:
raise Exception('electrodes_names_field not in the matlab file!')
# Loop for each condition
for cond_id, cond_name in enumerate(conditions):
field = field_cond_template.format(cond_name)
if field not in d:
field = field_cond_template.format(cond_name.title())
if field not in d:
print('{} not in the mat file!'.format(cond_name))
continue
# todo: downsample suppose to be a cmdline paramter
# dims: samples * electrodes * time (single triale) or electrodes * time (evoked)
if to_t == 0 or to_t == -1:
to_t = int(d[field].shape[-1] / downsample)
# initialize the data matrix (electrodes_num x T x 2)
# times = np.arange(0, to_t*2, 2)
cond_data = d[field] # [:, times]
if cond_id == 0:
single_trials = cond_data.ndim == 3
data = {} if single_trials else np.zeros((cond_data.shape[0], to_t - from_t, 2))
if single_trials:
# Different number of trials per condition, can't save it in a matrix
# data[cond_name] = np.zeros((cond_data.shape[0], cond_data.shape[1], to_t - from_t))
cond_data_downsample = utils.downsample_3d(cond_data, downsample)
data[cond_name] = cond_data_downsample[:, :, from_t:to_t]
else:
cond_data_downsample = utils.downsample_2d(cond_data, downsample)
data[:, :, cond_id] = cond_data_downsample[:, from_t:to_t]
if bipolar:
data, labels = bipolarize_data(data, labels)
if single_trials:
output_fname = op.join(MMVT_DIR, subject, 'electrodes', 'electrodes{}_data_st.npz'.format(
'_bipolar' if bipolar else ''))
data_conds = [(data[key], key) for key in data.keys()]
data = [d[0] for d in data_conds]
conditions = [d[1] for d in data_conds]
np.savez(output_fname, data=data, names=labels, conditions=conditions)
else:
data = utils.normalize_data(data, norm_by_percentile, norm_percs)
stat_data = calc_stat_data(data, stat)
calc_colors = partial(
utils.mat_to_colors_two_colors_maps, threshold=threshold, cm_big=cm_big, cm_small=cm_small, default_val=1,
flip_cm_big=flip_cm_big, flip_cm_small=flip_cm_small, min_is_abs_max=True, norm_percs=norm_percs)
if moving_average_win_size > 0:
# data_mv[:, :, cond_id] = utils.downsample_2d(data[:, :, cond_id], moving_average_win_size)
stat_data_mv = utils.moving_avg(stat_data, moving_average_win_size)
colors_mv = calc_colors(stat_data_mv)
np.savez(output_file_name, data=data, stat=stat_data_mv, names=labels, conditions=conditions, colors=colors_mv)
else:
colors = calc_colors(stat_data)
np.savez(output_file_name, data=data, names=labels, conditions=conditions, colors=colors)