def chop_raw_data(raw, start_time=60.0, stop_time=360.0):
'''
This function extracts specified duration of raw data
and write it into a fif file.
Five mins of data will be extracted by default.
Parameters
----------
raw: Raw object.
start_time: Time to extract data from in seconds. Default is 60.0 seconds.
stop_time: Time up to which data is to be extracted. Default is 360.0 seconds.
'''
# Check if data is longer than required chop duration.
if (raw.n_times / (raw.info['sfreq'])) < (stop_time + 60.0):
logger.info("The data is not long enough.")
return
# Obtain indexes for start and stop times.
assert start_time < stop_time, "Start time is greater than stop time."
start_idx = raw.time_as_index(start_time)
stop_idx = raw.time_as_index(stop_time)
data, times = raw[:, start_idx:stop_idx]
raw._data, raw._times = data, times
dur = int((stop_time - start_time) / 60)
raw.save(raw.info['filename'].split('/')[-1].split('.')[0] + '_' +
str(dur) + 'm.fif')
# For the moment, simply warn.
logger.warning('The file name is not saved in standard form.')
return
def chop_raw_data(raw, start_time=60.0, stop_time=360.0):
'''
This function extracts specified duration of raw data
and write it into a fif file.
Five mins of data will be extracted by default.
Parameters
----------
raw: Raw object.
start_time: Time to extract data from in seconds. Default is 60.0 seconds.
stop_time: Time up to which data is to be extracted. Default is 360.0 seconds.
'''
# Check if data is longer than required chop duration.
if (raw.n_times / (raw.info['sfreq'])) < (stop_time + 60.0):
logger.info("The data is not long enough.")
return
# Obtain indexes for start and stop times.
assert start_time < stop_time, "Start time is greater than stop time."
start_idx = raw.time_as_index(start_time)
stop_idx = raw.time_as_index(stop_time)
data, times = raw[:, start_idx:stop_idx]
raw._data,raw._times = data, times
dur = int((stop_time - start_time) / 60)
raw.save(raw.info['filename'].split('/')[-1].split('.')[0]+'_'+str(dur)+'m.fif')
# For the moment, simply warn.
logger.warning('The file name is not saved in standard form.')
return
def read_log(path):
"""
Read the json file if exists, otherwise, create an "empty" file.
"""
json_fname = op.join(op.dirname(path), 'eeg_cleaner.json')
if op.exists(json_fname):
with open(json_fname, 'r') as f:
logs = json.load(f)
else:
logs = {}
if 'raws' not in logs:
logs['raws'] = {}
if 'epochs' not in logs:
logs['epochs'] = {}
if 'icas' not in logs:
logs['icas'] = {}
git_hash = _get_git_hash()
if 'config' not in logs:
logs['config'] = {'version': git_hash}
else:
prev_hash = logs['config']['version']
if prev_hash != git_hash:
logger.warning(
'The specified subject was cleaned with a previous '
'version of EEG cleaner. ({}). The new version ({}) '
'might fail.'.format(prev_hash, git_hash))
save_log(path, logs)
return logs
def _fit(self, epochs):
if self.parent is None:
raise ValueError('Need a parent to be able to fit')
self._check()
if not hasattr(self.parent, 'data_'):
logger.warning(
'Parent not fit. If this is part of a feature collection, '
'it should be placed after the corresponding estimator.')
self.parent.fit(epochs)
def save(self, fname, overwrite=False):
if not fname.endswith('-markers.hdf5'):
logger.warning('Feature collections file name should end '
'with "-markers.hdf5". Some NICE markers '
'might not work.')
write_hdf5(fname,
list(self.keys()),
title='nice/markers/order',
overwrite=overwrite,
slash='replace')
for meas in self.values():
meas.save(fname, overwrite='update')
def _fit(self, epochs):
if self.numerator is None:
raise ValueError('Need a numerator to be able to fit')
if self.denominator is None:
raise ValueError('Need a denominator to be able to fit')
self._check()
if not hasattr(self.numerator, 'data_'):
logger.warning(
'Numerator not fit. If this is part of a feature collection, '
'it should be placed after the corresponding estimator.')
self.numerator.fit(epochs)
if not hasattr(self.denominator, 'data_'):
logger.warning(
'Denominator not fit. If this is part of a feature collection,'
' it should be placed after the corresponding estimator.')
self.denominator.fit(epochs)
def read_evokeds_hcp(subject, data_type, onset='stim', sensor_mode='mag',
hcp_path=op.curdir):
"""Read HCP processed data
Parameters
----------
subject : str, file_map
The subject
data_type : str
The kind of data to read. The following options are supported:
'rest'
'task_motor'
'task_story_math'
'task_working_memory'
onset : {'stim', 'resp'}
The event onset. Only considered for epochs and evoked outputs
The mapping is generous, everything that is not a response is a
stimulus, in the sense of internal or external events.
sensor_mode : {'mag', 'planar'}
The sensor projection. Defaults to 'mag'. Only relevant for
evoked output.
hcp_path : str
The HCP directory, defaults to op.curdir.
Returns
-------
epochs : instance of mne.Epochs
The MNE epochs. Note, these are pseudo-epochs in the case of
onset == 'rest'.
"""
try:
info = read_info_hcp(subject=subject, data_type=data_type,
hcp_path=hcp_path, run_index=0)
except (ValueError, IOError):
logger.warning('could not find config to complete info.'
'reading only channel positions without transforms.')
info = None
evoked_files = list()
for fname in get_file_paths(
subject=subject, data_type=data_type, onset=onset,
output='evoked', sensor_mode=sensor_mode, hcp_path=hcp_path):
evoked_files.extend(_read_evoked(fname, sensor_mode, info))
return evoked_files
def _make_interpolator(inst, bad_channels):
"""Find indexes and interpolation matrix to interpolate bad channels
Parameters
----------
inst : mne.io.Raw, mne.Epochs or mne.Evoked
The data to interpolate. Must be preloaded.
"""
bads_idx = np.zeros(len(inst.ch_names), dtype=np.bool)
goods_idx = np.zeros(len(inst.ch_names), dtype=np.bool)
picks = pick_types(inst.info, meg=False, eeg=True, exclude=[])
bads_idx[picks] = [inst.ch_names[ch] in bad_channels for ch in picks]
goods_idx[picks] = True
goods_idx[bads_idx] = False
if bads_idx.sum() != len(bad_channels):
logger.warning('Channel interpolation is currently only implemented '
'for EEG. The MEG channels marked as bad will remain '
'untouched.')
pos = get_channel_positions(inst, picks)
# Make sure only EEG are used
bads_idx_pos = bads_idx[picks]
goods_idx_pos = goods_idx[picks]
pos_good = pos[goods_idx_pos]
pos_bad = pos[bads_idx_pos]
# test spherical fit
radius, center = _fit_sphere(pos_good)
distance = np.sqrt(np.sum((pos_good - center) ** 2, 1))
distance = np.mean(distance / radius)
if np.abs(1. - distance) > 0.1:
logger.warning('Your spherical fit is poor, interpolation results are '
'likely to be inaccurate.')
logger.info('Computing interpolation matrix from {0} sensor '
'positions'.format(len(pos_good)))
interpolation = _make_interpolation_matrix(pos_good, pos_bad)
return goods_idx, bads_idx, interpolation
def set_raw_montage_from_locs(raw: Raw,
path_to_locs: str,
show_montage: False = bool) -> Raw:
"""
Reads channels locations from a file and applies them to Raw instance.
Parameters
----------
raw: the Raw instance with missing channel locations
path_to_locs: the full path to the channel locations file
(e.g. Starstim20.locs)
show_montage: whether to show channel locations in a plot
Returns
-------
Raw instance
"""
if Path(path_to_locs).exists():
montage = read_custom_montage(path_to_locs)
# check if there are any channels not in raw instance
missing_channel_locations = [
ch_name for ch_name in raw.ch_names
if ch_name not in montage.ch_names
]
if missing_channel_locations:
logger.info(f"There are {len(missing_channel_locations)} channel "
f"positions not present in the "
f"{Path(path_to_locs).stem} file.")
logger.info(f"Assuming these ({missing_channel_locations}) "
f"are not EEG channels, dropping them from Raw.")
raw.drop_channels(missing_channel_locations)
logger.info("Applying channel locations to Raw instance.")
raw.set_montage(montage)
if show_montage:
raw.plot_sensors(show_names=True)
else:
logger.warning(f"Montage file {path_to_locs} path does not exist! "
f"Returning unmodified raw.")
return raw
def _make_interpolator(inst, bad_channels):
"""Find indexes and interpolation matrix to interpolate bad channels
Parameters
----------
inst : mne.io.Raw, mne.Epochs or mne.Evoked
The data to interpolate. Must be preloaded.
"""
bads_idx = np.zeros(len(inst.ch_names), dtype=np.bool)
goods_idx = np.zeros(len(inst.ch_names), dtype=np.bool)
picks = pick_types(inst.info, meg=False, eeg=True, exclude=[])
bads_idx[picks] = [inst.ch_names[ch] in bad_channels for ch in picks]
goods_idx[picks] = True
goods_idx[bads_idx] = False
if bads_idx.sum() != len(bad_channels):
logger.warning('Channel interpolation is currently only implemented '
'for EEG. The MEG channels marked as bad will remain '
'untouched.')
pos = get_channel_positions(inst, picks)
# Make sure only EEG are used
bads_idx_pos = bads_idx[picks]
goods_idx_pos = goods_idx[picks]
pos_good = pos[goods_idx_pos]
pos_bad = pos[bads_idx_pos]
# test spherical fit
radius, center = _fit_sphere(pos_good)
distance = np.sqrt(np.sum((pos_good - center)**2, 1))
distance = np.mean(distance / radius)
if np.abs(1. - distance) > 0.1:
logger.warning('Your spherical fit is poor, interpolation results are '
'likely to be inaccurate.')
logger.info('Computing interpolation matrix from {0} sensor '
'positions'.format(len(pos_good)))
interpolation = _make_interpolation_matrix(pos_good, pos_bad)
return goods_idx, bads_idx, interpolation
def _prepare_data(self, picks, target):
this_picks = {k: None for k in ['channels', 'epochs']}
if picks is not None:
if any([x not in this_picks.keys() for x in picks.keys()]):
raise ValueError('Picking is not compatible for {}'.format(
self._get_title()))
if picks is None:
picks = {}
if 'frequency' in picks:
logger.warning('Picking in frequency axis is currently not '
'supported. This will not have effect.')
this_picks.update(picks)
to_preserve = self._get_preserve_axis(target)
if len(to_preserve) > 0:
for axis in to_preserve:
this_picks[axis] = None
freqs = self.estimator.freqs_
start = np.searchsorted(freqs, self.fmin, 'left')
end = np.searchsorted(freqs, self.fmax, 'right')
ch_picks = this_picks['channels']
if ch_picks is None:
ch_picks = Ellipsis
epochs_picks = this_picks['epochs']
if epochs_picks is None:
epochs_picks = Ellipsis
this_psds = self.estimator.data_norm_[
..., start:end][:, ch_picks][epochs_picks]
this_freqs = freqs[start:end]
cumulative_spectra = np.cumsum(this_psds, axis=-1)
idx = np.argmin((cumulative_spectra - self.percentile)**2, axis=-1)
if this_psds.ndim > 2:
data = np.zeros_like(idx, dtype=np.float)
for iepoch in range(cumulative_spectra.shape[0]):
data[iepoch] = freqs[idx[iepoch]]
else:
data = this_freqs[idx]
return data
def _prepare_data(self, picks, target):
this_picks = {k: None for k in ['channels', 'epochs']}
if picks is not None:
if any([x not in this_picks.keys() for x in picks.keys()]):
raise ValueError('Picking is not compatible for {}'.format(
self._get_title()))
if picks is None:
picks = {}
if 'frequency' in picks:
logger.warning('Picking in frequency axis is currently not '
'supported. This will not have effect.')
this_picks.update(picks)
to_preserve = self._get_preserve_axis(target)
if len(to_preserve) > 0:
for axis in to_preserve:
this_picks[axis] = None
freqs = self.estimator.freqs_
start = np.searchsorted(freqs, self.fmin, 'left')
end = np.searchsorted(freqs, self.fmax, 'right')
ch_picks = this_picks['channels']
if ch_picks is None:
ch_picks = Ellipsis
epochs_picks = this_picks['epochs']
if epochs_picks is None:
epochs_picks = Ellipsis
if self.normalize:
this_psds = self.estimator.data_norm_[
..., start:end][:, ch_picks][epochs_picks]
else:
this_psds = self.estimator.data_[...,
start:end][:,
ch_picks][epochs_picks]
if self.dB is True and self.normalize is False:
this_psds = 10 * np.log10(this_psds)
return this_psds
def get_mean_amplitude(erp, tmin, tmax, mode):
data = erp.crop(tmin=tmin, tmax=tmax).data
sign_mean_data = data.squeeze()
if mode == "pos":
if not np.any(data > 0):
logger.warning(
f"{erp.comment} :"
"No positive values encountered. Using default mode.")
else:
sign_mean_data = data[data > 0]
elif mode == "neg":
if not np.any(data < 0):
logger.warning(
f"{erp.comment} :"
"No negative values encountered. Using default mode.")
else:
sign_mean_data = data[data < 0]
elif mode == "abs":
sign_mean_data = abs(sign_mean_data)
mean_amplitude = sign_mean_data.mean(axis=0) * 1e6
return mean_amplitude
def save_log(path, logs):
"""
Save the json file, ensuring all the needed keys are there
"""
json_fname = op.join(op.dirname(path), 'eeg_cleaner.json')
if 'raws' not in logs:
logs['raws'] = {}
if 'epochs' not in logs:
logs['epochs'] = {}
if 'icas' not in logs:
logs['icas'] = {}
git_hash = _get_git_hash()
if 'config' not in logs:
logs['config'] = {'version': git_hash}
else:
prev_hash = logs['config']['version']
if prev_hash != git_hash:
logger.warning(
'The specified subject was cleaned with a previous '
'version of EEG cleaner. ({}). The new version ({}) '
'might fail.'.format(prev_hash, git_hash))
with open(json_fname, 'w') as f:
json.dump(logs, f)
def write_labels_to_annot(labels,
subject=None,
parc=None,
overwrite=False,
subjects_dir=None,
annot_fname=None,
colormap='hsv',
hemi='both'):
"""Create a FreeSurfer annotation from a list of labels
FIX: always write both hemispheres
Parameters
----------
labels : list with instances of mne.Label
The labels to create a parcellation from.
subject : str | None
The subject for which to write the parcellation for.
parc : str | None
The parcellation name to use.
overwrite : bool
Overwrite files if they already exist.
subjects_dir : string, or None
Path to SUBJECTS_DIR if it is not set in the environment.
annot_fname : str | None
Filename of the .annot file. If not None, only this file is written
and 'parc' and 'subject' are ignored.
colormap : str
Colormap to use to generate label colors for labels that do not
have a color specified.
hemi : 'both' | 'lh' | 'rh'
The hemisphere(s) for which to write *.annot files (only applies if
annot_fname is not specified; default is 'both').
verbose : bool, str, int, or None
If not None, override default verbose level (see mne.verbose).
Notes
-----
Vertices that are not covered by any of the labels are assigned to a label
named "unknown".
"""
subjects_dir = get_subjects_dir(subjects_dir)
# get the .annot filenames and hemispheres
annot_fname, hemis = _get_annot_fname(annot_fname, subject, hemi, parc,
subjects_dir)
if not overwrite:
for fname in annot_fname:
if op.exists(fname):
raise ValueError('File %s exists. Use "overwrite=True" to '
'overwrite it' % fname)
# prepare container for data to save:
to_save = []
# keep track of issues found in the labels
duplicate_colors = []
invalid_colors = []
overlap = []
no_color = (-1, -1, -1, -1)
no_color_rgb = (-1, -1, -1)
for hemi, fname in zip(hemis, annot_fname):
hemi_labels = [label for label in labels if label.hemi == hemi]
n_hemi_labels = len(hemi_labels)
if n_hemi_labels == 0:
ctab = np.empty((0, 4), dtype=np.int32)
ctab_rgb = ctab[:, :3]
else:
hemi_labels.sort(key=lambda label: label.name)
# convert colors to 0-255 RGBA tuples
hemi_colors = [
no_color if label.color is None else tuple(
int(round(255 * i)) for i in label.color)
for label in hemi_labels
]
ctab = np.array(hemi_colors, dtype=np.int32)
ctab_rgb = ctab[:, :3]
# make color dict (for annot ID, only R, G and B count)
labels_by_color = defaultdict(list)
for label, color in zip(hemi_labels, ctab_rgb):
labels_by_color[tuple(color)].append(label.name)
# check label colors
for color, names in labels_by_color.items():
if color == no_color_rgb:
continue
if color == (0, 0, 0):
# we cannot have an all-zero color, otherw. e.g. tksurfer
# refuses to read the parcellation
msg = ('At least one label contains a color with, "r=0, '
'g=0, b=0" value. Some FreeSurfer tools may fail '
'to read the parcellation')
logger.warning(msg)
if any(i > 255 for i in color):
msg = ("%s: %s (%s)" % (color, ', '.join(names), hemi))
invalid_colors.append(msg)
if len(names) > 1:
msg = "%s: %s (%s)" % (color, ', '.join(names), hemi)
duplicate_colors.append(msg)
# replace None values (labels with unspecified color)
if labels_by_color[no_color_rgb]:
default_colors = _n_colors(n_hemi_labels,
bytes_=True,
cmap=colormap)
safe_color_i = 0 # keep track of colors known to be in hemi_colors
for i in xrange(n_hemi_labels):
if ctab[i, 0] == -1:
color = default_colors[i]
# make sure to add no duplicate color
while np.any(np.all(color[:3] == ctab_rgb, 1)):
color = default_colors[safe_color_i]
safe_color_i += 1
# assign the color
ctab[i] = color
# find number of vertices in surface
if subject is not None and subjects_dir is not None:
fpath = op.join(subjects_dir, subject, 'surf', '%s.white' % hemi)
points, _ = read_surface(fpath)
n_vertices = len(points)
else:
if len(hemi_labels) > 0:
max_vert = max(np.max(label.vertices) for label in hemi_labels)
n_vertices = max_vert + 1
else:
n_vertices = 1
msg = (' Number of vertices in the surface could not be '
'verified because the surface file could not be found; '
'specify subject and subjects_dir parameters.')
logger.warning(msg)
# Create annot and color table array to write
annot = np.empty(n_vertices, dtype=np.intp)
annot[:] = -1
# create the annotation ids from the colors
annot_id_coding = np.array((1, 2**8, 2**16))
annot_ids = list(np.sum(ctab_rgb * annot_id_coding, axis=1))
for label, annot_id in zip(hemi_labels, annot_ids):
# make sure the label is not overwriting another label
if np.any(annot[label.vertices] != -1):
other_ids = set(annot[label.vertices])
other_ids.discard(-1)
other_indices = (annot_ids.index(i) for i in other_ids)
other_names = (hemi_labels[i].name for i in other_indices)
other_repr = ', '.join(other_names)
msg = "%s: %s overlaps %s" % (hemi, label.name, other_repr)
overlap.append(msg)
annot[label.vertices] = annot_id
hemi_names = [label.name for label in hemi_labels]
# Assign unlabeled vertices to an "unknown" label
unlabeled = (annot == -1)
if np.any(unlabeled):
msg = ("Assigning %i unlabeled vertices to "
"'unknown-%s'" % (unlabeled.sum(), hemi))
logger.info(msg)
# find an unused color (try shades of gray first)
for i in range(1, 257):
if not np.any(np.all((i, i, i) == ctab_rgb, 1)):
break
if i < 256:
color = (i, i, i, 0)
else:
err = ("Need one free shade of gray for 'unknown' label. "
"Please modify your label colors, or assign the "
"unlabeled vertices to another label.")
raise ValueError(err)
# find the id
annot_id = np.sum(annot_id_coding * color[:3])
# update data to write
annot[unlabeled] = annot_id
ctab = np.vstack((ctab, color))
hemi_names.append("unknown")
# convert to FreeSurfer alpha values
ctab[:, 3] = 255 - ctab[:, 3]
# remove hemi ending in names
hemi_names = [
name[:-3] if name.endswith(hemi) else name for name in hemi_names
]
to_save.append((fname, annot, ctab, hemi_names))
issues = []
if duplicate_colors:
msg = ("Some labels have the same color values (all labels in one "
"hemisphere must have a unique color):")
duplicate_colors.insert(0, msg)
issues.append('\n'.join(duplicate_colors))
if invalid_colors:
msg = ("Some labels have invalid color values (all colors should be "
"RGBA tuples with values between 0 and 1)")
invalid_colors.insert(0, msg)
issues.append('\n'.join(invalid_colors))
if overlap:
msg = ("Some labels occupy vertices that are also occupied by one or "
"more other labels. Each vertex can only be occupied by a "
"single label in *.annot files.")
overlap.insert(0, msg)
issues.append('\n'.join(overlap))
if issues:
raise ValueError('\n\n'.join(issues))
# write it
for fname, annot, ctab, hemi_names in to_save:
logger.info(' writing %d labels to %s' % (len(hemi_names), fname))
_write_annot(fname, annot, ctab, hemi_names)
logger.info('[done]')
def data_(self):
logger.warning('This attribute should not be accessed directly '
'as it depends on the parent data_ '
'attribute')
return self.parent.data_
def run_tfr_pipeline(source: Path, target: Path, conf: dict):
def _file_exists(file_path: str) -> bool:
return os.path.exists(file_path) and not conf["analysis"]["overwrite"]
def _run_tfr(epochs_tfr):
if mode == "power":
power = compute_power(epochs=epochs_tfr, config=conf["power"])
power.comment = file_path
save_to_hdf5(power=power)
elif mode == "erp":
erp = compute_erp(epochs=epochs_tfr, config=conf["erp"])
erp.comment = str(file_path)
erp.save(f"{file_path}_{mode}-{conf[mode]['postfix']}")
elif mode == "con":
con = compute_connectivity(epochs=epochs_tfr, config=conf["con"])
con.attrs.update(
comment=str(file_path),
ch_names=epochs_tfr.info["ch_names"],
sfreq=epochs_tfr.info["sfreq"],
ch_types=conf["analysis"]["picks"],
)
con.save(
f"{file_path}_{conf[mode]['method']}-{conf[mode]['postfix']}")
else:
pass
files = sorted(list(source.rglob(f"*{EPOCHS_FILE_POSTFIX}")))
if not len(files):
logger.warning(
f"There are no files with the expected {EPOCHS_FILE_POSTFIX}."
f"Doing nothing ...")
return
condition_names = conf["analysis"]["conditions"]
mode = conf["analysis"]["mode"]
metadata = pd.DataFrame(columns=['fid', 'n_epochs'] + condition_names)
pbar = tqdm(sorted(files))
for file in pbar:
pbar.set_description("Processing %s" % file.stem)
fid = "_".join(str(file.stem.replace(" ", "_")).split("_")[:3])
epochs = read_epochs(os.path.join(source, file),
preload=False,
verbose=0)
epochs.load_data().pick_types(**{conf["analysis"]["picks"]: True})
# make sure average ref is applied
if not epochs.info["custom_ref_applied"]:
epochs.load_data().set_eeg_reference("average")
if condition_names:
if isinstance(epochs.metadata, pd.DataFrame):
epochs.metadata = create_metadata(epochs)
epochs.metadata = epochs.metadata.astype(str)
if all(name in epochs.metadata.columns
for name in condition_names):
for condition_values, _ in epochs.metadata.groupby(
condition_names):
if isinstance(condition_values, str):
condition_values = [condition_values]
query = [
f'{a} == "{b}"'
for a, b in zip(condition_names, condition_values)
]
query_str = " & ".join(query)
epochs_query = epochs[query_str]
data = dict(fid=fid,
n_epochs=len(epochs_query),
**dict(
zip(condition_names,
condition_values)),
**conf[mode])
metadata = metadata.append(data, ignore_index=True)
file_name = f"{fid}_{'_'.join(condition_values)}"
floats = re.findall("[-+]?\d*\.\d+", file_name)
if floats:
for num in floats:
file_name = file_name.replace(
num, str(int(float(num))))
file_path = target / file_name
if _file_exists(
file_path=
f"{file_path}_{mode}-{conf[mode]['postfix']}"):
pbar.update(1)
continue
_run_tfr(epochs_tfr=epochs_query)
else:
for event_id in epochs.event_id:
epochs_per_event_id = epochs[event_id]
metadata = metadata.append(dict(
fid=fid,
n_epochs=len(epochs_per_event_id),
event_id=event_id,
**conf[mode]),
ignore_index=True)
file_path = target / f"{fid}_{event_id}"
if _file_exists(f"{file_path}_{mode}-{conf[mode]['postfix']}"):
pbar.update(1)
continue
_run_tfr(epochs_tfr=epochs_per_event_id)
pbar.close()
metadata_file_path = os.path.join(target, f"{mode}_metadata.csv")
metadata.to_csv(metadata_file_path, index=False)
logger.info(f"Metadata file can be found at:\n{metadata_file_path}")
logger.info("\n[PIPELINE FINISHED]")
def combine_meeg(raw_fname, eeg_fname, flow=0.6, fhigh=200,
filter_order=2, njobs=-1):
'''
Functions combines meg data with eeg data. This is done by: -
1. Adjust MEG and EEG data length.
2. Resampling EEG data channels to match sampling
frequency of MEG signals.
3. Write EEG channels into MEG fif file and write to disk.
Parameters
----------
raw_fname: FIF file containing MEG data.
eeg_fname: FIF file containing EEG data.
flow, fhigh: Low and high frequency limits for filtering.
(default 0.6-200 Hz)
filter_order: Order of the Butterworth filter used for filtering.
njobs : Number of jobs.
Warning: Please make sure that the filter settings provided
are stable for both MEG and EEG data.
Only channels ECG 001, EOG 001, EOG 002 and STI 014 are written.
'''
import numpy as np
import mne
from mne.utils import logger
if not raw_fname.endswith('-meg.fif') and \
not eeg_fname.endswith('-eeg.fif'):
logger.warning('Files names are not standard. \
Please use standard file name extensions.')
raw = mne.io.Raw(raw_fname, preload=True)
eeg = mne.io.Raw(eeg_fname, preload=True)
# Filter both signals
filter_type = 'butter'
logger.info('The MEG and EEG signals will be filtered from %s to %s' \
% (flow, fhigh))
picks_fil = mne.pick_types(raw.info, meg=True, eog=True, \
ecg=True, exclude='bads')
raw.filter(flow, fhigh, picks=picks_fil, n_jobs=njobs, method='iir', \
iir_params={'ftype': filter_type, 'order': filter_order})
picks_fil = mne.pick_types(eeg.info, meg=False, eeg=True, exclude='bads')
eeg.filter(flow, fhigh, picks=picks_fil, n_jobs=njobs, method='iir', \
iir_params={'ftype': filter_type, 'order': filter_order})
# Find sync pulse S128 in stim channel of EEG signal.
start_idx_eeg = mne.find_events(eeg, stim_channel='STI 014', \
output='onset')[0, 0]
# Find sync pulse S128 in stim channel of MEG signal.
start_idx_raw = mne.find_events(raw, stim_channel='STI 014', \
output='onset')[0, 0]
# Start times for both eeg and meg channels
start_time_eeg = eeg.times[start_idx_eeg]
start_time_raw = raw.times[start_idx_raw]
# Stop times for both eeg and meg channels
stop_time_eeg = eeg.times[eeg.last_samp]
stop_time_raw = raw.times[raw.last_samp]
# Choose channel with shortest duration (usually MEG)
meg_duration = stop_time_eeg - start_time_eeg
eeg_duration = stop_time_raw - start_time_raw
diff_time = min(meg_duration, eeg_duration)
# Reset both the channel times based on shortest duration
end_time_eeg = diff_time + start_time_eeg
end_time_raw = diff_time + start_time_raw
# Calculate the index of the last time points
stop_idx_eeg = eeg.time_as_index(round(end_time_eeg, 3))[0]
stop_idx_raw = raw.time_as_index(round(end_time_raw, 3))[0]
events = mne.find_events(eeg, stim_channel='STI 014', output='onset',
consecutive=True)
events = events[np.where(events[:, 0] < stop_idx_eeg)[0], :]
events = events[np.where(events[:, 0] > start_idx_eeg)[0], :]
events[:, 0] -= start_idx_eeg
eeg_data, eeg_times = eeg[:, start_idx_eeg:stop_idx_eeg]
_, raw_times = raw[:, start_idx_raw:stop_idx_raw]
# Resample eeg signal
resamp_list = jumeg_resample(raw.info['sfreq'], eeg.info['sfreq'], \
raw_times.shape[0], events=events)
# Update eeg signal
eeg._data, eeg._times = eeg_data[:, resamp_list], eeg_times[resamp_list]
# Update meg signal
raw._data, raw._times = raw[:, start_idx_raw:stop_idx_raw]
raw._first_samps[0] = 0
raw._last_samps[0] = raw._data.shape[1] - 1
# Identify raw channels for ECG, EOG and STI and replace it with relevant data.
logger.info('Only ECG 001, EOG 001, EOG002 and STI 014 will be updated.')
raw._data[raw.ch_names.index('ECG 001')] = eeg._data[0]
raw._data[raw.ch_names.index('EOG 001')] = eeg._data[1]
raw._data[raw.ch_names.index('EOG 002')] = eeg._data[2]
raw._data[raw.ch_names.index('STI 014')] = eeg._data[3]
# Write the combined FIF file to disk.
raw.save(raw_fname.split('-')[0] + '-raw.fif', overwrite=True)
def combine_meeg(raw_fname,
eeg_fname,
flow=0.6,
fhigh=200,
filter_order=2,
njobs=-1):
'''
Functions combines meg data with eeg data. This is done by: -
1. Adjust MEG and EEG data length.
2. Resampling EEG data channels to match sampling
frequency of MEG signals.
3. Write EEG channels into MEG fif file and write to disk.
Parameters
----------
raw_fname: FIF file containing MEG data.
eeg_fname: FIF file containing EEG data.
flow, fhigh: Low and high frequency limits for filtering.
(default 0.6-200 Hz)
filter_order: Order of the Butterworth filter used for filtering.
njobs : Number of jobs.
Warning: Please make sure that the filter settings provided
are stable for both MEG and EEG data.
Only channels ECG 001, EOG 001, EOG 002 and STI 014 are written.
'''
import numpy as np
import mne
from mne.utils import logger
if not raw_fname.endswith('-meg.fif') and \
not eeg_fname.endswith('-eeg.fif'):
logger.warning('Files names are not standard. \
Please use standard file name extensions.')
raw = mne.io.Raw(raw_fname, preload=True)
eeg = mne.io.Raw(eeg_fname, preload=True)
# Filter both signals
filter_type = 'butter'
logger.info('The MEG and EEG signals will be filtered from %s to %s' \
% (flow, fhigh))
picks_fil = mne.pick_types(raw.info, meg=True, eog=True, \
ecg=True, exclude='bads')
raw.filter(flow, fhigh, picks=picks_fil, n_jobs=njobs, method='iir', \
iir_params={'ftype': filter_type, 'order': filter_order})
picks_fil = mne.pick_types(eeg.info, meg=False, eeg=True, exclude='bads')
eeg.filter(flow, fhigh, picks=picks_fil, n_jobs=njobs, method='iir', \
iir_params={'ftype': filter_type, 'order': filter_order})
# Find sync pulse S128 in stim channel of EEG signal.
start_idx_eeg = mne.find_events(eeg, stim_channel='STI 014', \
output='onset')[0, 0]
# Find sync pulse S128 in stim channel of MEG signal.
start_idx_raw = mne.find_events(raw, stim_channel='STI 014', \
output='onset')[0, 0]
# Start times for both eeg and meg channels
start_time_eeg = eeg.times[start_idx_eeg]
start_time_raw = raw.times[start_idx_raw]
# Stop times for both eeg and meg channels
stop_time_eeg = eeg.times[eeg.last_samp]
stop_time_raw = raw.times[raw.last_samp]
# Choose channel with shortest duration (usually MEG)
meg_duration = stop_time_eeg - start_time_eeg
eeg_duration = stop_time_raw - start_time_raw
diff_time = min(meg_duration, eeg_duration)
# Reset both the channel times based on shortest duration
end_time_eeg = diff_time + start_time_eeg
end_time_raw = diff_time + start_time_raw
# Calculate the index of the last time points
stop_idx_eeg = eeg.time_as_index(round(end_time_eeg, 3))[0]
stop_idx_raw = raw.time_as_index(round(end_time_raw, 3))[0]
events = mne.find_events(eeg,
stim_channel='STI 014',
output='onset',
consecutive=True)
events = events[np.where(events[:, 0] < stop_idx_eeg)[0], :]
events = events[np.where(events[:, 0] > start_idx_eeg)[0], :]
events[:, 0] -= start_idx_eeg
eeg_data, eeg_times = eeg[:, start_idx_eeg:stop_idx_eeg]
_, raw_times = raw[:, start_idx_raw:stop_idx_raw]
# Resample eeg signal
resamp_list = jumeg_resample(raw.info['sfreq'], eeg.info['sfreq'], \
raw_times.shape[0], events=events)
# Update eeg signal
eeg._data, eeg._times = eeg_data[:, resamp_list], eeg_times[resamp_list]
# Update meg signal
raw._data, raw._times = raw[:, start_idx_raw:stop_idx_raw]
raw._first_samps[0] = 0
raw._last_samps[0] = raw._data.shape[1] - 1
# Identify raw channels for ECG, EOG and STI and replace it with relevant data.
logger.info('Only ECG 001, EOG 001, EOG002 and STI 014 will be updated.')
raw._data[raw.ch_names.index('ECG 001')] = eeg._data[0]
raw._data[raw.ch_names.index('EOG 001')] = eeg._data[1]
raw._data[raw.ch_names.index('EOG 002')] = eeg._data[2]
raw._data[raw.ch_names.index('STI 014')] = eeg._data[3]
# Write the combined FIF file to disk.
raw.save(raw_fname.split('-')[0] + '-raw.fif', overwrite=True)
def data_(self):
logger.warning('This attribute should not be accessed directly '
'as it depends on the numerator/denominator data_ '
'attribute')
return self.numerator.data_ / self.denominator.data_
def write_labels_to_annot(labels, subject=None, parc=None, overwrite=False,
subjects_dir=None, annot_fname=None,
colormap='hsv', hemi='both'):
"""Create a FreeSurfer annotation from a list of labels
FIX: always write both hemispheres
Parameters
----------
labels : list with instances of mne.Label
The labels to create a parcellation from.
subject : str | None
The subject for which to write the parcellation for.
parc : str | None
The parcellation name to use.
overwrite : bool
Overwrite files if they already exist.
subjects_dir : string, or None
Path to SUBJECTS_DIR if it is not set in the environment.
annot_fname : str | None
Filename of the .annot file. If not None, only this file is written
and 'parc' and 'subject' are ignored.
colormap : str
Colormap to use to generate label colors for labels that do not
have a color specified.
hemi : 'both' | 'lh' | 'rh'
The hemisphere(s) for which to write *.annot files (only applies if
annot_fname is not specified; default is 'both').
verbose : bool, str, int, or None
If not None, override default verbose level (see mne.verbose).
Notes
-----
Vertices that are not covered by any of the labels are assigned to a label
named "unknown".
"""
subjects_dir = get_subjects_dir(subjects_dir)
# get the .annot filenames and hemispheres
annot_fname, hemis = _get_annot_fname(annot_fname, subject, hemi, parc,
subjects_dir)
if not overwrite:
for fname in annot_fname:
if op.exists(fname):
raise ValueError('File %s exists. Use "overwrite=True" to '
'overwrite it' % fname)
# prepare container for data to save:
to_save = []
# keep track of issues found in the labels
duplicate_colors = []
invalid_colors = []
overlap = []
no_color = (-1, -1, -1, -1)
no_color_rgb = (-1, -1, -1)
for hemi, fname in zip(hemis, annot_fname):
hemi_labels = [label for label in labels if label.hemi == hemi]
n_hemi_labels = len(hemi_labels)
if n_hemi_labels == 0:
ctab = np.empty((0, 4), dtype=np.int32)
ctab_rgb = ctab[:, :3]
else:
hemi_labels.sort(key=lambda label: label.name)
# convert colors to 0-255 RGBA tuples
hemi_colors = [no_color if label.color is None else
tuple(int(round(255 * i)) for i in label.color)
for label in hemi_labels]
ctab = np.array(hemi_colors, dtype=np.int32)
ctab_rgb = ctab[:, :3]
# make color dict (for annot ID, only R, G and B count)
labels_by_color = defaultdict(list)
for label, color in zip(hemi_labels, ctab_rgb):
labels_by_color[tuple(color)].append(label.name)
# check label colors
for color, names in labels_by_color.items():
if color == no_color_rgb:
continue
if color == (0, 0, 0):
# we cannot have an all-zero color, otherw. e.g. tksurfer
# refuses to read the parcellation
msg = ('At least one label contains a color with, "r=0, '
'g=0, b=0" value. Some FreeSurfer tools may fail '
'to read the parcellation')
logger.warning(msg)
if any(i > 255 for i in color):
msg = ("%s: %s (%s)" % (color, ', '.join(names), hemi))
invalid_colors.append(msg)
if len(names) > 1:
msg = "%s: %s (%s)" % (color, ', '.join(names), hemi)
duplicate_colors.append(msg)
# replace None values (labels with unspecified color)
if labels_by_color[no_color_rgb]:
default_colors = _n_colors(n_hemi_labels, bytes_=True,
cmap=colormap)
safe_color_i = 0 # keep track of colors known to be in hemi_colors
for i in xrange(n_hemi_labels):
if ctab[i, 0] == -1:
color = default_colors[i]
# make sure to add no duplicate color
while np.any(np.all(color[:3] == ctab_rgb, 1)):
color = default_colors[safe_color_i]
safe_color_i += 1
# assign the color
ctab[i] = color
# find number of vertices in surface
if subject is not None and subjects_dir is not None:
fpath = os.path.join(subjects_dir, subject, 'surf',
'%s.white' % hemi)
points, _ = read_surface(fpath)
n_vertices = len(points)
else:
if len(hemi_labels) > 0:
max_vert = max(np.max(label.vertices) for label in hemi_labels)
n_vertices = max_vert + 1
else:
n_vertices = 1
msg = (' Number of vertices in the surface could not be '
'verified because the surface file could not be found; '
'specify subject and subjects_dir parameters.')
logger.warning(msg)
# Create annot and color table array to write
annot = np.empty(n_vertices, dtype=np.int)
annot[:] = -1
# create the annotation ids from the colors
annot_id_coding = np.array((1, 2 ** 8, 2 ** 16))
annot_ids = list(np.sum(ctab_rgb * annot_id_coding, axis=1))
for label, annot_id in zip(hemi_labels, annot_ids):
# make sure the label is not overwriting another label
if np.any(annot[label.vertices] != -1):
other_ids = set(annot[label.vertices])
other_ids.discard(-1)
other_indices = (annot_ids.index(i) for i in other_ids)
other_names = (hemi_labels[i].name for i in other_indices)
other_repr = ', '.join(other_names)
msg = "%s: %s overlaps %s" % (hemi, label.name, other_repr)
overlap.append(msg)
annot[label.vertices] = annot_id
hemi_names = [label.name for label in hemi_labels]
# Assign unlabeled vertices to an "unknown" label
unlabeled = (annot == -1)
if np.any(unlabeled):
msg = ("Assigning %i unlabeled vertices to "
"'unknown-%s'" % (unlabeled.sum(), hemi))
logger.info(msg)
# find an unused color (try shades of gray first)
for i in range(1, 257):
if not np.any(np.all((i, i, i) == ctab_rgb, 1)):
break
if i < 256:
color = (i, i, i, 0)
else:
err = ("Need one free shade of gray for 'unknown' label. "
"Please modify your label colors, or assign the "
"unlabeled vertices to another label.")
raise ValueError(err)
# find the id
annot_id = np.sum(annot_id_coding * color[:3])
# update data to write
annot[unlabeled] = annot_id
ctab = np.vstack((ctab, color))
hemi_names.append("unknown")
# convert to FreeSurfer alpha values
ctab[:, 3] = 255 - ctab[:, 3]
# remove hemi ending in names
hemi_names = [name[:-3] if name.endswith(hemi) else name
for name in hemi_names]
to_save.append((fname, annot, ctab, hemi_names))
issues = []
if duplicate_colors:
msg = ("Some labels have the same color values (all labels in one "
"hemisphere must have a unique color):")
duplicate_colors.insert(0, msg)
issues.append(os.linesep.join(duplicate_colors))
if invalid_colors:
msg = ("Some labels have invalid color values (all colors should be "
"RGBA tuples with values between 0 and 1)")
invalid_colors.insert(0, msg)
issues.append(os.linesep.join(invalid_colors))
if overlap:
msg = ("Some labels occupy vertices that are also occupied by one or "
"more other labels. Each vertex can only be occupied by a "
"single label in *.annot files.")
overlap.insert(0, msg)
issues.append(os.linesep.join(overlap))
if issues:
raise ValueError('\n\n'.join(issues))
# write it
for fname, annot, ctab, hemi_names in to_save:
logger.info(' writing %d labels to %s' % (len(hemi_names), fname))
_write_annot(fname, annot, ctab, hemi_names)
logger.info('[done]')
def update_log(log_file_path: str, epochs: Epochs, notes: str) -> pd.DataFrame:
"""
Documents the changes during preprocessing for an Epochs object.
Custom description can be added with the notes argument.
Parameters
----------
log_file_path
epochs
notes
Returns
----------
log data
"""
fid = epochs.info["temp"]
dropped_epochs_marker = ["FASTER", "USER", "AUTOREJECT"]
n_bad_epochs = len(
[drop for drop in epochs.drop_log if np.isin(dropped_epochs_marker, drop).any()]
)
stimuli = list(epochs.event_id.keys())
n_epochs_per_stimuli = [
(", ").join([f"{ind}: {len(epochs[ind])}" for ind in stimuli])
]
log = pd.DataFrame(
{
"fid": [fid],
"highpass": [epochs.info["highpass"]],
"lowpass": [epochs.info["lowpass"]],
"n_components": [np.NaN],
"n_bad_epochs": [n_bad_epochs],
"total_drop_percentage": [round(epochs.drop_log_stats(), 2)],
"n_epochs_per_stimuli": n_epochs_per_stimuli,
"stimuli": [list(epochs.event_id.keys())],
"t_min": [epochs.tmin],
"t_max": [epochs.tmax],
"n_interpolated": [np.NaN],
"average_ref_applied": [bool(epochs.info["custom_ref_applied"])],
"baseline": [epochs.baseline if epochs.baseline else np.NaN],
"notes": [notes],
"date_of_update": [datetime.utcnow().isoformat()],
"author": [settings["log"]["author"]],
}
)
description = epochs.info["description"]
if description is None:
logger.warning(
'Failed to update parameters from epochs.info["description"], \n'
"Returning log object, consider adding description field manually and rerunning this function.\n"
'Formatting should match this format: "n_components: 1, interpolated: AF7, Fp2, P3, CP5, Fp1"'
)
return log
if "n_components" in description:
n_components = [x for x in description.split(",")[0] if x.isdigit()][0]
log["n_components"].update(int(n_components))
if "interpolated" in description:
description_plain = description.replace(" ", "").replace(":", "")
interpolated_channels_str = description_plain.split("interpolated")[1]
n_interpolated = len(interpolated_channels_str.split(","))
log["n_interpolated"].update(n_interpolated)
log["interpolated"] = interpolated_channels_str
author_clean = re.sub("\W+", "", settings["log"]["author"])
log_file_name = f"{author_clean}_log.csv"
if os.path.isfile(os.path.join(log_file_path, log_file_name)):
log.to_csv(
os.path.join(log_file_path, log_file_name),
mode="a",
index=False,
header=False,
)
else:
log.to_csv(os.path.join(log_file_path, log_file_name), index=False)
return log