def apply_ica(raw_fname=None,
raw=None,
path=None,
config=None,
label="ica",
fname_out=None):
'''
:param raw_fname:
:param raw:
:param cfg:
:return:
filename,raw-obj,True
'''
if not config.get("run"): return fname_out, raw
jICA = JuMEG_PIPELINES_ICA()
if not path:
path = os.path.dirname(raw_fname)
raw, raw_filtered_clean = jICA.run(raw=raw,
raw_fname=raw_fname,
path=path,
config=config)
if raw_filtered_clean:
raw_filtered_clean.close()
fname_out = jb.get_raw_filename(raw)
return fname_out, raw, True, None
def apply_filter(raw_fname,
raw=None,
config=None,
label="filter",
fname_out=None):
'''
:param raw_fname:
:param raw:
:param cfg:
:return:
filename,raw-obj,True
'''
#-- ini MNE_Filter class
jfi = JuMEG_MNE_FILTER(flow=config.get("flow"), fhigh=config.get("fhigh"))
if not config.get("run"):
return jfi.get_filter_filename(raw=raw), None, False, jfi.postfix
#-- filter inplace ; update file name in raw
jfi.apply(raw=raw,
flow=config.get("flow"),
fhigh=config.get("fhigh"),
picks=None,
save=False,
verbose=config.get("verbose"),
overwrite=config.get("overwrite"))
fname_out = jb.get_raw_filename(raw)
#-- add new postfix from filter e.g: fibp01.0-45.0
return fname_out, raw, True, jfi.postfix
def get_filter_filename(self, raw=None):
"""
Parameters
----------
raw : TYPE, optional
DESCRIPTION. The default is None.
Returns
-------
fname : filename of filtered raw
"""
self._update_postfix()
if raw:
fname = jb.get_raw_filename(raw, index=0)
else:
fname = self.fname
fname, ext = fname.rsplit('-', 1)
fname += "," + self.postfix
if self.dcoffset:
fname += "dc"
fname += "-" + ext
return fname
def _get_chop_name(self,
raw,
chop=None,
extention="-ica.fif",
postfix=None,
fullpath=False):
"""
raw
chop = None
extention= "-ica.fif" [-raw.fif]
postfix = None [ar]
fullpath = True
if True: includes path in filename
Return:
-------
fname chop,fname orig
"""
fname = jb.get_raw_filename(raw)
fname, fextention = op.basename(fname).rsplit('-', 1)
if fullpath:
if fname.startswith(self.path_ica_chops):
fchop = fname
else:
fchop = op.join(self.path_ica_chops, fname)
else:
fchop = os.path.basename(fname)
if postfix:
fchop += "," + postfix
try:
if len(chop):
if np.isnan(chop[1]):
fchop += ',{:04d}-{:04d}'.format(int(chop[0]),
int(self.raw.times[-1]))
else:
fchop += ',{:04d}-{:04d}'.format(int(chop[0]),
int(chop[1]))
except:
pass
if extention:
fchop += extention
return fchop, fname
def copy_crop_and_chop(raw, chop, verbose=False):
'''
chop MNE RAW obj [copy & crop]
Parameters
----------
raw : mne RAW obj
chop : np.array or list [chop start, chop end] in sec
verbose : bool, <False>
Returns
-------
raw_crop : croped RAW obj
'''
raw_crop = raw.copy().crop(tmin=chop[0], tmax=chop[1])
if verbose:
logger.info(
"RAW Crop Annotation : {}\n -> tmin: {} tmax: {}\n {}\n".format(
jb.get_raw_filename(raw), chop[0], chop[1],
raw_crop.annotations))
return raw_crop
def plot(self, **kwargs):
#raw=None,raw_clean=None,ch_name="ECG",event_id=999,picks=None,tmin=-0.4,tmax=0.4,title=None,
# colors=["black","yellow","red","magenta","green"],alpha=0.33,grid=True,show=False,
# scale={"raw":{"factor":10.0**15,"unit":"fT"},"ref":{"factor":10.0**3,"unit":"mV"}},
# offset=0.1,fontsize=12):
"""
:param raw:
:param raw_clean:
:param ch_name:
:param event_id:
:param picks:
:param tmin:
:param tmax:
:param colors:
:param alpha:
:param grid:
:param show:
:param scale: {"raw":{"factor":10**12,"unit":"pT"},"ref":{"factor":10**3,"unit":"mV"}},
:param offset: 0.1
:return:
"""
self._update_from_kwargs(**kwargs)
logger.info("RAW annotations: {}".format(self.raw.annotations))
#--- get epochs calc avgs + ref
annotat = mne.events_from_annotations(
self.raw,
event_id={self.ch_name: self.event_id},
use_rounding=True,
chunk_duration=None)
if not annotat:
logger.error("!!! ERROR No MNE Annotations found: {}\n".format(
jb.get_raw_filename(self.raw)))
return None
evt = annotat[0]
counts = evt.shape[0]
sig_raw, sig_clean, range, t = self._calc_data(self.raw,
self.raw_clean,
evt,
event_id=self.event_id,
tmin=self.tmin,
tmax=self.tmax,
picks=self.picks)
#--- ref channel e.g.: ECG
sig_ref, _, _ = self._calc_signal(self.raw,
evt,
event_id=self.event_id,
tmin=self.tmin,
tmax=self.tmax,
picks=jb.picks.labels2picks(
self.raw, self.ch_name))
if not self.figure:
self.init_plot_parameter()
self._figure = plt.figure()
#self.figure.suptitle(os.path.basename(jb.get_raw_filename(self.raw)),fontsize=12)
#--- subplot(nrows,ncols,idx)
ax1 = plt.subplot(self.n_rows, self.n_cols, self.idx)
#--- sig raw
scl = self.scale.get("raw")
ylim = self._calc_ylimits(ranges=range,
factor=scl.get("factor"),
offset=self.offset)
self._plot(ax1, t, sig_raw * scl.get("factor"), scl.get("unit"),
"black")
#--- sig clean
ax2 = plt.subplot(self.n_rows, self.n_cols, self.idx + self.n_cols)
self._plot(ax2, t, sig_clean * scl.get("factor"), scl.get("unit"),
"black")
#---
scl = self.scale.get("ref")
ax3 = ax1.twinx(
) # instantiate a second axes that shares the same x-axis
color = 'tab:blue'
self._plot(ax3, t, sig_ref * scl.get("factor"), scl.get("unit"), "red")
ax3.tick_params(axis='y', labelcolor=color)
ax3.legend([self.ch_name + " cnts {}".format(counts)],
loc=2,
prop={'size': 8})
ax4 = ax2.twinx(
) # instantiate a second axes that shares the same x-axis
color = 'tab:blue'
self._plot(ax4, t, sig_ref * scl.get("factor"), scl.get("unit"),
"green")
ax4.tick_params(axis='y', labelcolor=color)
ax4.legend(["Clean " + self.ch_name + " cnts {}".format(counts)],
loc=2,
prop={'size': 8})
ax1.set_ylim(ylim[0], ylim[1])
ax2.set_ylim(ylim[0], ylim[1])
#if self.save:
# self.save_figure()
if self.show:
plt.ion()
self.figure.tight_layout()
plt.show()
return self.figure
def update_annotations(self, save=False):
"""
update raw.annotattions with artefact events e.g.: ECG,EOG
save: save raw with annotations Talse
return annotations
"""
evt_annot = None
try:
raw_annot = self.raw.annotations #.copy()
#logger.info("Input Annotations in RAW obj:\n -> {}".format(raw_annot))
orig_time = raw_annot.orig_time
except:
raw_annot = None
orig_time = None #self.raw.info.get("meas_date",None) #self.raw.times[0])
#--- store event info into raw.anotations
# time_format = '%Y-%m-%d %H:%M:%S.%f'
for k in self.events.keys():
msg = ["update raw.annotations: {}".format(k)]
onset = self.events[k]['events'][:, 0] / self.raw.info["sfreq"]
#onset -= self.tmin
duration = np.ones(onset.shape[0]) / self.raw.info[
"sfreq"] # one line in raw.plot
evt_annot = mne.Annotations(
onset=onset.tolist(),
duration=duration.tolist(),
description=
k, # [condition for x in range(evt["events"].shape[0])],
orig_time=orig_time)
if raw_annot:
msg.append(
" --> found mne.annotations in RAW:\n -> {}".format(
raw_annot))
#--- clear old annotations
kidx = np.where(raw_annot.description == k)[0] # get index
if kidx.any():
msg.append(
" -> delete existing annotation {} counts: {}".format(
k, kidx.shape[0]))
raw_annot.delete(kidx)
self.raw.set_annotations(raw_annot + evt_annot)
raw_annot = self.raw.annotations
else:
self.raw.set_annotations(evt_annot)
raw_annot = self.raw.annotations
if save:
f = jb.get_raw_filename(self.raw)
fanato = f.replace("-raw.fif", "-anato.csv")
self.raw.annotations.save(fanato)
msg.append("storing mne.annotations in RAW obj:\n -> {}".format(
self.raw.annotations))
logger.info("\n".join(msg))
return self.raw.annotations
def run(self, **kwargs):
"""
:param kwargs:
:return:
raw_unfiltered_clean,raw_filtered_clean
"""
self._clear()
self._update_from_kwargs(**kwargs)
#--- load config
kwargs["useStruct"] = True
self._CFG.update(**kwargs)
self.useSVM = self.cfg.fit.use_svm
self.useArtifactRejection = self.cfg.fit.use_artifact_rejection
#--- init or load raw
self._initRawObj()
#--- find & store ECG/EOG events in raw.annotations
self._set_ecg_eog_annotations()
#--- chop times
self.Chopper.update(raw=self.raw,
length=self.cfg.chops.length,
and_mask=self.cfg.chops.and_mask,
exit_on_error=self.cfg.chops.exit_on_error,
description=self.cfg.chops.description,
time_window_sec=self.cfg.chops.time_window,
show=self.cfg.chops.show,
verbose=self.verbose,
debug=self.debug)
msg = [
"Apply ICA => FIT & Transform",
" -> filename : {}".format(self._raw_fname),
" -> ica chop path : {}".format(self.path_ica_chops), "-" * 40,
" -> chops [sec] : {}".format(self.Chopper.chops_as_string),
" -> chops [indices]: {}".format(self.Chopper.indices_as_string),
"-" * 40
]
#--- apply pre-filter
if self.cfg.pre_filter.run:
self.PreFilter.apply(flow=self.cfg.pre_filter.flow,
fhigh=self.cfg.pre_filter.fhigh,
save=self.cfg.pre_filter.save,
overwrite=self.cfg.pre_filter.overwrite,
raw=self.raw.copy(),
picks=self.picks,
annotations=self.raw.annotations.copy())
msg = self.PreFilter.GetInfo(msg=msg)
else:
self.PreFilter.clear()
logger.info("\n".join(msg))
ICA_objs = None
raw_filtered_clean = None
raw_unfiltered_clean = None
fimages_filtered = []
fimages_unfiltered = None
#--- apply raw-filter ica-fit,transform, save
if self.PreFilter.isFiltered:
raw_filtered_clean, ICA_objs, fimages_filtered = self._apply(
raw=self.PreFilter.raw,
run_transform=self.cfg.transform.run
and self.cfg.transform.filtered.run,
save_chops=self.cfg.transform.filtered.save_chop,
save_chops_clean=self.cfg.transform.filtered.save_chop_clean,
save_clean=self.cfg.transform.filtered.save)
self.PreFilter.raw.close()
#---apply transform for unfilterd data update data-mean
raw_unfiltered_clean, _, fimages_unfiltered = self._apply(
raw=self.raw,
ICAs=ICA_objs,
run_transform=self.cfg.transform.run
and self.cfg.transform.unfiltered.run,
save_chops=self.cfg.transform.unfiltered.save_chop,
save_chops_clean=self.cfg.transform.unfiltered.save_chop_clean,
save_clean=self.cfg.transform.unfiltered.save)
logger.info("DONE ICA FIT & Transpose\n" +
" -> filename : {}\n".format(
jb.get_raw_filename(raw_unfiltered_clean)) +
" -> time to process :{}".format(
datetime.timedelta(seconds=time.time() -
self._start_time)))
#--- plot
data = {"ICA-FI-AR": None, "ICA-AR": None}
if self.PreFilter.isFiltered:
self.ICAPerformance.plot(
raw=self.PreFilter.raw,
raw_clean=raw_filtered_clean,
plot_path=self.plot_dir,
text=None,
fout=self.PreFilter.fname.rsplit("-", 1)[0] + "-ar")
data["ICA-FI-AR"] = [
self.ICAPerformance.Plot.fout, *fimages_filtered
]
if raw_unfiltered_clean:
self.ICAPerformance.plot(raw=self.raw,
raw_clean=raw_unfiltered_clean,
verbose=True,
text=None,
plot_path=self.plot_dir,
fout=self.raw_fname.rsplit("-", 1)[0] +
"-ar")
data["ICA-AR"] = [
self.ICAPerformance.Plot.fout, *fimages_unfiltered
]
#-- check data shapes orig and transformed
shapes = [self._raw._data.shape]
labels = ["raw original"]
if raw_unfiltered_clean:
shapes.append(raw_unfiltered_clean._data.shape)
labels.append("raw unfiltered_clean")
if raw_filtered_clean:
shapes.append(raw_filtered_clean._data.shape)
labels.append("raw filtered_clean")
if not self.Chopper.compare_data_shapes(shapes, labels):
raise ValueError(" ERROR in chop & crop data: shapes not equal\n")
self._update_report(data)
self.clear(objects=ICA_objs)
return raw_unfiltered_clean, raw_filtered_clean
def _apply(self,
raw=None,
ICAs=None,
run_transform=False,
save_ica=False,
save_chops=False,
save_chops_clean=False,
save_clean=True):
"""
:param raw : raw filtered or unfilterd
:param run_transform : self.cfg.transform.run or self.cfg.transform.filtered.run
:param ICAs : list of ICA objs if None calc ICA fit
:param save_ica : save ICA obj
:param save_chops : self.cfg.transform.unfiltered.save_chop or self.cfg.transform.filtered.save_chop
:param save_chops_clean: self.cfg.transform.unfiltered.save_chop_clean or self.cfg.transform.filtered.save_chop_clean
:param save_clean : self.cfg.transform.filtered.save or self.cfg.transform.unfiltered.save
:return:
raw_clean,ICA_objs
ICAs obj list to transform with unfilterd data if self.PreFilter.isFiltered
titles
images as np.arry
"""
raw_clean = None
ica_objs = []
raw_chops_clean_list = []
fimages = []
for idx in range(self.Chopper.n_chops):
chop = self.Chopper.chops[idx]
logger.info("Start ICA FIT & Transform chop: {} / {}\n".format(
idx + 1, self.Chopper.n_chops))
#--- chop raw
raw_chop = self.Chopper.copy_crop_and_chop(raw, chop)
fname_chop, fname_raw = self._get_chop_name(raw_chop,
chop=chop,
extention="-raw.fif")
jb.set_raw_filename(raw_chop, fname_chop)
#--- ICA fit chop
if ICAs:
ica_obj = ICAs[idx]
else:
ica_obj, fname_ica = self._apply_fit(raw_chop=raw_chop,
chop=chop,
idx=idx)
ica_objs.append(ica_obj)
fname_chop, _ = self._get_chop_name(raw_chop, extention="-raw.fif")
fname_chop = os.path.join(self.path_ica_chops, fname_chop)
if save_chops:
raw_chop.save(fname_chop, overwrite=True)
#--- ICA Transform chop
if run_transform:
fout = jb.get_raw_filename(raw_chop)
raw_cc = self.apply_ica_artefact_rejection(
raw_chop,
ica_obj,
reject=self.CFG.GetDataDict(key="reject"))
raw_chops_clean_list.append(raw_cc)
#--- plot performance
txt = "ICs JuMEG/MNE: "
if self.useSVM:
if self.SVM.ICsMNE:
txt += ",".join([str(i) for i in self.SVM.ICsMNE])
txt += " SVM: {}".format(self.SVM.ICsSVM)
else:
txt += ",".join([str(i) for i in ica_obj.exclude])
# logger.info("raw chop:\n {}".format(raw_chop.annotations))
self.ICAPerformance.plot(raw=raw_chop,
raw_clean=raw_chops_clean_list[-1],
verbose=True,
text=txt,
plot_path=self.plot_dir,
fout=fout.rsplit("-", 1)[0] + "-ar")
fimages.append(self.ICAPerformance.Plot.fout)
if save_chops_clean:
fname_clean, _ = self._get_chop_name(raw_chop,
extention="-raw.fif",
postfix="ar")
fname_clean = os.path.join(self.path_ica_chops,
fname_clean)
raw_chops_clean_list[-1].save(fname_clean, overwrite=True)
logger.info("done ICA FIT & transform chop: {} / {}\n".format(
idx + 1, self.Chopper.n_chops))
#--- concat & save raw chops to raw_clean
if raw_chops_clean_list:
fname_clean = fname_raw.replace("-raw.fif", ",ar-raw.fif")
if not fname_clean.endswith(",ar-raw.fif"):
fname_clean += ",ar-raw.fif"
raw_clean = self.Chopper.concat_and_save(
raw_chops_clean_list,
fname=fname_clean,
annotations=raw.annotations,
save=save_clean)
del (raw_chops_clean_list)
return raw_clean, ica_objs, fimages
def fname(self):
return jb.get_raw_filename(self.raw, index=0)
def apply(self, **kwargs):
"""
wrapper function for MNE version 19.2 notch filter cls
data in raw-obj will be overwritten
filename is updated in raw-obj
call MNE <raw.notch_filter>
notch_filter(self,freqs,picks=None,filter_length='auto',notch_widths=None,trans_bandwidth=1.0,n_jobs=1,method='fir',
iir_params=None,mt_bandwidth=None,p_value=0.05,phase='zero',fir_window='hamming',fir_design='firwin',
pad='reflect_limited',verbose=None)[source]
:param kwargs:
Example
--------
-> notch all chanels 50.0,100.0,150.0 Hz except STIM
from jumeg.base.jumeg_base import jumeg_base as jb
from jumeg.filter.jumeg_mne_filter import JUMEG_NOTCH_FILTER
jNFI = JUMEG_NOTCH_FILTER()
fname = jNFI.apply(
freqs = [50.0,100.0,150.0]
picks = jb.picks.exclude_trigger(raw)
)
Example
--------
-> filter all chanels 0.1 -45.0 Hz except STIM
from jumeg.filter.jumeg_mne_filter import JUMEG_FILTER
jFI = JUMEG_FILTER()
fname = jFI.apply(
flow = 0.1,
fhigh = 45.0,
save = True,
raw = raw,
picks = jb.picks.exclude_trigger(raw) )
:return:
fname
"""
self._update_from_kwargs(**kwargs)
self._is_filtered = False
self._is_reloaded = False
v = jb.verbose
jb.verbose = self.verbose
logger.info("---> Filter start: {}".format(self.fname))
self._update_postfix()
fname, ext = self.fname.rsplit('-', 1) #raw.fif'
fname += "," + self.postfix + "-" + ext
#--- ck if load from disk
if not self.overwrite:
if jb.isFile(fname):
logger.debug("Notch Filtered RAW reloading from disk ...")
self.raw, fname = jb.get_raw_obj(fname, None)
self._is_filtered = True
self._is_reloaded = True
if not self._is_filtered:
logger.info("Notch Filter start MNE filter ...")
if isinstance(self.picks, (list, np.ndarray)):
picks = self.picks
else:
logger.warning(
"picks not defined : excluding channel group <stim> and <resp>"
)
picks = jb.picks.exclude_trigger(self.raw)
self.raw.notch_filter(self.freqs,
picks=picks,
filter_length=self.filter_length,
notch_widths=self.notch_widths,
trans_bandwidth=self.trans_bandwidth,
n_jobs=self.n_jobs,
method=self.method,
iir_params=self.iir_params,
mt_bandwidth=self.mt_bandwidth,
p_value=self.p_value,
phase=self.phase,
fir_window=self.fir_window,
fir_design=self.fir_design,
pad=self.pad,
verbose=self.verbose)
self._fname_orig = jb.get_raw_filename(self.raw)
self._is_filtered = True
if self.save:
logger.info("Notch Filter saving data")
fname = jb.apply_save_mne_data(self.raw,
fname=fname,
overwrite=True)
else:
jb.set_raw_filename(self.raw, fname)
logger.info("Notch Filter done: {}\n".format(self.fname) +
" -> reloaded from disk: {}".format(self._is_reloaded))
jb.verbose = v
return fname
def apply(self, **kwargs):
"""
wrapper function for MNE filter cls
raw is filtered with MNE filter function inplace
data in raw-obj will be overwritten
filename is updated in raw-obj
call MNE filter e.g.:
raw.filter(l_freq=flow,h_freq=fhigh,picks=picks)
208497_INTEXT01_190103_1010_1_c,rfDC,meeg,nr,bcc,int,ar
:param kwargs:
flow,fhigh,raw,picks
Example
--------
-> filter all chanels 0.1 -45.0 Hz except STIM
from jumeg.base.jumeg_base import jumeg_base as jb
from jumeg.filter.jumeg_mne_filter import JUMEG_FILTER
jFI = JUMEG_FILTER()
fname = jFI.apply(
flow = 0.1,
fhigh = 45.0,
save = True,
raw = raw,
picks = jb.picks.exclude_trigger(raw) )
:return:
fname
"""
self._update_from_kwargs(**kwargs)
self._is_filtered = False
self._is_reloaded = False
jb.verbose = self.verbose
logger.info("Filter start: {}".format(self.fname))
fname = self.get_filter_filename()
#--- ck if load from disk
if not self.overwrite:
if jb.isFile(fname):
logger.debug("Filtered RAW reloading from disk ...")
self.raw, fname = jb.get_raw_obj(fname, None)
self._fname_orig = fname
if self.annotations:
self.raw.set_annotations(self.annotations)
self._is_filtered = True
self._is_reloaded = True
if not self._is_filtered:
logger.info("Filter start MNE filter ...")
if isinstance(self.picks, (list, np.ndarray)):
picks = self.picks
else:
logger.warning(
"WARNING: picks not defined : excluding channel group <stim> and <resp>"
)
picks = jb.picks.exclude_trigger(self.raw)
if self.dcoffset:
self.apply_dcoffset()
self.raw.filter(l_freq=self.flow, h_freq=self.fhigh, picks=picks)
self._fname_orig = jb.get_raw_filename(self.raw)
self._is_filtered = True
if self.annotations:
self.raw.set_annotations(self.annotations.copy())
fname, _ = jb.update_and_save_raw(self.raw,
fout=fname,
save=self.save,
overwrite=True,
update_raw_filename=True)
if self.verbose:
self.GetInfo()
return fname