def apply_suggest_bads(raw_fname=None,
raw=None,
config=None,
label="suggest_bads",
fname_out=None):
"""
:param raw_fname:
:param raw:
:param cfg:
:return:
filename,raw-obj
"""
with jumeg_logger.StreamLoggerSTD(label=label):
marked, raw = suggest_bads(
raw) #,**cfg["parameter"]) #show_raw=config.get("show_raw") )
#--- set bads in global HDF
fhdf = os.path.join(config.get("stage"),
config.get("hdfname", "badchannels.hdf5"))
update_bads_in_hdf(fhdf=fhdf,
bads=marked,
fname=raw_fname,
verbose=config.get("verbose"))
return fname_out, raw, True
def apply_interpolate_bads(raw_fname, raw=None, **cfg):
"""
:param raw_fname:
:param raw:
:param cfg:
:return:
filename,raw-obj
"""
fname_out = None
logger.info(" -> apply_interpolate_bad file name: {}".format(raw_fname))
logger.debug(" -> config parameter:\n{}".format(cfg))
jb.verbose = cfg.get("verbose")
if not jb.check_file_extention(fname=raw_fname,
file_extention=cfg.get("file_extention")):
return
#--- return raw_fname,raw
if not cfg.get("run"):
return jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=False,
update_raw_filenname=True,
postfix=cfg.get("postfix", "int"),
overwrite=cfg.get("overwrite", True))
raw, raw_fname = jb.get_raw_obj(raw_fname, raw=raw)
if raw:
logger.info("fname: {}".format(raw_fname))
#--- Interpolate bad channels using jumeg
with jumeg_logger.StreamLoggerSTD(label="interpolate_bads"):
raw = jumeg_interpolate_bads(
raw
) #,**cfg.get("parameter")) #,origin=cfg.get("origin",None),reset_bads=cfg.get("reset_bads",True) )
#-- check results
if cfg.get("plot_block"):
raw.plot(block=cfg.get("plot_block"))
#--- update filename in raw and save if save
fname_out, raw = jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=cfg.get("save"),
postfix=cfg.get("postfix", "int"),
overwrite=cfg.get(
"overwrite", True))
if fname_out:
return fname_out, raw
else:
raise Exception("---> ERROR file name not defined !!!")
def apply_ica_artefact_rejection(self,
raw,
ICA,
fname_raw=None,
fname_clean=None,
replace_pre_whitener=True,
copy_raw=True,
reject=None):
"""
Applies ICA to the raw object. (ica transform)
Parameters
----------
raw : mne.io.Raw() (raw chop)
Raw object ICA is applied to
ica : ICA object
ICA object being applied d to the raw object
fname_raw : str | None
Path for saving the raw object
fname_clean : str | None
Path for saving the ICA cleaned raw object
reject: MNE reject dict
replace_pre_whitener : bool
If True, pre_whitener is replaced when applying ICA to
unfiltered data otherwise the original pre_whitener is used.
copy_raw: make a copy of raw
Returns
-------
raw_clean : mne.io.Raw()
Raw object after ICA cleaning
ica obj, copy & replace mean,std
"""
logger.info("Start ICA Transform => call <apply_ica_replace_mean_std>")
if copy_raw:
_raw = raw.copy()
else:
_raw = raw
raw_clean = None
ica = ICA.copy() # ToDo exclude copy
with jumeg_logger.StreamLoggerSTD(label="apply_ica_replace_mean_std"):
raw_clean = apply_ica_replace_mean_std(_raw,
ica,
picks=self.picks,
reject=reject,
exclude=ica.exclude,
n_pca_components=None)
return raw_clean
def apply_suggest_bads(raw_fname, raw=None, **cfg):
"""
:param raw_fname:
:param raw:
:param cfg:
:return:
filename,raw-obj
"""
fname_out = None
logger.info(" -> apply_suggest_bads file name: {}".format(raw_fname))
logger.debug(" -> config parameter:\n{}".format(cfg))
if not jb.check_file_extention(fname=raw_fname,
file_extention=cfg.get("file_extention")):
return
#--- return raw_fname,raw
if not cfg.get("run"):
return jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=False,
postfix=cfg.get("postfix", "bcc"),
overwrite=cfg.get("overwrite", True))
raw_changed = True
jb.verbose = cfg.get("verbose")
raw, raw_fname = jb.get_raw_obj(raw_fname, raw=raw)
if raw:
with jumeg_logger.StreamLoggerSTD(label="suggest_bads"):
marked, raw = suggest_bads(
raw) #,**cfg["parameter"]) #show_raw=cfg.get("show_raw") )
fname_out, raw = jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=cfg.get("save"),
update_raw_filenname=True,
postfix=cfg.get("postfix", "bcc"),
overwrite=cfg.get(
"overwrite", True))
if fname_out:
return fname_out, raw
else:
raise Exception("---> ERROR file name not defined !!!")
def apply_suggest_bads(raw_fname=None,
raw=None,
config=None,
label="suggest_bads",
fname_out=None):
'''
in config:
suggest_bads:
file_extention:
- ',nr-raw.fif'
- rfDC,nr-empty.fif
fmax: 100
overwrite: false
parameter:
epoch_length: None
fraction: 0.001
sensitivity_psd: 95
sensitivity_steps: 97
show_raw: false
summary_plot: false
validation: true
plot: false
plot_show: false
postfix: bcc
run: true
save: true
:param raw_fname:
:param raw:
:param cfg:
:return:
filename,raw-obj
'''
with jumeg_logger.StreamLoggerSTD(label=label):
marked, raw = suggest_bads(
raw) #,**cfg["parameter"]) #show_raw=config.get("show_raw") )
#--- set bads in global HDF
# fhdf = os.path.join(config.get("stage"),config.get("hdfname","badchannels.hdf5"))
# update_bads_in_hdf(fhdf=fhdf,bads=marked,fname=raw_fname,verbose=config.get("verbose"))
return fname_out, raw, True, None
def apply_interpolate_bads(raw_fname=None,
raw=None,
config=None,
label="interpolate bads",
fname_out=None):
'''
:param raw_fname:
:param raw:
:param cfg:
:return:
filename,raw-obj
'''
#--- Interpolate bad channels using jumeg
with jumeg_logger.StreamLoggerSTD(label=label):
raw = jumeg_interpolate_bads(raw)
#-- check results
if config.get("plot_block"):
raw.plot(block=config.get("plot_block"))
return fname_out, raw, True, None
def apply_noise_reducer(raw_fname, raw=None, **cfg):
'''
apply <magic ee noise reducer> thrice to reference channels with different freq parameters
save PSD plot in subfolder /plots
!!! overwrite raw-obj, works inplace !!!
0) reset bads and check for dead channels
1) apply nr low pass filter for freq below e.g.: 5Hz <reflp>
2) apply nr high pass filter if defined <reflp>
3) apply nr notch filter to remove power line noise <refnotch>
4) save PSD plot
Parameter:
-----------
<noise_reducer> parameter used in this function :
fname_raw : input raw filename
raw : <None>\n
cfg : dict, part of config file <None>
from config file <noise_reducer> part\n
reflp : <None>\n
refhp : <None>\n
refnotch : <None>\n
plot: True
plot_show : True
plot_dir : subdir to save plots
postfix : "nr"
file_extention: ["meeg-raw.fif","rfDC-empty.fif"]
run : True
save : True
overwrite: True
ToDo add parameter extended
<noise_reducer> parameter extended
signals=[], noiseref=[], detrending=None,
tmin=None, tmax=None,
exclude_artifacts=True, checkresults=True, return_raw=False,
complementary_signal=False, fnout=None, verbose=False
Return:
--------
filename,raw-obj
'''
#--- init plot
fname_out = None
logger.info(" -> apply_noise_reducer file name: {}".format(raw_fname))
logger.debug(" -> config parameter:\n{}".format(cfg))
if not jb.check_file_extention(fname=raw_fname,
file_extention=cfg.get("file_extention")):
return
if not cfg.get("run"):
#--- return raw_fname,raw
return jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=False,
postfix=cfg.get("postfix", "nr"),
overwrite=cfg.get("overwrite", True))
logger.info(
" --> preproc noise_reducer for raw file: {}".format(raw_fname))
#--- noise reduction
# apply noise reducer thrice to reference channels with different freq parameters
# !!! overwrite raw-obj !!!
save = False
raw_changed = False
jb.verbose = cfg.get("verbose")
#--- load raw, reset bads
raw, raw_fname = jb.get_raw_obj(raw_fname, raw=raw, reset_bads=True)
#--- check dead channes and mark them as bad
jb.picks.check_dead_channels(raw=raw)
#--- start plot denoising orig raw psd, avoid reloading raw data
if cfg.get("plot"):
jplt = JuMEG_PLOT_PSD(n_plots=2, name="denoising", verbose=True)
jplt.plot(raw,
title="orig: " + os.path.basename(raw_fname),
check_dead_channels=False)
#--- with redirect stdout/err
with jumeg_logger.StreamLoggerSTD(label="noise_reducer"):
#--- 1 nr low pass filter for freq below 5 hz
if cfg.get("reflp"):
raw = noise_reducer(None,
raw=raw,
reflp=cfg.get("reflp"),
return_raw=True,
verbose=cfg.get("verbose"),
exclude_artifacts=False)
raw_changed = True
#--- 2 nr high pass filter
if cfg.get("refhp"):
raw = noise_reducer(None,
raw=raw,
reflp=cfg.get("refhp"),
return_raw=True,
verbose=cfg.get("verbose"),
exclude_artifacts=False)
raw_changed = True
#--- 3 nr notch filter to remove power line noise
if cfg.get("refnotch"):
raw = noise_reducer(None,
raw=raw,
refnotch=cfg.get("refnotch"),
fnout=None,
return_raw=True,
verbose=cfg.get("verbose"),
exclude_artifacts=False)
raw_changed = True
#--- save and update filename in raw
if cfg.get("save"):
save = raw_changed
#--- update filename in raw and save if save
fname_out, raw = jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=save,
update_raw_filenname=True,
postfix=cfg.get("postfix", "nr"),
overwrite=cfg.get(
"overwrite", True))
#--- plot results, avoid reloading raw data
if cfg.get("plot"):
jplt.plot(raw,
title="denoised: " + os.path.basename(fname_out),
check_dead_channels=False)
if cfg.get("plot_show"):
jplt.show()
jplt.save(fname=fname_out, plot_dir=cfg.get("plor_dir", "plots"))
if fname_out:
return fname_out, raw
else:
raise Exception("---> ERROR file name not defined !!!")
def _apply_fit(self, raw_chop=None, chop=None, idx=None):
"""
call to jumeg fit_ica
raw_chop = None
chop = None
ToDo
if not overwrite
if ICA file exist: load ICA
else calc ICA
:return:
ICA obj, ica-filename
"""
ica_obj = None
self._ics_found_svm = None
fname_ica, fname = self._get_chop_name(raw_chop, chop=None)
msg = [
"start ICA FIT chop: {} / {}".format(idx + 1,
self.Chopper.n_chops),
" --> chop id : {}".format(chop),
" -> ica fname : {}".format(fname_ica),
" -> ica chop path: {}".format(self.path_ica_chops),
" -> raw filename : {}".format(fname)
]
logger.info("\n".join(msg))
#--- ck for ovewrite & ICA exist
load_from_disk = False
if not self.cfg.fit.overwrite:
load_from_disk = jb.isFile(fname_ica, path=self.path_ica_chops)
if load_from_disk:
# self._ica_obj,fname_ica = jb.get_raw_obj(fname_ica,path=self.path_ica_chops)
ica_obj, fname_ica = jb.get_raw_obj(fname_ica,
path=self.path_ica_chops)
logger.info(
"DONE LOADING ICA chop form disk: {}\n -> ica filename: {}".
format(chop, fname_ica))
else:
if self.useArtifactRejection:
with jumeg_logger.StreamLoggerSTD(label="ica fit"):
ica_obj = fit_ica(
raw=raw_chop,
picks=self.picks,
reject=self.CFG.GetDataDict(key="reject"),
ecg_ch=self.cfg.ecg.ch_name,
ecg_thresh=self.cfg.ecg.thresh,
flow_ecg=self.cfg.ecg.flow,
fhigh_ecg=self.cfg.ecg.fhigh,
#---
eog_hor=self.cfg.eog.hor_ch,
eog_ver=self.cfg.eog.ver_ch,
flow_eog=self.cfg.eog.flow,
fhigh_eog=self.cfg.eog.fhigh,
eog_thresh=self.cfg.eog.thresh,
#---
use_jumeg=self.cfg.ecg.use_jumeg,
random_state=self.cfg.random_state)
ica_obj.exclude = list(set(ica_obj.exclude))
if self.useSVM:
if not ica_obj:
logger.info('SVM start ICA FIT: init ICA object')
#--- !!! ToDo put parameter in CFG file
ica_obj = ICA(method='fastica',
n_components=40,
random_state=42,
max_pca_components=None,
max_iter=5000,
verbose=False)
ica_obj.fit(raw_chop,
picks=self.picks,
decim=None,
reject=self.CFG.GetDataDict(key="reject"),
verbose=True)
else:
logger.info('SVM ICA Obj start')
#--- !!! do_copy = True => resample
ica_obj, _ = self.SVM.run(raw=self.raw,
ICA=ica_obj,
picks=self.picks,
do_crop=False,
do_copy=True)
logger.info('DONE SVM ICA FIT: apply ICA.fit')
#-- save ica object
if self.cfg.fit.save and not load_from_disk:
logger.info("saving ICA chop : {} / {}\n".format(
idx + 1, self.Chopper.n_chops) +
" -> ica filename : {}".format(fname_ica))
ica_obj.save(os.path.join(self.path_ica_chops, fname_ica))
logger.info(
"done ICA FIT for chop: {}\n".format(chop) +
" -> raw chop filename : {}\n".format(fname_ica) + "-" * 30 +
"\n" + " -> ICs found JuMEG/MNE : {}\n".format(self.SVM.ICsMNE) +
" -> ICs found SVM : {}\n".format(self.SVM.ICsSVM) +
" -> ICs excluded : {}\n".format(ica_obj.exclude) +
"-" * 30 + "\n" +
" -> save ica fit : {}".format(self.cfg.fit.save))
return ica_obj, fname_ica
def apply_noise_reducer(raw_fname=None,
raw=None,
config=None,
label='noise reducer',
fname_out=None):
'''
apply <magic ee noise reducer> thrice to reference channels with different freq parameters
save PSD plot in subfolder /plots
!!! overwrite raw-obj, works inplace !!!
0) reset bads and check for dead channels
1) apply nr low pass filter for freq below e.g.: 5Hz <reflp>
2) apply nr high pass filter if defined <reflp>
3) apply nr notch filter to remove power line noise <refnotch>
4) save PSD plot
IN config
noise_reducer:
file_extention:
- meeg-raw.fif
- rfDC-empty.fif
fmax: 300
noiseref_hp:
- RFG ...
overwrite: false
plot: true
plot_dir: report
plot_show: false
postfix: nr
refhp: 0.1
reflp: 5.0
refnotch:
- 50.0
- 100.0
- 150.0
- 200.0
- 250.0
- 300.0
- 350.0
- 400.0
run: true
save: true
Parameter:
-----------
<noise_reducer> parameter used in this function :
fname_raw : input raw filename
raw : <None>\n
cfg : dict, part of config file <None>
from config file <noise_reducer> part\n
reflp : <None>\n
refhp : <None>\n
refnotch : <None>\n
plot: True
plot_show : True
plot_dir : subdir to save plots
postfix : "nr"
file_extention: ["meeg-raw.fif","rfDC-empty.fif"]
run : True
save : True
overwrite: True
ToDo add parameter extended
<noise_reducer> parameter extended
signals=[], noiseref=[], detrending=None,
tmin=None, tmax=None,
exclude_artifacts=True, checkresults=True, return_raw=False,
complementary_signal=False, fnout=None, verbose=False
Return:
--------
filename,raw-obj
'''
#with JuMEG_PipelineFrame(raw_fname=raw_fname,raw=raw,name="noise reducer",config=cfg) as JPF:
RawIsChanged = False
#--- check dead channes and mark them as bad
jb.picks.check_dead_channels(raw=raw)
#--- start plot denoising orig raw psd, avoid reloading raw data
if config.get("plot"):
jplt = JuMEG_PLOT_PSD(n_plots=3, name="denoising",
verbose=True) #,pick_types=["meg","ref"])
jplt.plot(raw,
color="green",
title="REF: " + os.path.basename(raw_fname),
check_dead_channels=False,
fmax=config.get("fmax"),
picks=jb.picks.ref_nobads(raw))
jplt.plot(raw,
color="blue",
title="MEG orig: " + os.path.basename(raw_fname),
check_dead_channels=False,
fmax=config.get("fmax"),
picks=jb.picks.meg_nobads(raw))
#self.picks = jb.picks.meg_nobads(raw))
#--- with redirect stdout/err
with jumeg_logger.StreamLoggerSTD(label=label):
#--- 1 nr low pass filter for freq below 5 hz
if config.get("reflp"):
raw = noise_reducer(None,
raw=raw,
reflp=config.get("reflp"),
return_raw=True,
verbose=config.get("verbose"),
exclude_artifacts=False)
RawIsChanged = True
#--- 2 nr high pass filter
if config.get("refhp"):
raw = noise_reducer(None,
raw=raw,
reflp=config.get("refhp"),
return_raw=True,
verbose=config.get("verbose"),
exclude_artifacts=False)
RawIsChanged = True
#--- 3 nr notch filter to remove power line noise
if config.get("refnotch"):
raw = noise_reducer(None,
raw=raw,
refnotch=config.get("refnotch"),
fnout=None,
return_raw=True,
verbose=config.get("verbose"),
exclude_artifacts=False)
RawIsChanged = True
#--- plot results, avoid reloading raw data
if config.get("plot"):
jplt.plot(raw,
title="MEG denoised: " + os.path.basename(fname_out),
check_dead_channels=False,
fmax=config.get("fmax"),
picks=jb.picks.meg_nobads(raw))
if config.get("plot_show"):
jplt.show()
fout = jplt.save(fname=fname_out,
plot_dir=config.get("plot_dir", "report"))
#--- update image list in report-config for later update MNE Report
CFG = jCFG()
data = None
report_path = os.path.dirname(fout)
report_config = os.path.join(
report_path,
raw_fname.rsplit("_", 1)[0] + "-report.yaml")
if not CFG.load_cfg(fname=report_config):
data = {"noise_reducer": {"files": os.path.basename(fout)}}
else:
CFG.config["noise_reducer"] = {"files": os.path.basename(fout)}
CFG.save_cfg(fname=report_config, data=data)
return fname_out, raw, RawIsChanged, None
def apply_noise_reducer(raw_fname=None,
raw=None,
config=None,
label="noise reducer",
fname_out=None):
'''
apply <magic ee noise reducer> thrice to reference channels with different freq parameters
save PSD plot in subfolder /plots
!!! overwrite raw-obj, works inplace !!!
0) reset bads and check for dead channels
1) apply nr low pass filter for freq below e.g.: 5Hz <reflp>
2) apply nr high pass filter if defined <reflp>
3) apply nr notch filter to remove power line noise <refnotch>
4) save PSD plot
Parameter:
-----------
<noise_reducer> parameter used in this function :
fname_raw : input raw filename
raw : <None>\n
cfg : dict, part of config file <None>
from config file <noise_reducer> part\n
reflp : <None>\n
refhp : <None>\n
refnotch : <None>\n
plot: True
plot_show : True
plot_dir : subdir to save plots
postfix : "nr"
file_extention: ["meeg-raw.fif","rfDC-empty.fif"]
run : True
save : True
overwrite: True
ToDo add parameter extended
<noise_reducer> parameter extended
signals=[], noiseref=[], detrending=None,
tmin=None, tmax=None,
exclude_artifacts=True, checkresults=True, return_raw=False,
complementary_signal=False, fnout=None, verbose=False
Return:
--------
filename,raw-obj
'''
#with JuMEG_PipelineFrame(raw_fname=raw_fname,raw=raw,name="noise reducer",config=cfg) as JPF:
RawIsChanged = False
#--- check dead channes and mark them as bad
jb.picks.check_dead_channels(raw=raw)
#--- start plot denoising orig raw psd, avoid reloading raw data
if config.get("plot"):
jplt = JuMEG_PLOT_PSD(n_plots=2, name="denoising", verbose=True)
jplt.plot(raw,
title="orig: " + os.path.basename(raw_fname),
check_dead_channels=False,
fmax=config.get("fmax"))
#--- with redirect stdout/err
with jumeg_logger.StreamLoggerSTD(label=label):
#--- 1 nr low pass filter for freq below 5 hz
if config.get("reflp"):
raw = noise_reducer(None,
raw=raw,
reflp=config.get("reflp"),
return_raw=True,
verbose=config.get("verbose"),
exclude_artifacts=False)
RawIsChanged = True
#--- 2 nr high pass filter
if config.get("refhp"):
raw = noise_reducer(None,
raw=raw,
reflp=config.get("refhp"),
return_raw=True,
verbose=config.get("verbose"),
exclude_artifacts=False)
RawIsChanged = True
#--- 3 nr notch filter to remove power line noise
if config.get("refnotch"):
raw = noise_reducer(None,
raw=raw,
refnotch=config.get("refnotch"),
fnout=None,
return_raw=True,
verbose=config.get("verbose"),
exclude_artifacts=False)
RawIsChanged = True
#--- plot results, avoid reloading raw data
if config.get("plot"):
jplt.plot(raw,
title="denoised: " + os.path.basename(fname_out),
check_dead_channels=False,
fmax=config.get("fmax"))
if config.get("plot_show"):
jplt.show()
jplt.save(fname=fname_out, plot_dir=config.get("plor_dir", "plots"))
return fname_out, raw, RawIsChanged