def test():
#--- init/update logger
from jumeg.base import jumeg_logger
jumeg_logger.setup_script_logging(logger=logger,level="DEBUG")
jplt = JuMEG_PLOT_PSD(n_plots=2,fmax=300.0,name="denoising",verbose=True)
#--- logfile prefix
p = "$JUMEG_LOCAL_DATA/exp/JUMEGTest/FV/211747"
fraw1 = "211747_FREEVIEW01_180109_1049_1_c,rfDC,meeg-raw.fif"
fraw2 = "211747_FREEVIEW01_180109_1049_1_c,rfDC,meeg,nr-raw.fif"
bads=['MEG 007', 'MEG 010', 'MEG 142', 'MEG 156', 'RFM 011']
raw,fnraw1 = jb.get_raw_obj( os.path.join(p,fraw1) )
jb.verbose = True
jb.update_bad_channels(fnraw1,raw=raw,bads=bads,save=True)
jplt.plot(raw,title=fraw1)
raw,fnraw2 = jb.get_raw_obj( os.path.join(p,fraw2) )
jb.update_bad_channels(fnraw2,raw=raw,bads=bads,save=True)
jplt.plot(raw,title=fraw2)
jplt.show()
plot_name = fraw2.rsplit('-raw.fif')[0] + '-plot'
jplt.save(fname=os.path.join(p,plot_name))
def apply_interpolate_bads(raw_fname, raw=None, **cfg):
"""
:param raw_fname:
:param raw:
:param cfg:
:return:
filename,raw-obj
"""
fname_out = None
logger.info(" -> apply_interpolate_bad file name: {}".format(raw_fname))
logger.debug(" -> config parameter:\n{}".format(cfg))
jb.verbose = cfg.get("verbose")
if not jb.check_file_extention(fname=raw_fname,
file_extention=cfg.get("file_extention")):
return
#--- return raw_fname,raw
if not cfg.get("run"):
return jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=False,
update_raw_filenname=True,
postfix=cfg.get("postfix", "int"),
overwrite=cfg.get("overwrite", True))
raw, raw_fname = jb.get_raw_obj(raw_fname, raw=raw)
if raw:
logger.info("fname: {}".format(raw_fname))
#--- Interpolate bad channels using jumeg
with jumeg_logger.StreamLoggerSTD(label="interpolate_bads"):
raw = jumeg_interpolate_bads(
raw
) #,**cfg.get("parameter")) #,origin=cfg.get("origin",None),reset_bads=cfg.get("reset_bads",True) )
#-- check results
if cfg.get("plot_block"):
raw.plot(block=cfg.get("plot_block"))
#--- update filename in raw and save if save
fname_out, raw = jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=cfg.get("save"),
postfix=cfg.get("postfix", "int"),
overwrite=cfg.get(
"overwrite", True))
if fname_out:
return fname_out, raw
else:
raise Exception("---> ERROR file name not defined !!!")
def test2():
#--- init/update logger
jumeg_logger.setup_script_logging(logger=logger)
raw = None
stage = "$JUMEG_PATH_LOCAL_DATA/exp/MEG94T/mne"
fcfg = os.path.join(stage, "meg94t_config01.yaml")
fpath = "206720/MEG94T0T2/130820_1335/1/"
path = os.path.join(stage, fpath)
#fraw = "206720_MEG94T0T2_130820_1335_1_c,rfDC,meeg,nr,bcc,int,000516-000645-raw.fif"
#fraw_ar = "206720_MEG94T0T2_130820_1335_1_c,rfDC,meeg,nr,bcc,int,000516-000645,ar-raw.fif"
fraw = "206720_MEG94T0T2_130820_1335_1_c,rfDC,meeg,nr,bcc,int,fibp0.10-45.0-raw.fif"
fraw_ar = "206720_MEG94T0T2_130820_1335_1_c,rfDC,meeg,nr,bcc,int,fibp0.10-45.0,ar-raw.fif"
logger.info("JuMEG Pipeline ICA Performance ICA mne-version: {}".format(
mne.__version__))
#---
f = os.path.join(path, fraw)
raw, raw_fname = jb.get_raw_obj(f, raw=None)
raw_path = os.path.dirname(raw_fname)
picks = jb.picks.meg_nobads(raw)
#---
f = os.path.join(path, fraw_ar)
raw_ar, raw_ar_fname = jb.get_raw_obj(f, raw=None)
#--- read config
CFG = jCFG()
CFG.update(config=fcfg)
config = CFG.GetDataDict("ica")
#
jIP = JuMEG_ICA_PERFORMANCE(raw=raw, raw_clean=raw_ar, picks=picks)
#jIP.report()
fout = raw_fname.rsplit("-", 1)[0] + "-ar"
jIP.plot(verbose=True, fout=fout)
def apply_suggest_bads(raw_fname, raw=None, **cfg):
"""
:param raw_fname:
:param raw:
:param cfg:
:return:
filename,raw-obj
"""
fname_out = None
logger.info(" -> apply_suggest_bads file name: {}".format(raw_fname))
logger.debug(" -> config parameter:\n{}".format(cfg))
if not jb.check_file_extention(fname=raw_fname,
file_extention=cfg.get("file_extention")):
return
#--- return raw_fname,raw
if not cfg.get("run"):
return jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=False,
postfix=cfg.get("postfix", "bcc"),
overwrite=cfg.get("overwrite", True))
raw_changed = True
jb.verbose = cfg.get("verbose")
raw, raw_fname = jb.get_raw_obj(raw_fname, raw=raw)
if raw:
with jumeg_logger.StreamLoggerSTD(label="suggest_bads"):
marked, raw = suggest_bads(
raw) #,**cfg["parameter"]) #show_raw=cfg.get("show_raw") )
fname_out, raw = jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=cfg.get("save"),
update_raw_filenname=True,
postfix=cfg.get("postfix", "bcc"),
overwrite=cfg.get(
"overwrite", True))
if fname_out:
return fname_out, raw
else:
raise Exception("---> ERROR file name not defined !!!")
def test():
'''
from jumeg.base.jumeg_base import jumeg_base as jb
from jumeg.base.pipelines.jumeg_pipelines_ica_perfromance import ICAPerformance
from jumeg.base.pipelines.jumeg_base_pipelines_chopper import JuMEG_PIPELINES_CHOPPER,copy_crop_and_chop,concat_and_save
'''
stage = "$JUMEG_TEST_DATA/mne/201772/INTEXT01/190212_1334/2"
fn = "201772_INTEXT01_190212_1334_2_c,rfDC,meeg,nr,bcc,int-raw.fif"
#stage="/media/fboers/USB_2TB/exp/INTEXT/mne/208548/INTEXT01/181023_1355/1"
#fn="208548_INTEXT01_181023_1355_1_c,rfDC,meeg,nr,bcc,int-raw.fif"
fin = os.path.join(stage, fn)
raw, fname = jb.get_raw_obj(fname=fin)
#--- ck for annotations in raw
try:
annota = raw.annotations
except:
from jumeg.base.pipelines.jumeg_pipelines_ica_perfromance import ICAPerformance
IP = ICAPerformance()
#--- find ECG
IP.ECG.find_events(raw=raw)
IP.ECG.GetInfo(debug=True)
#--- find EOG
IP.EOG.find_events(raw=raw)
IP.EOG.GetInfo(debug=True)
jCP = JuMEG_PIPELINES_CHOPPER()
jCP.update(raw=raw, verbose=True, debug=True, show=True)
#--- test chop crop
raw_chops = []
# raw = jCP.stim_raw
for chop in jCP.chops:
raw_chop = copy_crop_and_chop(raw=raw, chop=chop)
raw_chops.append(raw_chop)
#--- concat chpos
raw_concat = concat_and_save(raw_chops, annotations=raw.annotations)
if not compare_data(raw._data[0], raw_concat._data[0], verbose=False):
logger.exception("Error raw and raw_concat not equal")
sys.exit()
def apply_resample(raw_fname, raw=None, **cfg):
"""
:param raw_fname:
:param raw:
:param cfg:
:return:
filename,raw-obj
"""
return
logger.info(" -> apply_resample file name: {}".format(raw_fname))
logger.debug(" -> config parameter:\n{}".format(cfg))
if not jb.check_file_extention(fname=raw_fname,
file_extention=cfg.get("file_extention")):
return
#--- return raw_fname,raw
if not cfg.get("run"):
return jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=False,
postfix=cfg.get("postfix", "res"),
overwrite=cfg.get("overwrite", True))
#--- catch stdout,stderr
#jumeg_logger.log_stdout(label="filter")
#jumeg_logger.log_stderr(label="filter")
jb.verbose = cfg.get("verbose")
raw, raw_fname = jb.get_raw_obj(raw_fname, raw=raw)
#--- ToDo setup resampling
#raw,raw_fname = jumeg_mne_fileter(raw)
raw_changed = True
#--- save and update filename in raw
#if raw_changed:
# fname_out,raw = jb.update_and_save_raw(raw,fin=raw_fname,fout=None,save=cfg.get("save"),update_raw_filenname=True,
# postfix=cfg.get("postfix","bcc"),overwrite=cfg.get("overwrite",True))
#--- return back stdout/stderr from logger
#jumeg_logger.log_stdout(reset=True)
#jumeg_logger.log_stderr(reset=True)
return fname_out, raw
def _initRawObj(self):
"""
load or get RAW obj
init & mkdir path tree <stage>/../ica/chops
init picks from RAW
init report HDF file name
"""
self._raw, self._raw_fname = jb.get_raw_obj(self.raw_fname,
raw=self.raw)
self._raw_path = os.path.dirname(self._raw_fname)
if self.stage:
self._raw_path = os.join(self.stage, self._raw_path)
#---
mkpath(self.path_ica_chops, mode=0o770)
#--- get picks from raw
self._picks = jb.picks.meg_nobads(self._raw)
def update_from_kwargs(self,**kwargs):
if kwargs.get("raw",None):
self.raw = kwargs.get("raw")
elif kwargs.get("fname",None): # full filepath
self.raw,self._fname = jb.get_raw_obj(kwargs.get("fname"),raw=None)
self.do_crop = kwargs.get("do_crop",self.do_crop)
self.do_copy = kwargs.get("do_copy",self.do_copy)
self.tmin = kwargs.get("tmin", self.tmin)
self.tmax = kwargs.get("tmax", self.tmax)
#---
self.sfreq = kwargs.get("sfreq", self.sfreq)
self.n_jobs = kwargs.get("n_jobs",self.n_jobs)
self.l_freq = kwargs.get("l_freq",self.l_freq)
self.h_freq = kwargs.get("h_freq",self.h_freq)
self.threshold = kwargs.get("threshold",self.threshold)
self.n_components = kwargs.get("n_components",self.n_components)
self.method = kwargs.get("method",self.method)
self.picks = kwargs.get("picks",self.picks)
self.ICA = kwargs.get("ICA",self.ICA)
def test1():
#--- init/update logger
jumeg_logger.setup_script_logging(logger=logger)
raw = None
stage = "$JUMEG_PATH_LOCAL_DATA/exp/MEG94T/mne"
fcfg = os.path.join(stage, "meg94t_config01.yaml")
fpath = "206720/MEG94T0T2/130820_1335/1/"
path = os.path.join(stage, fpath)
raw_fname = "206720_MEG94T0T2_130820_1335_1_c,rfDC,meeg,nr,bcc,int-raw.fif"
logger.info("JuMEG Pipeline ICA Performance ICA mne-version: {}".format(
mne.__version__))
f = os.path.join(path, raw_fname)
raw, raw_fname = jb.get_raw_obj(f, raw=None)
raw_path = os.path.dirname(raw_fname)
#--- get picks from raw
picks = jb.picks.meg_nobads(raw)
#---
CFG = jCFG()
CFG.update(config=fcfg)
config = CFG.GetDataDict("ica")
#--
ICAPerformance = JuMEG_ICA_PERFORMANCE(
raw=raw,
path=path,
fname=raw_fname,
)
#--- find ECG
ICAPerformance.ECG.find_events(raw=raw, **config.get("ecg"))
ICAPerformance.ECG.GetInfo(debug=True)
#--- find EOG
ICAPerformance.EOG.find_events(raw=raw, **config.get("eog"))
ICAPerformance.EOG.GetInfo(debug=True)
def apply_noise_reducer(raw_fname, raw=None, **cfg):
'''
apply <magic ee noise reducer> thrice to reference channels with different freq parameters
save PSD plot in subfolder /plots
!!! overwrite raw-obj, works inplace !!!
0) reset bads and check for dead channels
1) apply nr low pass filter for freq below e.g.: 5Hz <reflp>
2) apply nr high pass filter if defined <reflp>
3) apply nr notch filter to remove power line noise <refnotch>
4) save PSD plot
Parameter:
-----------
<noise_reducer> parameter used in this function :
fname_raw : input raw filename
raw : <None>\n
cfg : dict, part of config file <None>
from config file <noise_reducer> part\n
reflp : <None>\n
refhp : <None>\n
refnotch : <None>\n
plot: True
plot_show : True
plot_dir : subdir to save plots
postfix : "nr"
file_extention: ["meeg-raw.fif","rfDC-empty.fif"]
run : True
save : True
overwrite: True
ToDo add parameter extended
<noise_reducer> parameter extended
signals=[], noiseref=[], detrending=None,
tmin=None, tmax=None,
exclude_artifacts=True, checkresults=True, return_raw=False,
complementary_signal=False, fnout=None, verbose=False
Return:
--------
filename,raw-obj
'''
#--- init plot
fname_out = None
logger.info(" -> apply_noise_reducer file name: {}".format(raw_fname))
logger.debug(" -> config parameter:\n{}".format(cfg))
if not jb.check_file_extention(fname=raw_fname,
file_extention=cfg.get("file_extention")):
return
if not cfg.get("run"):
#--- return raw_fname,raw
return jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=False,
postfix=cfg.get("postfix", "nr"),
overwrite=cfg.get("overwrite", True))
logger.info(
" --> preproc noise_reducer for raw file: {}".format(raw_fname))
#--- noise reduction
# apply noise reducer thrice to reference channels with different freq parameters
# !!! overwrite raw-obj !!!
save = False
raw_changed = False
jb.verbose = cfg.get("verbose")
#--- load raw, reset bads
raw, raw_fname = jb.get_raw_obj(raw_fname, raw=raw, reset_bads=True)
#--- check dead channes and mark them as bad
jb.picks.check_dead_channels(raw=raw)
#--- start plot denoising orig raw psd, avoid reloading raw data
if cfg.get("plot"):
jplt = JuMEG_PLOT_PSD(n_plots=2, name="denoising", verbose=True)
jplt.plot(raw,
title="orig: " + os.path.basename(raw_fname),
check_dead_channels=False)
#--- with redirect stdout/err
with jumeg_logger.StreamLoggerSTD(label="noise_reducer"):
#--- 1 nr low pass filter for freq below 5 hz
if cfg.get("reflp"):
raw = noise_reducer(None,
raw=raw,
reflp=cfg.get("reflp"),
return_raw=True,
verbose=cfg.get("verbose"),
exclude_artifacts=False)
raw_changed = True
#--- 2 nr high pass filter
if cfg.get("refhp"):
raw = noise_reducer(None,
raw=raw,
reflp=cfg.get("refhp"),
return_raw=True,
verbose=cfg.get("verbose"),
exclude_artifacts=False)
raw_changed = True
#--- 3 nr notch filter to remove power line noise
if cfg.get("refnotch"):
raw = noise_reducer(None,
raw=raw,
refnotch=cfg.get("refnotch"),
fnout=None,
return_raw=True,
verbose=cfg.get("verbose"),
exclude_artifacts=False)
raw_changed = True
#--- save and update filename in raw
if cfg.get("save"):
save = raw_changed
#--- update filename in raw and save if save
fname_out, raw = jb.update_and_save_raw(raw,
fin=raw_fname,
fout=None,
save=save,
update_raw_filenname=True,
postfix=cfg.get("postfix", "nr"),
overwrite=cfg.get(
"overwrite", True))
#--- plot results, avoid reloading raw data
if cfg.get("plot"):
jplt.plot(raw,
title="denoised: " + os.path.basename(fname_out),
check_dead_channels=False)
if cfg.get("plot_show"):
jplt.show()
jplt.save(fname=fname_out, plot_dir=cfg.get("plor_dir", "plots"))
if fname_out:
return fname_out, raw
else:
raise Exception("---> ERROR file name not defined !!!")
def apply(self, **kwargs):
"""
wrapper function for MNE filter cls
raw is filtered with MNE filter function inplace
data in raw-obj will be overwritten
filename is updated in raw-obj
call MNE filter e.g.:
raw.filter(l_freq=flow,h_freq=fhigh,picks=picks)
208497_INTEXT01_190103_1010_1_c,rfDC,meeg,nr,bcc,int,ar
:param kwargs:
flow,fhigh,raw,picks
Example
--------
-> filter all chanels 0.1 -45.0 Hz except STIM
from jumeg.base.jumeg_base import jumeg_base as jb
from jumeg.filter.jumeg_mne_filter import JUMEG_FILTER
jFI = JUMEG_FILTER()
fname = jFI.apply(
flow = 0.1,
fhigh = 45.0,
save = True,
raw = raw,
picks = jb.picks.exclude_trigger(raw) )
:return:
fname
"""
self._update_from_kwargs(**kwargs)
self._is_filtered = False
self._is_reloaded = False
jb.verbose = self.verbose
logger.info("Filter start: {}".format(self.fname))
fname = self.get_filter_filename()
#--- ck if load from disk
if not self.overwrite:
if jb.isFile(fname):
logger.debug("Filtered RAW reloading from disk ...")
self.raw, fname = jb.get_raw_obj(fname, None)
self._fname_orig = fname
if self.annotations:
self.raw.set_annotations(self.annotations)
self._is_filtered = True
self._is_reloaded = True
if not self._is_filtered:
logger.info("Filter start MNE filter ...")
if isinstance(self.picks, (list, np.ndarray)):
picks = self.picks
else:
logger.warning(
"WARNING: picks not defined : excluding channel group <stim> and <resp>"
)
picks = jb.picks.exclude_trigger(self.raw)
if self.dcoffset:
self.apply_dcoffset()
self.raw.filter(l_freq=self.flow, h_freq=self.fhigh, picks=picks)
self._fname_orig = jb.get_raw_filename(self.raw)
self._is_filtered = True
if self.annotations:
self.raw.set_annotations(self.annotations.copy())
fname, _ = jb.update_and_save_raw(self.raw,
fout=fname,
save=self.save,
overwrite=True,
update_raw_filename=True)
if self.verbose:
self.GetInfo()
return fname
def apply(self, **kwargs):
"""
wrapper function for MNE version 19.2 notch filter cls
data in raw-obj will be overwritten
filename is updated in raw-obj
call MNE <raw.notch_filter>
notch_filter(self,freqs,picks=None,filter_length='auto',notch_widths=None,trans_bandwidth=1.0,n_jobs=1,method='fir',
iir_params=None,mt_bandwidth=None,p_value=0.05,phase='zero',fir_window='hamming',fir_design='firwin',
pad='reflect_limited',verbose=None)[source]
:param kwargs:
Example
--------
-> notch all chanels 50.0,100.0,150.0 Hz except STIM
from jumeg.base.jumeg_base import jumeg_base as jb
from jumeg.filter.jumeg_mne_filter import JUMEG_NOTCH_FILTER
jNFI = JUMEG_NOTCH_FILTER()
fname = jNFI.apply(
freqs = [50.0,100.0,150.0]
picks = jb.picks.exclude_trigger(raw)
)
Example
--------
-> filter all chanels 0.1 -45.0 Hz except STIM
from jumeg.filter.jumeg_mne_filter import JUMEG_FILTER
jFI = JUMEG_FILTER()
fname = jFI.apply(
flow = 0.1,
fhigh = 45.0,
save = True,
raw = raw,
picks = jb.picks.exclude_trigger(raw) )
:return:
fname
"""
self._update_from_kwargs(**kwargs)
self._is_filtered = False
self._is_reloaded = False
v = jb.verbose
jb.verbose = self.verbose
logger.info("---> Filter start: {}".format(self.fname))
self._update_postfix()
fname, ext = self.fname.rsplit('-', 1) #raw.fif'
fname += "," + self.postfix + "-" + ext
#--- ck if load from disk
if not self.overwrite:
if jb.isFile(fname):
logger.debug("Notch Filtered RAW reloading from disk ...")
self.raw, fname = jb.get_raw_obj(fname, None)
self._is_filtered = True
self._is_reloaded = True
if not self._is_filtered:
logger.info("Notch Filter start MNE filter ...")
if isinstance(self.picks, (list, np.ndarray)):
picks = self.picks
else:
logger.warning(
"picks not defined : excluding channel group <stim> and <resp>"
)
picks = jb.picks.exclude_trigger(self.raw)
self.raw.notch_filter(self.freqs,
picks=picks,
filter_length=self.filter_length,
notch_widths=self.notch_widths,
trans_bandwidth=self.trans_bandwidth,
n_jobs=self.n_jobs,
method=self.method,
iir_params=self.iir_params,
mt_bandwidth=self.mt_bandwidth,
p_value=self.p_value,
phase=self.phase,
fir_window=self.fir_window,
fir_design=self.fir_design,
pad=self.pad,
verbose=self.verbose)
self._fname_orig = jb.get_raw_filename(self.raw)
self._is_filtered = True
if self.save:
logger.info("Notch Filter saving data")
fname = jb.apply_save_mne_data(self.raw,
fname=fname,
overwrite=True)
else:
jb.set_raw_filename(self.raw, fname)
logger.info("Notch Filter done: {}\n".format(self.fname) +
" -> reloaded from disk: {}".format(self._is_reloaded))
jb.verbose = v
return fname
def _on_enter(self, **kwargs):
"""
:return:
raw_fname, raw obj
"""
self._update_from_kwargs(**kwargs)
#print(Fore.GREEN)
logger.info("Start: < {} > file name: {}".format(
self.label, self._raw_fname))
#print(Style.RESET_ALL)
if self._debug:
logger.debug("config parameter:\n{}".format(self._cfg))
# self._raw = None
self._run = True
self._fname_out = None
#--- check file extention in list
if not jb.check_file_extention(
fname=self._raw_fname,
file_extention=self._cfg.get("file_extention")):
logger.info(
"preproc {}: SKIPPED : <file extention not in extention list>\n"
.format(self.label) + self.info())
self._run = False
self._raw = None
return None, None
self._fname_out, self._raw = jb.update_and_save_raw(
self._raw,
fin=self._raw_fname,
fout=None,
save=False,
postfix=self.postfix,
overwrite=False)
#--- return raw_fname,raw
if not self._cfg.get("run"):
logger.info(
"preproc {}: SKIPPED : <run> is False\n".format(self.label) +
self.info())
self._run = False
self._raw = None
return self._raw_fname, None
#--- if file exist and do not overwrite
if os.path.isfile(self._fname_out) and (self._cfg.get(
"overwrite", False) == False):
logger.info(
"preproc {}: SKIPPED : do not overwrite existing output file\n"
.format(self.label) + self.info())
self._run = False
self._raw = None
return self._fname_out, None
#--- OK load raw, reset bads
self._raw, self._raw_fname = jb.get_raw_obj(
self._raw_fname, raw=self._raw, reset_bads=self._reset_bads)
logger.info("preproc {}\n".format(self.label) + self.info())
return self._raw_fname, self._raw
def apply_fit(self, **kwargs):
self._clear()
self._update_from_kwargs(**kwargs)
#--- load config
self._CFG.update(**kwargs)
#--- init or load raw
self._raw, self._raw_fname = jb.get_raw_obj(self.raw_fname,
raw=self.raw)
#--- get picks from raw
self._picks = jb.picks.meg_nobads(self._raw)
#--- chop times
if self.cfg.chops.epocher.use:
"""ToDo use epocher information chop onset,offset"""
pass
else:
self._calc_chop_times()
if not isinstance(self._chop_times, (np.ndarray)):
logger.error("---> No <chop times> defined for ICA\n" +
" -> raw filename : {}\n".format(self._raw_fname))
return None
#--- ck for 1.filter => filter inplace:
if self.cfg.pre_filter.run:
filename = self._FiPre.apply(flow=self.cfg.pre_filter.flow,
fhigh=self.cfg.pre_filter.fhigh,
save=self.cfg.pre_filter.save,
raw=self.raw,
picks=self._picks)
self._raw_isfiltered = True
else:
filename = self._raw_fname
fname, fextention = op.basename(filename).rsplit('-', 1) #raw.fif
path_ica = op.join(os.path.dirname(filename), "ica")
path_ica_chops = op.join(path_ica, "chops")
mkpath(path_ica_chops, mode=0o770)
msg = [
"---> Apply ICA => FIT ICA -> mkdirs\n -> ica : {}\n -> chops : {}\n -> filename: {}\n -> raw filename: {}"
.format(path_ica, path_ica_chops, fname, self._raw_fname),
" -> is filtered: {}".format(self._raw_isfiltered)
]
logger.info("\n".join(msg))
for idx in range(self._chop_times.shape[0]):
chop = self._chop_times[idx]
ica_fname = op.join(path_ica_chops, fname)
if np.isnan(chop[1]):
ica_fname = fname + ',{:06d}-{:06d}-ica.fif'.format(
int(chop[0]), int(self.raw.times[-1]))
else:
ica_fname = fname + ',{:06d}-{:06d}-ica.fif'.format(
int(chop[0]), int(chop[1]))
logger.info("---> Start ICA chop: {} / {}\n".format(
idx + 1, self._chop_times.shape[0]) +
" --> chop id : {}\n".format(chop) +
" -> ica fname : {}\n".format(ica_fname) +
" -> ica chop path: {}\n".format(path_ica_chops) +
" -> raw filename : {}\n".format(self._raw_fname))
if self._chop_times.shape[0] > 1:
raw_chop = self._raw.copy().crop(tmin=chop[0], tmax=chop[1])
else:
raw_chop = self._raw
#--- get dict from struct
reject = self.CFG.GetDataDict(key="reject")
self._ica_obj = fit_ica(raw=raw_chop,
picks=self._picks,
reject=reject,
ecg_ch=self.cfg.ecg.channel,
eog_hor=self.cfg.eog.hor_ch,
eog_ver=self.cfg.eog.ver_ch,
flow_ecg=self.cfg.ecg.flow,
fhigh_ecg=self.cfg.ecg.fhigh,
flow_eog=self.cfg.eog.flow,
fhigh_eog=self.cfg.eog.fhigh,
ecg_thresh=self.cfg.ecg.thresh,
eog_thresh=self.cfg.eog.thresh,
use_jumeg=self.cfg.ecg.use_jumeg,
random_state=self.cfg.random_state)
#--- save ica object
if self.cfg.save:
self._ica_obj.save(os.path.join(path_ica_chops, ica_fname))
logger.info(
"---> DONE ICA chop: {} / {} -> ica filename: {}".format(
idx + 1, self._chop_times.shape[0], ica_fname))
logger.info("---> DONE ICA FITs: {} -> raw filename: {}".format(
self._chop_times.shape[0], self._raw_fname))
def apply_filter(opt):
"""
apply jumeg filter with argparser options
load fif file (mne) get raw-obj
calc FIR FFT filter [bp,hp,lp,notches] for raw-obj data
save raw-obj to new file
filter_method : mne => fft mne-filter
bw => fft butterwoth
ws => fft - windowed sinc
Parameter
---------
opt: argparser option obj
Result
-------
raw-obj
"""
raw = None
fout = None
jb.verbose = opt.verbose
#--- check if fullfile exists
fin = jb.isfile(opt.fif_filename,
path=opt.fif_stage,
head="apply_suggest_bads: FIF File Check",
exit_on_error=False)
if not fin: return
#--- generate filter obj reset no defaults
filter_obj = jumeg_filter(filter_method=opt.method,
filter_type=opt.type,
fcut1=None,
fcut2=None,
remove_dcoffset=False,
notch=np.array([]),
notch_width=1.0,
order=4)
# !!! Py3 type(opt.fcut ) return < cls str>
#--- check and set opt parameter
if jb.isNotEmpty(opt.fcut1): filter_obj.fcut1 = float(opt.fcut1)
#---
if jb.isNotEmpty(opt.fcut2): filter_obj.fcut2 = float(opt.fcut2)
#---
if jb.isNotEmpty(opt.order): filter_obj.order = float(opt.order)
#--- calc notch array e.g. 50,100,150 .. max
if opt.notch:
filter_obj.calc_notches(opt.notch, opt.notch_max)
if jb.isNotEmpty(opt.notch_width):
filter_obj.notch_width = float(opt.notch_width)
#---
filter_obj.filter_window = opt.window
filter_obj.remove_dcoffset = opt.remove_dcoffset
filter_obj.verbose = opt.verbose
if opt.method.lower() == "mne":
filter_obj.mne_filter_method = opt.mne_method
if jb.isNotEmpty(opt.mne_length):
filter_obj.mne_filter_length = float(opt.mne_length)
if jb.isNotEmpty(opt.mne_trans_bandwith):
filter_obj.trans_bandwith = float(opt.mne_trans_bandwith)
#--- load fif
logger.info(" --> jumeg filter load MEG data: " + fin)
raw, _ = jb.get_raw_obj(fin, raw=raw)
#--- get MEG & Ref picks
picks = jb.picks.exclude_trigger(raw)
filter_obj.sampling_frequency = raw.info['sfreq']
#--- apply filter
logger.info(" --> jumeg filter apply filter")
filter_obj.apply_filter(raw._data, picks=picks)
logger.info(" --> jumeg filter info: " + filter_obj.filter_info)
filter_obj.update_info_filter_settings(
raw) # raw.info lowpass and highpass
#--- make output filename
name_raw = fin.split('-')[0]
fout = name_raw + "," + filter_obj.filter_name_postfix + opt.file_extention
if opt.save:
fout = jb.apply_save_mne_data(raw, fname=fout)
return raw
def _apply_fit(self, raw_chop=None, chop=None, idx=None):
"""
call to jumeg fit_ica
raw_chop = None
chop = None
ToDo
if not overwrite
if ICA file exist: load ICA
else calc ICA
:return:
ICA obj, ica-filename
"""
ica_obj = None
self._ics_found_svm = None
fname_ica, fname = self._get_chop_name(raw_chop, chop=None)
msg = [
"start ICA FIT chop: {} / {}".format(idx + 1,
self.Chopper.n_chops),
" --> chop id : {}".format(chop),
" -> ica fname : {}".format(fname_ica),
" -> ica chop path: {}".format(self.path_ica_chops),
" -> raw filename : {}".format(fname)
]
logger.info("\n".join(msg))
#--- ck for ovewrite & ICA exist
load_from_disk = False
if not self.cfg.fit.overwrite:
load_from_disk = jb.isFile(fname_ica, path=self.path_ica_chops)
if load_from_disk:
# self._ica_obj,fname_ica = jb.get_raw_obj(fname_ica,path=self.path_ica_chops)
ica_obj, fname_ica = jb.get_raw_obj(fname_ica,
path=self.path_ica_chops)
logger.info(
"DONE LOADING ICA chop form disk: {}\n -> ica filename: {}".
format(chop, fname_ica))
else:
if self.useArtifactRejection:
with jumeg_logger.StreamLoggerSTD(label="ica fit"):
ica_obj = fit_ica(
raw=raw_chop,
picks=self.picks,
reject=self.CFG.GetDataDict(key="reject"),
ecg_ch=self.cfg.ecg.ch_name,
ecg_thresh=self.cfg.ecg.thresh,
flow_ecg=self.cfg.ecg.flow,
fhigh_ecg=self.cfg.ecg.fhigh,
#---
eog_hor=self.cfg.eog.hor_ch,
eog_ver=self.cfg.eog.ver_ch,
flow_eog=self.cfg.eog.flow,
fhigh_eog=self.cfg.eog.fhigh,
eog_thresh=self.cfg.eog.thresh,
#---
use_jumeg=self.cfg.ecg.use_jumeg,
random_state=self.cfg.random_state)
ica_obj.exclude = list(set(ica_obj.exclude))
if self.useSVM:
if not ica_obj:
logger.info('SVM start ICA FIT: init ICA object')
#--- !!! ToDo put parameter in CFG file
ica_obj = ICA(method='fastica',
n_components=40,
random_state=42,
max_pca_components=None,
max_iter=5000,
verbose=False)
ica_obj.fit(raw_chop,
picks=self.picks,
decim=None,
reject=self.CFG.GetDataDict(key="reject"),
verbose=True)
else:
logger.info('SVM ICA Obj start')
#--- !!! do_copy = True => resample
ica_obj, _ = self.SVM.run(raw=self.raw,
ICA=ica_obj,
picks=self.picks,
do_crop=False,
do_copy=True)
logger.info('DONE SVM ICA FIT: apply ICA.fit')
#-- save ica object
if self.cfg.fit.save and not load_from_disk:
logger.info("saving ICA chop : {} / {}\n".format(
idx + 1, self.Chopper.n_chops) +
" -> ica filename : {}".format(fname_ica))
ica_obj.save(os.path.join(self.path_ica_chops, fname_ica))
logger.info(
"done ICA FIT for chop: {}\n".format(chop) +
" -> raw chop filename : {}\n".format(fname_ica) + "-" * 30 +
"\n" + " -> ICs found JuMEG/MNE : {}\n".format(self.SVM.ICsMNE) +
" -> ICs found SVM : {}\n".format(self.SVM.ICsSVM) +
" -> ICs excluded : {}\n".format(ica_obj.exclude) +
"-" * 30 + "\n" +
" -> save ica fit : {}".format(self.cfg.fit.save))
return ica_obj, fname_ica
def test():
'''
from jumeg.base.jumeg_base import jumeg_base as jb
from jumeg.base.pipelines.jumeg_pipelines_ica_perfromance import ICAPerformance
from jumeg.base.pipelines.jumeg_base_pipelines_chopper import JuMEG_PIPELINES_CHOPPER,copy_crop_and_chop,concat_and_save
'''
#---
from jumeg.base.jumeg_base_config import JuMEG_CONFIG as jCFG
verbose = True
debug = False
stage = "$JUMEG_TEST_DATA/mne/201772/INTEXT01/190212_1334/2"
fn = "201772_INTEXT01_190212_1334_2_c,rfDC,meeg,nr,bcc,int-raw.fif"
fn = "201772_INTEXT01_190212_1334_2_c,rfDC,meeg,nr,bcc,int,fibp0.10-45.0,ar-raw.fif"
stage = '$JUMEG_TEST_DATA/mne/212842/INTEXT01/190430_1001/4'
fn = '212842_INTEXT01_190430_1001_4_c,rfDC,meeg,nr,bcc,int,fibp0.10-45.0,ar-raw.fif'
fin = os.path.join(stage, fn)
raw, fname = jb.get_raw_obj(fname=fin)
#--- ck for annotations in raw
try:
annota = raw.annotations
except:
from jumeg.base.pipelines.jumeg_pipelines_ica_perfromance import ICAPerformance
IP = ICAPerformance()
#--- find ECG
IP.ECG.find_events(raw=raw)
IP.ECG.GetInfo(debug=True)
#--- find EOG
IP.EOG.find_events(raw=raw)
IP.EOG.GetInfo(debug=True)
#--- read epocher config
#--- define template
template_path = "$JUMEG_PATH_TEMPLATE_EPOCHER"
template_name = "INTEXT"
template_extention = "jumeg_epocher_template.yaml"
#--- CFG
CFG = jCFG()
fcfg = os.path.join(template_path,
template_name + "_" + template_extention)
CFG.update(config=fcfg)
condition_list = []
for k, itm in CFG.config.items():
if itm.get("run"):
condition_list.append(k)
if not condition_list: return
epocher_path = os.path.join(os.path.dirname(fname), "epocher")
hdf_path = epocher_path
evt_param = {
"condition_list": condition_list,
"template_path": template_path,
"template_name": template_name,
"use_yaml": True,
"hdf_path": hdf_path,
"verbose": verbose,
"debug": debug
}
ep_param = {
"condition_list": condition_list,
"template_path": template_path,
"template_name": template_name,
"hdf_path": hdf_path,
"save_raw": True,
"verbose": verbose,
"debug": debug,
"event_extention": ".eve",
"output_mode": {
"events": True,
"epochs": True,
"evoked": True,
"annotations": True,
"stage": epocher_path,
"use_condition_in_path": True
}
# "weights" :{"mode":"equal","method":"median","skip_first":null}
# "exclude_events":{"eog_events":{"tmin":-0.4,"tmax":0.6} } },
}
jEP = JuMEG_PIPELIENS_EPOCHER()
jEP.run(raw=raw,
fname=fname,
ep_param=ep_param,
evt_param=evt_param,
verbose=True,
debug=False,
show=True)
#---ToDo
# use mne plots
'''