def events_save_events(self,evt=None,condition=None,postfix="evt",
picks=None,reject=None,proj=False,
save_condition={"events":True,"epochs":True,"evoked":True}):
from jumeg.jumeg_4raw_data_plot import jumeg_4raw_data_plot as jplt
jplt.verbose = self.verbose
ep,bc = self.events_apply_epochs_and_baseline(self.raw,evt=evt,reject=reject,proj=proj,picks=picks)
postfix += '_' + condition
if bc:
postfix += '_bc'
#--- save events to txt file
if save_condition["events"]:
fname = jumeg_base.get_fif_name(raw=self.raw,postfix=postfix,extention=".eve",update_raw_fname=False)
mne.event.write_events( fname,evt['events'] )
print" ---> done jumeg epocher save events as => EVENTS :" +fname
#--- save epoch data
if save_condition["epochs"]:
fname = jumeg_base.get_fif_name(raw=self.raw,postfix=postfix,extention="-epo.fif",update_raw_fname=False)
ep.save( fname )
print" ---> done jumeg epocher save events as => EPOCHS :" +fname
#--- save averaged data
if save_condition["evoked"]:
fname = jumeg_base.get_fif_name(raw=self.raw,postfix=postfix,extention="-ave.fif",update_raw_fname=False)
mne.write_evokeds( fname,ep.average() )
print" ---> done jumeg epocher save events as => EVOKED (averaged) :" +fname
fname = jumeg_base.get_fif_name(raw=self.raw,postfix=postfix,extention="-ave",update_raw_fname=False)
#--- plot evoked
fname = jplt.plot_evoked(ep,fname=fname,condition=condition,show_plot=False,save_plot=True,plot_dir='plots')
print" ---> done jumeg epocher plot evoked (averaged) :" +fname
def perform_detrending(fname_raw,raw=None,save=True):
from mne.io import Raw
from numpy import poly1d, polyfit
raw = jumeg_base.get_raw_obj(fname_raw,raw=raw)
# get channels
picks = jumeg_base.pick_meg_and_ref_nobads()
xval = np.arange(raw._data.shape[1])
# loop over all channels
for ipick in picks:
coeff = polyfit(xval, raw._data[ipick, :], deg=1)
trend = poly1d(coeff)
raw._data[ipick, :] -= trend(xval)
# save detrended data
if save:
fname_out = jumeg_base.get_fif_name(raw=raw,postfix='dt')
jume_base.apply_save_mne_data(raw,fname=fname_out,overwrite=True)
return raw
def ctps_ica_brain_responses_clean(self,fname,raw=None,fname_ica=None,ica_raw=None,fhdf=None,template_name=None,
condition_list=None, njobs=4,fif_extention=".fif",fif_postfix="ctps",
clean_global={'save_raw':False,'save_epochs':False,'save_evoked':False},
clean_condition={'save_raw':False,'save_epochs':False,'save_evoked':False},
do_run=False,verbose=False,epocher_proj=None):
"""
clean ica brainresponses ics_global
clean ctps/condition/ics_selected -> where(ics ==True)
:param fname:
:param raw:
:param fname_ica:
:param ica_raw:
:param fhdf:
:param condition_list:
:param njobs:
:param fif_extention:
:param fif_postfix:
:param clean_global:
:param clean_condition:
:param save_epochs:
:param save_average:
:param save_raw:
:param do_run:
:param verbose:
:return: fhdf
"""
# add ctps ica artifact steady state e.g. 23,8 Hz, 60z
# TODO: split in fkt cleaning global & condition
from jumeg.jumeg_base import jumeg_base
self.ctps_init_brain_response_clean_data(fname,raw=raw,fname_ica=fname_ica,ica_raw=ica_raw,fhdf=fhdf,template_name=template_name)
ctps_condition_list = self.ctps_update_hdf_condition_list(condition_list,node='/ctps/')
#--- clean global
if any( clean_global.values() ) :
#--- get global Ics from pd.Series
ics = self.HDFobj['ics_global']
ica_picks = ics.values
print "CTPs Clean Global RAW"
raw_ctps_clean = self.ica_raw.apply(self.raw,include=ica_picks,n_pca_components=None,copy=True)
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'-raw',extention=fif_extention)
if clean_global['save_raw'] :
raw_ctps_clean.save(fout,overwrite=True)
print"---> save raw ctps global clean\n ---> " + fout +"\n"
if ( clean_global['save_epochs'] or clean_global['save_evoked'] ):
for condi in ctps_condition_list:
print "---> Init parameter global : " + condi
print " --> save epochs : %r" %(clean_global['save_epochs'])
print " --> save evoked : %r" %(clean_global['save_evoked'])
print " --> ICs global count : %d" %(ica_picks.size)
if not ica_picks.size: continue
#--- read epocher condi parameter & info
epocher_key = '/epocher/' + condi
ep_param = self.hdf_obj_get_attributes(key=epocher_key,attr='epocher_parameter')
#stimulus_info = self.hdf_obj_get_attributes(key=epocher_key,attr='info_parameter')
if self.verbose:
print " --> Epocher Parameter: "
print ep_param
#print " --> Stimulus Info :"
print "\n\n"
#--- make event array from < ctps condition events>
ctps_key= '/ctps/' + condi
ev_tsl = self.hdf_obj_get_dataframe(key= ctps_key+'/events')
ev = np.zeros(( ev_tsl.size, 3), dtype=np.int32)
try:
ev[:,2] = ep_param['stimulus']['event_id']
except:
ev[:,2] = ep_param['stimulus']['event_id'][0]
else:
ev[:,2] = 1
ev[:,0] = ev_tsl
#--- get global epochs
for k in ('time_pre','time_post','reject'):
if ep_param.has_key(k): continue
ep_param[k] = None
dout = mne.Epochs(raw_ctps_clean,events=ev,event_id=dict( condi = int(ev[0,2]) ),
tmin=ep_param['time_pre'],tmax=ep_param['time_post'],baseline=ep_param['baseline'],
reject=ep_param['reject'],
verbose=self.verbose,proj=epocher_proj)
#--- global condi epochs
if clean_global['save_epochs']:
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'-'+condi+'-epo',extention=fif_extention)
dout.save(fout)
print"---> save epochs ctps global condition clean: " + condi +"\n"+ fout +"\n"
#--- global evoked ( average condi epochs )
if clean_global['save_evoked']:
print "Global Evoked (Averager) :" + condi
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'-'+condi+'-ave',extention=fif_extention)
dout = dout.average()
dout.save(fout)
print"---> save evoked ctps global condition clean: " + condi +"\n"+ fout + "\n"
print"---> Done ctps global condition clean: " + condi + "\n"
#------
if any( clean_condition.values() ):
print "CTPs Clean Condition RAW"
for condi in ctps_condition_list:
print "---> Init parameter condition : " + condi
print " --> save epochs : %r" %(clean_condition['save_epochs'])
print " --> save evoked : %r" %(clean_condition['save_evoked'])
ctps_key = '/ctps/' + condi
ics = self.HDFobj[ctps_key+'/ics_selected']
ica_picks = np.array( np.where( ics ),dtype=np.int16).flatten()
print" ICs counts: %d" %( ica_picks.size )
print "ICS:"
print ica_picks
if not ica_picks.size: continue
raw_ctps_clean = self.ica_raw.apply(self.raw,include=ica_picks,n_pca_components=None,copy=True)
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'_co-'+ condi+'-raw',extention=fif_extention)
if clean_condition['save_raw']:
raw_ctps_clean.save(fout) # mne
print"---> save raw ctps condition clean: " + condi +"\n"+ fout +"\n"
if ( clean_condition['save_epochs'] or clean_condition['save_evoked'] ):
#--- read epocher condi parameter & info
epocher_key = '/epocher/' + condi
ep_param = self.hdf_obj_get_attributes(key=epocher_key,attr='epocher_parameter')
if self.verbose:
print " --> CTPs Condition Epocher Parameter: " + condi
print ep_param
#print " --> Stimulus Info :"
print "\n\n"
#--- make event array from < ctps condition events>
ev_tsl = self.hdf_obj_get_dataframe(key= ctps_key+'/events')
ev = np.zeros(( ev_tsl.size, 3), dtype=np.int32)
try:
ev[:,2] = ep_param['stimulus']['event_id']
except:
ev[:,2] = ep_param['stimulus']['event_id'][0]
else:
ev[:,2] = 1
ev[:,0] = ev_tsl
#--- get global epochs
for k in ('time_pre','time_post','reject'):
if ep_param.has_key(k): continue
ep_param[k] = None
dout = mne.Epochs(raw_ctps_clean,events=ev,event_id=dict( condi = int( ev[0,2] ) ),
tmin=ep_param['time_pre'],tmax=ep_param['time_post'],baseline=ep_param['baseline'],
reject=ep_param['reject'],
verbose=self.verbose,proj=epocher_proj)
#--- global condi epochs
if clean_condition['save_epochs']:
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'_co-'+condi+'-epo',extention=fif_extention)
dout.save(fout)
print"---> save epochs ctps condition clean: " + condi +"\n"+ fout +"\n"
#--- global evoked ( average condi epochs )
if clean_condition['save_evoked']:
print "Global Evoked (Averager) :" + condi
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'_co-'+condi+'-ave',extention=fif_extention)
dout = dout.average()
dout.save(fout)
print"---> save evoked ctps condition clean: " + condi +"\n"+ fout +"\n"
print"---> Done ctps condition clean: " + condi + "\n"
fhdf = self.HDFobj.filename
self.HDFobj.close()
return fhdf
def apply_noise_reducer_data(fname,raw=None,do_run=True,verbose=False,save=True,plot=False,
reflp=None, refhp=None, refnotch=None,fif_postfix="nr",fif_extention="-raw.fif",**kwargs):
'''
Applies the noise reducer to raw obj data or to fif-file.
the magic ee once
fb modified for raw obj support
imports jumeg_noise_reducer_4raw_data
'''
import os
#--- import noise_reducer and plot_power_spectrum function
from jumeg.jumeg_4raw_data_noise_reducer import noise_reducer_4raw_data, plot_denoising_4raw_data
fname_out = None
nr_done = False
if do_run :
raw,fname_raw = jumeg_base.get_raw_obj(fname,raw=raw)
fname_out = jumeg_base.get_fif_name(raw=raw,postfix=fif_postfix,extention=fif_extention,update_raw_fname=False)
#--- apply noise reducer for 50 Hz (and harmonics)
if (reflp or refhp):
raw,fname_out = noise_reducer_4raw_data(fname,raw=raw,reflp=reflp,refhp=refhp,verbose=verbose,save=False,**kwargs['parameter'])
kwargs['parameter']['detrending'] = None
nr_done = True
if refnotch:
for refn in refnotch:
raw,fname_out = noise_reducer_4raw_data(None,raw=raw,refnotch=refn,verbose=verbose,save=False,**kwargs['parameter'])
kwargs['parameter']['detrending'] = None
nr_done = True
# raw.info['filename'] = fname_out
if not nr_done :
return fname_raw,raw
raw.info['filename'] = fname_out
if save:
fname_out = jumeg_base.apply_save_mne_data(raw,fname=fname_out)
if plot:
print " --> noise reducer plot power spectrum"
from distutils.dir_util import mkpath
p,pdf = os.path.split(fname_raw)
plot_dir = p+ '/plots/'
mkpath(plot_dir)
fn_power_spect = plot_dir + pdf[:pdf.rfind('-raw.fif') ]+ ',denoising'
plot_denoising_4raw_data([fname_raw,fname_out],show=False,fnout=fn_power_spect)
print"---> noise reducer plot :" + fn_power_spect
print "---> Done noise reducer: "+ fname_out
return (fname_out,raw)
#--- filter raw data
if tmp_pp_raw.filter.do_run:
tmp_pp_raw.filter['verbose'] = verbose
print "\n===> PP Info: start apply filter raw:"
print "File : " + fif_file
if verbose:
print "Parameter :"
print tmp_pp_raw.filter
print "\n\n"
(fname, raw) = jppd.apply_filter_data(fif_file,
raw=raw,
**tmp_pp_raw.filter)
print "\n\n==> PP Info: done apply filter raw\n ---> " + fname
else:
fname = jumeg_base.get_fif_name(
fif_file,
postfix=tmp_pp_raw.filter.fif_postfix,
extention=tmp_pp_raw.filter.fif_extention)
#--- average raw filtered data
# if tmp_pp_raw.average.do_run :
# tmp_pp_raw.average.verbose = verbose
# print"\n===> PP Info: start apply averager raw"
# print"File :" + fname
# if verbose:
# print"Parameter :"
# print tmp_pp_raw.average
# print"\n\n"
# jppd.apply_averager(fn_raw_list,**tmp_pp_raw.averager)
# print"\n\n==> PP Info: done apply averager filterd raw data\n"
#--- ocarta
def apply_noise_reducer_data(fname,
raw=None,
do_run=True,
verbose=False,
save=True,
plot=False,
reflp=None,
refhp=None,
refnotch=None,
fif_postfix="nr",
fif_extention="-raw.fif",
**kwargs):
'''
Applies the noise reducer to raw obj data or to fif-file.
the magic ee once
fb modified for raw obj support
imports jumeg_noise_reducer_4raw_data
'''
import os
#--- import noise_reducer and plot_power_spectrum function
from jumeg.jumeg_4raw_data_noise_reducer import noise_reducer_4raw_data, plot_denoising_4raw_data
fname_out = None
nr_done = False
if do_run:
raw, fname_raw = jumeg_base.get_raw_obj(fname, raw=raw)
fname_out = jumeg_base.get_fif_name(raw=raw,
postfix=fif_postfix,
extention=fif_extention,
update_raw_fname=False)
#--- apply noise reducer for 50 Hz (and harmonics)
if (reflp or refhp):
raw, fname_out = noise_reducer_4raw_data(fname,
raw=raw,
reflp=reflp,
refhp=refhp,
verbose=verbose,
save=False,
**kwargs['parameter'])
kwargs['parameter']['detrending'] = None
nr_done = True
if refnotch:
for refn in refnotch:
raw, fname_out = noise_reducer_4raw_data(None,
raw=raw,
refnotch=refn,
verbose=verbose,
save=False,
**kwargs['parameter'])
kwargs['parameter']['detrending'] = None
nr_done = True
# raw.info['filename'] = fname_out
if not nr_done:
return fname_raw, raw
raw.info['filename'] = fname_out
if save:
fname_out = jumeg_base.apply_save_mne_data(raw, fname=fname_out)
if plot:
print " --> noise reducer plot power spectrum"
from distutils.dir_util import mkpath
p, pdf = os.path.split(fname_raw)
plot_dir = p + '/plots/'
mkpath(plot_dir)
fn_power_spect = plot_dir + pdf[:pdf.rfind('-raw.fif'
)] + ',denoising'
plot_denoising_4raw_data([fname_raw, fname_out],
show=False,
fnout=fn_power_spect)
print "---> noise reducer plot :" + fn_power_spect
print "---> Done noise reducer: " + fname_out
return (fname_out, raw)
def main(argv):
#-- default argv
# experiment_name = None
# path_list = os.getcwd()
# fname_list = None
# # fif_name = None
# pfif = os.getcwd()
# path_mne_stage = None
# bad_channels = None
# verbose, do_run = False,False
# condition_list = None
epocher_hdf_fname = None
#--- get args from parser
opt,parser = jpp_utils.get_args()
#--- set debug & test mode
if opt.debug :
opt.experiment = 'TEST01'
opt.stage = '/localdata/frank/data/Chrono/mne'
#path_list = "/localdata/frank/data/Chrono/doc"
#fname_list = 'chrono_normal_inkomp.txt'
opt.fifname = '201195_Chrono01_110516_1413_1_c,rfDC-raw.fif' #'201195_test.fif' #None
opt.pathfif = '201195/Chrono01/110516_1413/1' #None
opt.verbose = True
opt.run = True
#condition_list = ('LRst','LRrt')
#---
if opt.verbose :
print"\n---> ARGV parameter:"
print"experiment : " + str(opt.experiment)
print"condition : " + str(opt.conditions)
print"stage : " + str(opt.stage)
print"path to fif : " + str(opt.pathfif)
print"fif name : " + str(opt.fifname)
print"path to list: " + str(opt.pathlist)
print"fname list : " + str(opt.fnamelist)
print"verbose : " + str(opt.verbose)
print"run : " + str(opt.run)
print"debug mode : " + str(opt.debug)
print"\n\n"
#--- update base
jumeg_base.verbose = opt.verbose
#--- init experiment template parameter
jtmp_exp.template_name = opt.experiment
jtmp_exp.verbose = opt.verbose
#- read template parameters into dict
tmp = jtmp_exp.template_update_file()
#--- make obj from dict
TMP = jtmp_exp.template_get_as_obj()
path_exp = TMP.experiment.path.experiment
if opt.stage is None:
opt.stage = TMP.experiment.path.mne
#---
if opt.verbose :
print"\n---> Experiment Template parameter:"
print" --> name : "+ jtmp_exp.template_name
print" --> template file: "+ jtmp_exp.template_full_file_name
print"\n"
#---
if not opt.run:
print "===> Done jumeg preprocessing , set run flag for real data processing\n"
print parser.print_help()
print"\n"
exit()
#--- get existing files from list
# 005/MEG94T/121219_1311/1/005_MEG94T_121219_1311_1_c,rfDC-raw.fif -nc=A1
# 007/MEG94T/121217_1239/1/007_MEG94T_121217_1239_1_c,rfDC-raw.fif -nc=A1,A2
fn_raw_list=[]
fn_raw_bad_channel_dict=[]
if opt.fnamelist:
fn_raw_list,fn_raw_bad_channel_dict = jumeg_base.get_filename_list_from_file(opt.pathlist + "/" + opt.fnamelist,start_path = opt.stage)
#--- check & add fif file to list update bad channel dict
if opt.fifname:
if opt.pathfif :
f = opt.pathfif +"/"+ opt.fifname
else:
f = opt.fifname
if os.path.isfile(f):
fn_raw_list.append(f)
if opt.bads: #--- bad channels
fn_raw_bad_channel_dict[f]= opt.bads
elif os.path.isfile(opt.stage + '/' + f):
fn_raw_list.append(opt.stage + '/' + f)
if opt.bads:
fn_raw_bad_channel_dict[f]= opt.bads
#--- raw obj short-cut
tmp_pp_raw = TMP.experiment.data_preprocessing.raw
#--- brainresponse obj short-cut
tmp_pp_brs = TMP.experiment.data_preprocessing.brainresponse
#--- loop preproc for each fif file
for fif_file in (fn_raw_list) :
raw = None
#--- check / set bad channels
if ( fif_file in fn_raw_bad_channel_dict ):
print "\n ===> BAD Channel -> " + fif_file
print" --> BADs: " + str(fn_raw_bad_channel_dict[fif_file])
if fn_raw_bad_channel_dict[fif_file]:
raw,bads_dict = jumeg_base.update_bad_channels(fif_file,raw=raw,bads=fn_raw_bad_channel_dict[fif_file],save=True)
#--- epocher search for events save to HDF
if tmp_pp_raw.epocher.do_run :
tmp_pp_raw.epocher['verbose'] = opt.verbose
print"\n===> PP Info: start apply epocher => event code search"
print"File : " + fif_file
if opt.verbose:
print"Parameter:"
print tmp_pp_raw.epocher
print"\n\n"
(fname,raw,epocher_hdf_fname) = jppd.apply_epocher_events_data(fif_file,raw=raw,condition_list=opt.conditions, **tmp_pp_raw.epocher)
#--- noise_covariance
#--- will search and find empty room file if fif is no empty room file
if tmp_pp_raw.noise_covariance.do_run :
tmp_pp_raw.noise_covariance['verbose'] = opt.verbose
print"\n===> PP Info: start apply create noise_covariance"
print"File :" + fif_file
if opt.verbose:
print"Parameter:"
print tmp_pp_raw.noise_covariance
print"\n\n"
fname_noise_covariance = jppd.apply_create_noise_covariance_data(fif_file,raw=raw,**tmp_pp_raw.noise_covariance)
print"\n\n==> PP Info: done apply create noise_covariance :\n ---> "
try:
print fname_noise_covariance +"\n"
except:
print " !!! not found !!!\n\n"
#--- noise_reducer
#--- will apply magic ee noise reducer
if tmp_pp_raw.noise_reducer.do_run :
tmp_pp_raw.noise_reducer['verbose'] = opt.verbose
print"\n===> PP Info: start apply ee noise_reducer"
print"File :" + fif_file
if opt.verbose:
print"Parameter:"
print tmp_pp_raw.noise_reducer
print"\n\n"
(fif_file, raw) = jppd.apply_noise_reducer_data(fif_file,raw=raw,**tmp_pp_raw.noise_reducer)
print"\n\n==> PP Info: done apply noise reducer raw\n ---> " + fname
else:
fif_file = jumeg_base.get_fif_name(fif_file,postfix=tmp_pp_raw.noise_reducer.fif_postfix,extention=tmp_pp_raw.noise_reducer.fif_extention)
raw = None
#--- filter raw data
if tmp_pp_raw.filter.do_run :
tmp_pp_raw.filter['verbose'] = opt.verbose
print"\n===> PP Info: start apply filter raw:"
print"File : " + fif_file
if opt.verbose:
print"Parameter :"
print tmp_pp_raw.filter
print"\n\n"
(fname, raw) = jppd.apply_filter_data(fif_file,raw=raw,**tmp_pp_raw.filter)
print"\n\n==> PP Info: done apply filter raw\n ---> " + fname
else:
fname = jumeg_base.get_fif_name(fif_file,postfix=tmp_pp_raw.filter.fif_postfix,extention=tmp_pp_raw.filter.fif_extention)
raw = None
#--- average raw filtered data
# if tmp_pp_raw.average.do_run :
# tmp_pp_raw.average.verbose = verbose
# print"\n===> PP Info: start apply averager raw"
# print"File :" + fname
# if verbose:
# print"Parameter :"
# print tmp_pp_raw.average
# print"\n\n"
# jppd.apply_averager(fn_raw_list,**tmp_pp_raw.averager)
# print"\n\n==> PP Info: done apply averager filterd raw data\n"
#--- ocarta
if tmp_pp_raw.ocarta.do_run :
tmp_pp_raw.ocarta['verbose'] = opt.verbose
print"\n===> PP Info: start apply ocarta fit"
print"File :" + fname
if opt.verbose :
print"Parameter :"
print tmp_pp_raw.ocarta
print"\n\n"
(fname_oca,raw,fhdf) = jppd.apply_ocarta_data(fname,raw=raw,**tmp_pp_raw.ocarta)
print"\n\n==> PP Info: done apply ocarta\n ---> " + fname_oca
else:
fname_oca = jumeg_base.get_fif_name(fname,postfix=tmp_pp_raw.ocarta.fif_postfix,extention=tmp_pp_raw.ocarta.fif_extention)
raw = None
#--- brain response apply mne ica: fastica
if tmp_pp_brs.ica.do_run :
tmp_pp_brs.ica['verbose'] = opt.verbose
print "\n===> PP Info: start apply brain-response ica"
print"File : " + fname_oca
if opt.verbose:
print"Parameter :"
print tmp_pp_brs.ica
print"\n\n"
(fname_oca_ica,raw,ICAobj) = jppd.apply_ica_data(fname_oca,raw=raw,**tmp_pp_brs.ica)
print"\n\n==> PP Info: done apply ica for brain responses\n ---> " + fname_oca_ica
else:
fname_oca_ica = jumeg_base.get_fif_name(fname_oca,postfix=tmp_pp_brs.ica.fif_postfix,extention=tmp_pp_brs.ica.fif_extention)
raw = None
# 201195_Chrono01_110516_1413_1_c,rfDC,fihp1n,ocarta-ica.fif
#--- brain response ctps
#--- run for all condition combine and or exclude CTPs-ICs for different conditions
if tmp_pp_brs.ctps.do_run :
tmp_pp_brs.ctps['verbose'] = opt.verbose
print"\n===> PP Info: start brainresponse ica"
print"File : " + fname_oca_ica
if opt.verbose :
print"Parameters:"
print tmp_pp_brs.ctps
print"\n\n"
(fname_oca,raw,fhdf)=jppd.apply_ctps_brain_responses_data(fname_oca,raw=raw,fname_ica=fname_oca_ica,ica_raw=None,
condition_list=condition_list,**tmp_pp_brs.ctps)
print"\n\n==> PP Info: done apply ctp for brain responses\n ---> " + fname_oca
else:
fhdf = None
raw = None
#--- brain response ctps ica cleaning
#--- run for global ics, all ctp-ics-condition create raw,epochs,average
if tmp_pp_brs.clean.do_run :
tmp_pp_brs.clean['verbose'] = opt.verbose
print"\n===> PP Info: start brain-response cleaning"
print"File : " + fname_oca
if opt.verbose :
print"Parameters:"
print tmp_pp_brs.clean
print"\n\n"
fhdf = jppd.apply_ctps_brain_responses_cleaning_data(fname_oca,raw=raw,fname_ica=fname_oca_ica,ica_raw=None,fhdf=fhdf,
condition_list=condition_list,**tmp_pp_brs.clean)
print"===> Done JuMEG Pre-Processing Data"
print" --> FIF file input : " + fif_file
print" --> Information stored in HDF file: " + str(fhdf)
print"\n\n"
raw = None
except:
print " !!! not found !!!\n\n"
#--- filter raw data
if tmp_pp_raw.filter.do_run :
tmp_pp_raw.filter['verbose'] = verbose
print"\n===> PP Info: start apply filter raw:"
print"File : " + fif_file
if verbose:
print"Parameter :"
print tmp_pp_raw.filter
print"\n\n"
(fname, raw) = jppd.apply_filter_data(fif_file,raw=raw,**tmp_pp_raw.filter)
print"\n\n==> PP Info: done apply filter raw\n ---> " + fname
else:
fname = jumeg_base.get_fif_name(fif_file,postfix=tmp_pp_raw.filter.fif_postfix,extention=tmp_pp_raw.filter.fif_extention)
raw = None
#--- average raw filtered data
# if tmp_pp_raw.average.do_run :
# tmp_pp_raw.average.verbose = verbose
# print"\n===> PP Info: start apply averager raw"
# print"File :" + fname
# if verbose:
# print"Parameter :"
# print tmp_pp_raw.average
# print"\n\n"
# jppd.apply_averager(fn_raw_list,**tmp_pp_raw.averager)
# print"\n\n==> PP Info: done apply averager filterd raw data\n"
#--- ocarta
def ctps_ica_brain_responses_clean(self,fname,raw=None,fname_ica=None,ica_raw=None,fhdf=None,template_name=None,
condition_list=None, njobs=4,fif_extention=".fif",fif_postfix="ctps",
clean_global={'save_raw':False,'save_epochs':False,'save_evoked':False},
clean_condition={'save_raw':False,'save_epochs':False,'save_evoked':False},
do_run=False,verbose=False,epocher_proj=None):
"""
clean ica brainresponses ics_global
clean ctps/condition/ics_selected -> where(ics ==True)
:param fname:
:param raw:
:param fname_ica:
:param ica_raw:
:param fhdf:
:param condition_list:
:param njobs:
:param fif_extention:
:param fif_postfix:
:param clean_global:
:param clean_condition:
:param save_epochs:
:param save_average:
:param save_raw:
:param do_run:
:param verbose:
:return: fhdf
"""
# add ctps ica artifact steady state e.g. 23,8 Hz, 60z
# TODO: split in fkt cleaning global & condition
from jumeg.jumeg_base import jumeg_base
self.ctps_init_brain_response_clean_data(fname,raw=raw,fname_ica=fname_ica,ica_raw=ica_raw,fhdf=fhdf,template_name=template_name)
ctps_condition_list = self.ctps_update_hdf_condition_list(condition_list,node='/ctps/')
#--- clean global
if any( clean_global.values() ) :
#--- get global Ics from pd.Series
ics = self.HDFobj['ics_global']
ica_picks = ics.values
print "CTPs Clean Global RAW"
raw_ctps_clean = self.ica_raw.apply(self.raw,include=ica_picks,n_pca_components=None,copy=True)
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'-raw',extention=fif_extention)
if clean_global['save_raw'] :
raw_ctps_clean.save(fout,overwrite=True)
print"---> save raw ctps global clean\n ---> " + fout +"\n"
if ( clean_global['save_epochs'] or clean_global['save_evoked'] ):
for condi in ctps_condition_list:
print "---> Init parameter global : " + condi
print " --> save epochs : %r" %(clean_global['save_epochs'])
print " --> save evoked : %r" %(clean_global['save_evoked'])
print " --> ICs global count : %d" %(ica_picks.size)
if not ica_picks.size: continue
#--- read epocher condi parameter & info
epocher_key = '/epocher/' + condi
ep_param = self.hdf_obj_get_attributes(key=epocher_key,attr='epocher_parameter')
#stimulus_info = self.hdf_obj_get_attributes(key=epocher_key,attr='info_parameter')
if self.verbose:
print " --> Epocher Parameter: "
print ep_param
#print " --> Stimulus Info :"
print "\n\n"
#--- make event array from < ctps condition events>
ctps_key= '/ctps/' + condi
ev_tsl = self.hdf_obj_get_dataframe(key= ctps_key+'/events')
ev = np.zeros(( ev_tsl.size, 3), dtype=np.int32)
try:
ev[:,2] = ep_param['stimulus']['event_id']
except:
ev[:,2] = ep_param['stimulus']['event_id'][0]
else:
ev[:,2] = 1
ev[:,0] = ev_tsl
#--- get global epochs
for k in ('time_pre','time_post','reject'):
if ep_param.has_key(k): continue
ep_param[k] = None
dout = mne.Epochs(raw_ctps_clean,events=ev,event_id=dict( condi = int(ev[0,2]) ),
tmin=ep_param['time_pre'],tmax=ep_param['time_post'],baseline=ep_param['baseline'],
reject=ep_param['reject'],
verbose=self.verbose,proj=epocher_proj)
#--- global condi epochs
if clean_global['save_epochs']:
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'-'+condi+'-epo',extention=fif_extention)
dout.save(fout)
print"---> save epochs ctps global condition clean: " + condi +"\n"+ fout +"\n"
#--- global evoked ( average condi epochs )
if clean_global['save_evoked']:
print "Global Evoked (Averager) :" + condi
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'-'+condi+'-ave',extention=fif_extention)
dout = dout.average()
dout.save(fout)
print"---> save evoked ctps global condition clean: " + condi +"\n"+ fout + "\n"
print"---> Done ctps global condition clean: " + condi + "\n"
#------
if any( clean_condition.values() ):
print "CTPs Clean Condition RAW"
for condi in ctps_condition_list:
print "---> Init parameter condition : " + condi
print " --> save epochs : %r" %(clean_condition['save_epochs'])
print " --> save evoked : %r" %(clean_condition['save_evoked'])
ctps_key = '/ctps/' + condi
ics = self.HDFobj[ctps_key+'/ics_selected']
ica_picks = np.array( np.where( ics ),dtype=np.int16).flatten()
print" ICs counts: %d" %( ica_picks.size )
print "ICS:"
print ica_picks
if not ica_picks.size: continue
raw_ctps_clean = self.ica_raw.apply(self.raw,include=ica_picks,n_pca_components=None,copy=True)
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'_co-'+ condi+'-raw',extention=fif_extention)
if clean_condition['save_raw']:
raw_ctps_clean.save(fout) # mne
print"---> save raw ctps condition clean: " + condi +"\n"+ fout +"\n"
if ( clean_condition['save_epochs'] or clean_condition['save_evoked'] ):
#--- read epocher condi parameter & info
epocher_key = '/epocher/' + condi
ep_param = self.hdf_obj_get_attributes(key=epocher_key,attr='epocher_parameter')
if self.verbose:
print " --> CTPs Condition Epocher Parameter: " + condi
print ep_param
#print " --> Stimulus Info :"
print "\n\n"
#--- make event array from < ctps condition events>
ev_tsl = self.hdf_obj_get_dataframe(key= ctps_key+'/events')
ev = np.zeros(( ev_tsl.size, 3), dtype=np.int32)
try:
ev[:,2] = ep_param['stimulus']['event_id']
except:
ev[:,2] = ep_param['stimulus']['event_id'][0]
else:
ev[:,2] = 1
ev[:,0] = ev_tsl
#--- get global epochs
for k in ('time_pre','time_post','reject'):
if ep_param.has_key(k): continue
ep_param[k] = None
dout = mne.Epochs(raw_ctps_clean,events=ev,event_id=dict( condi = int( ev[0,2] ) ),
tmin=ep_param['time_pre'],tmax=ep_param['time_post'],baseline=ep_param['baseline'],
reject=ep_param['reject'],
verbose=self.verbose,proj=epocher_proj)
#--- global condi epochs
if clean_condition['save_epochs']:
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'_co-'+condi+'-epo',extention=fif_extention)
dout.save(fout)
print"---> save epochs ctps condition clean: " + condi +"\n"+ fout +"\n"
#--- global evoked ( average condi epochs )
if clean_condition['save_evoked']:
print "Global Evoked (Averager) :" + condi
fout = jumeg_base.get_fif_name(fname,postfix=fif_postfix+'_co-'+condi+'-ave',extention=fif_extention)
dout = dout.average()
dout.save(fout)
print"---> save evoked ctps condition clean: " + condi +"\n"+ fout +"\n"
print"---> Done ctps condition clean: " + condi + "\n"
fhdf = self.HDFobj.filename
self.HDFobj.close()
return fhdf
def main(argv):
epocher_hdf_fname = None
#--- get args from parser
opt, parser = jpp_utils.get_args()
#--- set debug & test mode
if opt.debug:
opt.experiment = 'TEST01'
opt.stage = '/localdata/frank/data/Chrono/mne'
#path_list = "/localdata/frank/data/Chrono/doc"
#fname_list = 'chrono_normal_inkomp.txt'
opt.fifname = '201195_Chrono01_110516_1413_1_c,rfDC-raw.fif' #'201195_test.fif' #None
opt.pathfif = '201195/Chrono01/110516_1413/1' #None
opt.verbose = True
opt.run = True
#condition_list = ('LRst','LRrt')
#---
if opt.verbose:
print "\n---> ARGV parameter:"
print "experiment : " + str(opt.experiment)
print "condition : " + str(opt.conditions)
print "stage : " + str(opt.stage)
print "path to fif : " + str(opt.pathfif)
print "fif name : " + str(opt.fifname)
print "path to list: " + str(opt.pathlist)
print "fname list : " + str(opt.fnamelist)
print "verbose : " + str(opt.verbose)
print "run : " + str(opt.run)
print "debug mode : " + str(opt.debug)
print "\n\n"
#--- update base
jumeg_base.verbose = opt.verbose
#--- init experiment template parameter
jtmp_exp.template_name = opt.experiment
jtmp_exp.verbose = opt.verbose
#- read template parameters into dict
tmp = jtmp_exp.template_update_file()
#--- make obj from dict
TMP = jtmp_exp.template_get_as_obj()
path_exp = TMP.experiment.path.experiment
if opt.stage is None:
opt.stage = TMP.experiment.path.mne
#---
if opt.verbose:
print "\n---> Experiment Template parameter:"
print " --> name : " + jtmp_exp.template_name
print " --> template file: " + jtmp_exp.template_full_file_name
print "\n"
#---
if not opt.run:
print "===> Done jumeg preprocessing , set run flag for real data processing\n"
print parser.print_help()
print "\n"
exit()
#--- get existing files from list e.g.
# 005/MEG94T/121219_1311/1/005_MEG94T_121219_1311_1_c,rfDC-raw.fif -nc=A1
# 007/MEG94T/121217_1239/1/007_MEG94T_121217_1239_1_c,rfDC-raw.fif -nc=A1,A2
fn_raw_list = []
fn_raw_bad_channel_dict = []
if opt.fnamelist:
fn_raw_list, fn_raw_bad_channel_dict = jumeg_base.get_filename_list_from_file(
opt.pathlist + "/" + opt.fnamelist, start_path=opt.stage)
#--- check & add fif file to list update bad channel dict
if opt.fifname:
if opt.pathfif:
f = opt.pathfif + "/" + opt.fifname
else:
f = opt.fifname
if os.path.isfile(f):
fn_raw_list.append(f)
if opt.bads: #--- bad channels
fn_raw_bad_channel_dict[f] = opt.bads
elif os.path.isfile(opt.stage + '/' + f):
fn_raw_list.append(opt.stage + '/' + f)
if opt.bads:
fn_raw_bad_channel_dict[f] = opt.bads
#--- raw obj short-cut
tmp_pp_raw = TMP.experiment.data_preprocessing.raw
#--- brainresponse obj short-cut
tmp_pp_brs = TMP.experiment.data_preprocessing.brainresponse
#--- loop preproc for each fif file
for fif_file in (fn_raw_list):
raw = None
fif_file_raw = fif_file
#--- check / set bad channels
if (fif_file in fn_raw_bad_channel_dict):
print "\n ===> BAD Channel -> " + fif_file
print " --> BADs: " + str(fn_raw_bad_channel_dict[fif_file])
if fn_raw_bad_channel_dict[fif_file]:
raw, bads_dict = jumeg_base.update_bad_channels(
fif_file,
raw=raw,
bads=fn_raw_bad_channel_dict[fif_file],
save=True)
#--- epocher search for events save to HDF
if hasattr(tmp_pp_raw.epocher, "do_run"):
if tmp_pp_raw.epocher.do_run:
tmp_pp_raw.epocher['verbose'] = opt.verbose
print "\n===> PP Info: start apply epocher => event code search"
print "File : " + fif_file
if opt.verbose:
print "Parameter:"
print tmp_pp_raw.epocher
print "\n\n"
(fname, raw,
epocher_hdf_fname) = jppd.apply_epocher_events_data(
fif_file,
raw=raw,
condition_list=opt.conditions,
**tmp_pp_raw.epocher)
else:
jpp_utils.msg_on_attribute_error("raw epocher")
#--- noise_covariance
#--- will search and find empty room file if fif is no empty room file
if hasattr(tmp_pp_raw.noise_covariance, "do_run"):
if tmp_pp_raw.noise_covariance.do_run:
tmp_pp_raw.noise_covariance['verbose'] = opt.verbose
print "\n===> PP Info: start apply create noise_covariance"
print "File :" + fif_file
if opt.verbose:
print "Parameter:"
print tmp_pp_raw.noise_covariance
print "\n\n"
fname_noise_covariance = jppd.apply_create_noise_covariance_data(
fif_file, raw=raw, **tmp_pp_raw.noise_covariance)
print "\n\n==> PP Info: done apply create noise_covariance :\n ---> "
try:
print fname_noise_covariance + "\n"
except:
print " !!! not found !!!\n\n"
else:
jpp_utils.msg_on_attribute_error("raw noise covariance")
#--- noise_reducer
#--- will apply magic ee noise reducer7
if hasattr(tmp_pp_raw.noise_reducer, "do_run"):
if tmp_pp_raw.noise_reducer.do_run:
tmp_pp_raw.noise_reducer['verbose'] = opt.verbose
print "\n===> PP Info: start apply ee noise_reducer"
print "File :" + fif_file
if opt.verbose:
print "Parameter:"
print tmp_pp_raw.noise_reducer
print "\n\n"
(fif_file, raw) = jppd.apply_noise_reducer_data(
fif_file, raw=raw, **tmp_pp_raw.noise_reducer)
print "\n\n==> PP Info: done apply noise reducer raw\n ---> " + fif_file
print "RAW fname: " + raw.info['filename']
else:
fif_file = jumeg_base.get_fif_name(
fname=fif_file,
postfix=tmp_pp_raw.noise_reducer.fif_postfix,
extention=tmp_pp_raw.noise_reducer.fif_extention)
raw = None
else:
jpp_utils.msg_on_attribute_error("raw noise reducer")
#--- filter raw data
if hasattr(tmp_pp_raw.filter, "do_run"):
if tmp_pp_raw.filter.do_run:
tmp_pp_raw.filter['verbose'] = opt.verbose
print "\n===> PP Info: start apply filter raw:"
print "File : " + fif_file
if opt.verbose:
print "Parameter :"
print tmp_pp_raw.filter
print "\n\n"
(fif_file, raw) = jppd.apply_filter_data(fif_file,
raw=raw,
**tmp_pp_raw.filter)
print "\n\n==> PP Info: done apply filter raw\n ---> " + fname
else:
fif_file = jumeg_base.get_fif_name(
fname=fif_file,
postfix=tmp_pp_raw.filter.fif_postfix,
extention=tmp_pp_raw.filter.fif_extention)
raw = None
else:
jpp_utils.msg_on_attribute_error("raw filter")
#--- average raw filtered data
# if tmp_pp_raw.average.do_run :
# tmp_pp_raw.average.verbose = verbose
# print"\n===> PP Info: start apply averager raw"
# print"File :" + fname
# if verbose:
# print"Parameter :"
# print tmp_pp_raw.average
# print"\n\n"
# jppd.apply_averager(fn_raw_list,**tmp_pp_raw.averager)
# print"\n\n==> PP Info: done apply averager filterd raw data\n"
#--- ocarta
if hasattr(tmp_pp_raw.ocarta, "do_run"):
if tmp_pp_raw.ocarta.do_run:
tmp_pp_raw.ocarta['verbose'] = opt.verbose
print "\n===> PP Info: start apply ocarta fit"
print "File :" + fif_file
if opt.verbose:
print "Parameter :"
print tmp_pp_raw.ocarta
print "\n\n"
(fif_file, raw,
fhdf) = jppd.apply_ocarta_data(fif_file,
raw=raw,
**tmp_pp_raw.ocarta)
print "\n\n==> PP Info: done apply ocarta\n ---> " + fif_file
else:
fif_file = jumeg_base.get_fif_name(
fname=fif_file,
postfix=tmp_pp_raw.ocarta.fif_postfix,
extention=tmp_pp_raw.ocarta.fif_extention)
raw = None
else:
jpp_utils.msg_on_attribute_error("raw ocarta")
#--- events
if hasattr(tmp_pp_raw.events, "do_run"):
if tmp_pp_raw.events.do_run:
tmp_pp_raw.events['verbose'] = opt.verbose
print "\n===> PP Info: start apply event export"
print "File :" + fif_file
if opt.verbose:
print "Parameter :"
print tmp_pp_raw.events
print "\n\n"
(fif_file, raw,
event_ids) = jppd.apply_epocher_export_events_data(
fif_file,
raw=raw,
condition_list=opt.conditions,
**tmp_pp_raw.events)
print "\n\n==> PP Info: done apply events\n ---> " + fif_file
print event_ids
print "\n\n"
else:
jpp_utils.msg_on_attribute_error("raw events")
#--- brain response apply mne ica: fastica
if hasattr(tmp_pp_brs.ica, "do_run"):
if tmp_pp_brs.ica.do_run:
tmp_pp_brs.ica['verbose'] = opt.verbose
print "\n===> PP Info: start apply brain-response ica"
print "File : " + fif_file
if opt.verbose:
print "Parameter :"
print tmp_pp_brs.ica
print "\n\n"
(fif_file_ica, raw,
ICAobj) = jppd.apply_ica_data(fif_file,
raw=raw,
**tmp_pp_brs.ica)
print "\n\n==> PP Info: done apply ica for brain responses\n ---> " + fif_file_ica
else:
fif_file_ica = jumeg_base.get_fif_name(
fname=fif_file,
postfix=tmp_pp_brs.ica.fif_postfix,
extention=tmp_pp_brs.ica.fif_extention)
raw = None
else:
jpp_utils.msg_on_attribute_error("brain response ica")
# 201195_Chrono01_110516_1413_1_c,rfDC,fihp1n,ocarta-ica.fif
#--- brain response ctps
#--- run for all condition combine and or exclude CTPs-ICs for different conditions
if hasattr(tmp_pp_brs.ctps, "do_run"):
if tmp_pp_brs.ctps.do_run:
tmp_pp_brs.ctps['verbose'] = opt.verbose
print "\n===> PP Info: start brainresponse ica"
print "File : " + fif_file_ica
if opt.verbose:
print "Parameters:"
print tmp_pp_brs.ctps
print "\n\n"
(fif_file, raw, fhdf) = jppd.apply_ctps_brain_responses_data(
fif_file,
raw=raw,
fname_ica=fif_file_ica,
ica_raw=None,
condition_list=opt.condition,
**tmp_pp_brs.ctps)
print "\n\n==> PP Info: done apply ctp for brain responses\n ---> " + fif_file
else:
fhdf = raw = None
else:
jpp_utils.msg_on_attribute_error("brain response ctps")
#--- brain response ctps ica cleaning
#--- run for global ics, all ctp-ics-condition create raw,epochs,average
if hasattr(tmp_pp_brs.clean, "do_run"):
if tmp_pp_brs.clean.do_run:
tmp_pp_brs.clean['verbose'] = opt.verbose
print "\n===> PP Info: start brain-response cleaning"
print "File : " + fif_file
if opt.verbose:
print "Parameters:"
print tmp_pp_brs.clean
print "\n\n"
fhdf = jppd.apply_ctps_brain_responses_cleaning_data(
fif_file,
raw=raw,
fname_ica=fif_file_ica,
ica_raw=None,
fhdf=fhdf,
condition_list=opt.condition,
**tmp_pp_brs.clean)
else:
jpp_utils.msg_on_attribute_error("brain response clean")
print "===> Done JuMEG Pre-Processing Data"
print " --> FIF file input : " + fif_file_raw
print " --> Information stored in HDF file: " + str(fhdf)
print "\n\n"
raw = None
def noise_reducer_4raw_data(fname_raw,raw=None,signals=[],noiseref=[],detrending=None,
tmin=None,tmax=None,reflp=None,refhp=None,refnotch=None,
exclude_artifacts=True,checkresults=True,
fif_extention="-raw.fif",fif_postfix="nr",
reject={'grad':4000e-13,'mag':4e-12,'eeg':40e-6,'eog':250e-6},
complementary_signal=False,fnout=None,verbose=False,save=True):
"""Apply noise reduction to signal channels using reference channels.
!!! ONLY ONE RAW Obj Interface Version FB !!!
Parameters
----------
fname_raw : rawfile name
raw : fif raw object
signals : list of string
List of channels to compensate using noiseref.
If empty use the meg signal channels.
noiseref : list of string | str
List of channels to use as noise reference.
If empty use the magnetic reference channsls (default).
signals and noiseref may contain regexp, which are resolved
using mne.pick_channels_regexp(). All other channels are copied.
tmin : lower latency bound for weight-calc [start of trace]
tmax : upper latency bound for weight-calc [ end of trace]
Weights are calc'd for (tmin,tmax), but applied to entire data set
refhp : high-pass frequency for reference signal filter [None]
reflp : low-pass frequency for reference signal filter [None]
reflp < refhp: band-stop filter
reflp > refhp: band-pass filter
reflp is not None, refhp is None: low-pass filter
reflp is None, refhp is not None: high-pass filter
refnotch : (base) notch frequency for reference signal filter [None]
use raw(ref)-notched(ref) as reference signal
exclude_artifacts: filter signal-channels thru _is_good() [True]
(parameters are at present hard-coded!)
complementary_signal : replaced signal by traces that would be subtracted [False]
(can be useful for debugging)
checkresults : boolean to control internal checks and overall success [True]
reject = dict for rejection threshold
units:
grad: T / m (gradiometers)
mag: T (magnetometers)
eeg/eog: uV (EEG channels)
default=>{'grad':4000e-13,'mag':4e-12,'eeg':40e-6,'eog':250e-6}
save : save data to fif file
Outputfile:
-------
<wawa>,nr-raw.fif for input <wawa>-raw.fif
Returns
-------
TBD
Bugs
----
- artifact checking is incomplete (and with arb. window of tstep=0.2s)
- no accounting of channels used as signal/reference
- non existing input file handled ungracefully
"""
tc0 = time.clock()
tw0 = time.time()
if type(complementary_signal) != bool:
raise ValueError("Argument complementary_signal must be of type bool")
raw,fname_raw = jumeg_base.get_raw_obj(fname_raw,raw=raw)
if detrending:
raw = perform_detrending(None,raw=raw, save=False)
tc1 = time.clock()
tw1 = time.time()
if verbose:
print ">>> loading raw data took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tc0), (tw1 - tw0))
# Time window selection
# weights are calc'd based on [tmin,tmax], but applied to the entire data set.
# tstep is used in artifact detection
# tmin,tmax variables must not be changed here!
if tmin is None:
itmin = 0
else:
itmin = int(floor(tmin * raw.info['sfreq']))
if tmax is None:
itmax = raw.last_samp
else:
itmax = int(ceil(tmax * raw.info['sfreq']))
if itmax - itmin < 2:
raise ValueError("Time-window for noise compensation empty or too short")
if verbose:
print ">>> Set time-range to [%7.3f,%7.3f]" % \
(raw.index_as_time(itmin)[0], raw.index_as_time(itmax)[0])
if signals is None or len(signals) == 0:
sigpick = jumeg_base.pick_meg_nobads(raw)
else:
sigpick = channel_indices_from_list(raw.info['ch_names'][:], signals,
raw.info.get('bads'))
nsig = len(sigpick)
if nsig == 0:
raise ValueError("No channel selected for noise compensation")
if noiseref is None or len(noiseref) == 0:
# References are not limited to 4D ref-chans, but can be anything,
# incl. ECG or powerline monitor.
if verbose:
print ">>> Using all refchans."
refexclude = "bads"
refpick = jumeg_base.pick_ref_nobads(raw)
else:
refpick = channel_indices_from_list(raw.info['ch_names'][:], noiseref,
raw.info.get('bads'))
nref = len(refpick)
if nref == 0:
raise ValueError("No channel selected as noise reference")
if verbose:
print ">>> sigpick: %3d chans, refpick: %3d chans" % (nsig, nref)
if reflp is None and refhp is None and refnotch is None:
use_reffilter = False
use_refantinotch = False
else:
use_reffilter = True
if verbose:
print "########## Filter reference channels:"
use_refantinotch = False
if refnotch is not None:
if reflp is None and reflp is None:
use_refantinotch = True
freqlast = np.min([5.01 * refnotch, 0.5 * raw.info['sfreq']])
if verbose:
print ">>> notches at freq %.1f and harmonics below %.1f" % (refnotch, freqlast)
else:
raise ValueError("Cannot specify notch- and high-/low-pass"
"reference filter together")
else:
if verbose:
if reflp is not None:
print ">>> low-pass with cutoff-freq %.1f" % reflp
if refhp is not None:
print ">>> high-pass with cutoff-freq %.1f" % refhp
# Adapt followg drop-chans cmd to use 'all-but-refpick'
droplist = [raw.info['ch_names'][k] for k in xrange(raw.info['nchan']) if not k in refpick]
tct = time.clock()
twt = time.time()
fltref = raw.drop_channels(droplist, copy=True)
if use_refantinotch:
rawref = raw.drop_channels(droplist, copy=True)
freqlast = np.min([5.01 * refnotch, 0.5 * raw.info['sfreq']])
fltref.notch_filter(np.arange(refnotch, freqlast, refnotch),
picks=np.array(xrange(nref)), method='iir')
fltref._data = (rawref._data - fltref._data)
else:
fltref.filter(refhp, reflp, picks=np.array(xrange(nref)), method='iir')
tc1 = time.clock()
tw1 = time.time()
if verbose:
print ">>> filtering ref-chans took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
if verbose:
print "########## Calculating sig-ref/ref-ref-channel covariances:"
# Calculate sig-ref/ref-ref-channel covariance:
# (there is no need to calc inter-signal-chan cov,
# but there seems to be no appropriat fct available)
# Here we copy the idea from compute_raw_data_covariance()
# and truncate it as appropriate.
tct = time.clock()
twt = time.time()
# The following reject and infosig entries are only
# used in _is_good-calls.
# _is_good() from mne-0.9.git-py2.7.egg/mne/epochs.py seems to
# ignore ref-channels (not covered by dict) and checks individual
# data segments - artifacts across a buffer boundary are not found.
#--- !!! FB put to kwargs
#reject = dict(grad=4000e-13, # T / m (gradiometers)
# mag=4e-12, # T (magnetometers)
# eeg=40e-6, # uV (EEG channels)
# eog=250e-6) # uV (EOG channels)
infosig = copy.copy(raw.info)
infosig['chs'] = [raw.info['chs'][k] for k in sigpick]
infosig['ch_names'] = [raw.info['ch_names'][k] for k in sigpick]
infosig['nchan'] = len(sigpick)
idx_by_typesig = channel_indices_by_type(infosig)
# Read data in chunks:
tstep = 0.2
itstep = int(ceil(tstep * raw.info['sfreq']))
sigmean = 0
refmean = 0
sscovdata = 0
srcovdata = 0
rrcovdata = 0
n_samples = 0
for first in range(itmin, itmax, itstep):
last = first + itstep
if last >= itmax:
last = itmax
raw_segmentsig, times = raw[sigpick, first:last]
if use_reffilter:
raw_segmentref, times = fltref[:, first:last]
else:
raw_segmentref, times = raw[refpick, first:last]
if not exclude_artifacts or \
_is_good(raw_segmentsig, infosig['ch_names'], idx_by_typesig, reject, flat=None,
ignore_chs=raw.info['bads']):
sigmean += raw_segmentsig.sum(axis=1)
refmean += raw_segmentref.sum(axis=1)
sscovdata += (raw_segmentsig * raw_segmentsig).sum(axis=1)
srcovdata += np.dot(raw_segmentsig, raw_segmentref.T)
rrcovdata += np.dot(raw_segmentref, raw_segmentref.T)
n_samples += raw_segmentsig.shape[1]
else:
logger.info("Artefact detected in [%d, %d]" % (first, last))
if n_samples <= 1:
raise ValueError('Too few samples to calculate weights')
sigmean /= n_samples
refmean /= n_samples
sscovdata -= n_samples * sigmean[:] * sigmean[:]
sscovdata /= (n_samples - 1)
srcovdata -= n_samples * sigmean[:, None] * refmean[None, :]
srcovdata /= (n_samples - 1)
rrcovdata -= n_samples * refmean[:, None] * refmean[None, :]
rrcovdata /= (n_samples - 1)
sscovinit = np.copy(sscovdata)
if verbose:
print ">>> Normalize srcov..."
rrslope = copy.copy(rrcovdata)
for iref in xrange(nref):
dtmp = rrcovdata[iref, iref]
if dtmp > TINY:
srcovdata[:, iref] /= dtmp
rrslope[:, iref] /= dtmp
else:
srcovdata[:, iref] = 0.
rrslope[:, iref] = 0.
if verbose:
print ">>> Number of samples used : %d" % n_samples
tc1 = time.clock()
tw1 = time.time()
print ">>> sigrefchn covar-calc took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
if checkresults:
if verbose:
print "########## Calculated initial signal channel covariance:"
# Calculate initial signal channel covariance:
# (only used as quality measure)
print ">>> initl rt(avg sig pwr) = %12.5e" % np.sqrt(np.mean(sscovdata))
for i in xrange(5):
print ">>> initl signal-rms[%3d] = %12.5e" % (i, np.sqrt(sscovdata.flatten()[i]))
print ">>>"
U, s, V = np.linalg.svd(rrslope, full_matrices=True)
if verbose:
print ">>> singular values:"
print s
print ">>> Applying cutoff for smallest SVs:"
dtmp = s.max() * SVD_RELCUTOFF
s *= (abs(s) >= dtmp)
sinv = [1. / s[k] if s[k] != 0. else 0. for k in xrange(nref)]
if verbose:
print ">>> singular values (after cutoff):"
print s
stat = np.allclose(rrslope, np.dot(U, np.dot(np.diag(s), V)))
if verbose:
print ">>> Testing svd-result: %s" % stat
if not stat:
print " (Maybe due to SV-cutoff?)"
# Solve for inverse coefficients:
# Set RRinv.tr=U diag(sinv) V
RRinv = np.transpose(np.dot(U, np.dot(np.diag(sinv), V)))
if checkresults:
stat = np.allclose(np.identity(nref), np.dot(RRinv, rrslope))
if stat:
if verbose:
print ">>> Testing RRinv-result (should be unit-matrix): ok"
else:
print ">>> Testing RRinv-result (should be unit-matrix): failed"
print np.transpose(np.dot(RRinv, rrslope))
print ">>>"
if verbose:
print "########## Calc weight matrix..."
# weights-matrix will be somewhat larger than necessary,
# (to simplify indexing in compensation loop):
weights = np.zeros((raw._data.shape[0], nref))
for isig in xrange(nsig):
for iref in xrange(nref):
weights[sigpick[isig],iref] = np.dot(srcovdata[isig,:], RRinv[:,iref])
if verbose:
print "########## Compensating signal channels:"
if complementary_signal:
print ">>> Caveat: REPLACING signal by compensation signal"
tct = time.clock()
twt = time.time()
# Work on entire data stream:
for isl in xrange(raw._data.shape[1]):
slice = np.take(raw._data, [isl], axis=1)
if use_reffilter:
refslice = np.take(fltref._data, [isl], axis=1)
refarr = refslice[:].flatten() - refmean
# refarr = fltres[:,isl]-refmean
else:
refarr = slice[refpick].flatten() - refmean
subrefarr = np.dot(weights[:], refarr)
if not complementary_signal:
raw._data[:, isl] -= subrefarr
else:
raw._data[:, isl] = subrefarr
if (isl % 10000 == 0) and verbose:
print "\rProcessed slice %6d" % isl
if verbose:
print "\nDone."
tc1 = time.clock()
tw1 = time.time()
print ">>> compensation loop took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
if checkresults:
if verbose:
print "########## Calculating final signal channel covariance:"
# Calculate final signal channel covariance:
# (only used as quality measure)
tct = time.clock()
twt = time.time()
sigmean = 0
sscovdata = 0
n_samples = 0
for first in range(itmin, itmax, itstep):
last = first + itstep
if last >= itmax:
last = itmax
raw_segmentsig, times = raw[sigpick, first:last]
# Artifacts found here will probably differ from pre-noisered artifacts!
if not exclude_artifacts or \
_is_good(raw_segmentsig, infosig['ch_names'], idx_by_typesig, reject,
flat=None, ignore_chs=raw.info['bads']):
sigmean += raw_segmentsig.sum(axis=1)
sscovdata += (raw_segmentsig * raw_segmentsig).sum(axis=1)
n_samples += raw_segmentsig.shape[1]
sigmean /= n_samples
sscovdata -= n_samples * sigmean[:] * sigmean[:]
sscovdata /= (n_samples - 1)
if verbose:
print ">>> no channel got worse: ", np.all(np.less_equal(sscovdata, sscovinit))
print ">>> final rt(avg sig pwr) = %12.5e" % np.sqrt(np.mean(sscovdata))
for i in xrange(5):
print ">>> final signal-rms[%3d] = %12.5e" % (i, np.sqrt(sscovdata.flatten()[i]))
tc1 = time.clock()
tw1 = time.time()
print ">>> signal covar-calc took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
print ">>>"
#--- fb update 21.07.2015
fname_out = jumeg_base.get_fif_name(raw=raw,postfix=fif_postfix,extention=fif_extention)
if save:
jumeg_base.apply_save_mne_data(raw,fname=fname_out,overwrite=True)
tc1 = time.clock()
tw1 = time.time()
if verbose:
print ">>> Total run took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tc0), (tw1 - tw0))
return raw,fname_out
def _epochs_save_events(self,
raw=None,
evt=None,
condition=None,
postfix=None,
postfix_extention="evt",
weighted=False,
picks=None,
reject=None,
proj=False,
save_mode={
"events": True,
"epochs": True,
"evoked": True
}):
"""saving event-,epoch-,evoked data using mnefunctions with file nameing conventions
saving results as *.png in /XYZ/plots folder
Parameters
-----------
raw : raw obj <None> use global
evt : event dict <None>
condition: name of condition <None>
postfix : filename postfix = <condition postfix> from HDF and template file <None>
postfix_extention: final postfix extention e.g.: CONDI_evt <"evt">
MNE parameter for mne.epochs
picks : channel index array []
reject : None
proj : False,
save_mode : <{"events":True,"epochs":True,"evoked":True}>
events: save mne events <...,CONDI_evt.eve>
epochs: save mne epochs <...,CONDI_evt-epo.fif>
evoked: save mne evoked, averaged data <...,CONDI_evt_bc-ave.fif>
if baseline corrected: <...,CONDI_evt_bc.xyz>
weighted : sort of equal number of trail over all conditions <False>
e.g: response matching task or EOG artifact rejection
if weighted <...,CONDI_evtW.xyz>
if weighted and baseline corrected
<...,CONDI_evtW_bc.xyz>
Returns
-----------
None
"""
if raw:
self.raw = raw
if postfix:
postfix += "_" + postfix_extention
else:
postfix = postfix_extention
if evt['baseline_corrected']:
postfix += '_bc'
#--- save events to txt file
if save_mode["events"]:
fname = jumeg_base.get_fif_name(raw=self.raw,
postfix=postfix,
extention=".eve",
update_raw_fname=False)
mne.event.write_events(fname, evt['events'])
print " ---> done jumeg epocher save events as => EVENTS :" + fname
print " -> number of events : %d" % (evt['events'].shape[0])
self.line()
#--- save epoch data
if save_mode["epochs"]:
fname = jumeg_base.get_fif_name(raw=self.raw,
postfix=postfix,
extention="-epo.fif",
update_raw_fname=False)
evt['epochs'].save(fname)
print " ---> done jumeg epocher save events as => EPOCHS :" + fname + "\n"
self.line()
#--- save averaged data
if save_mode["evoked"]:
fname = jumeg_base.get_fif_name(raw=self.raw,
postfix=postfix,
extention="-ave.fif",
update_raw_fname=False)
#evt_avg = evt['epochs'].average()
#print evt_avg
#evt_avg.plot()
mne.write_evokeds(
fname, evt['epochs'].average(picks=jumeg_base.picks.all(
self.raw))) # in case avg only trigger or response
print " ---> done jumeg epocher save events as => EVOKED (averaged) :" + fname + "\n"
fname = jumeg_base.get_fif_name(raw=self.raw,
postfix=postfix,
extention="-ave",
update_raw_fname=False)
#--- plot evoked
fname = jplt.plot_evoked(evt['epochs'],
fname=fname,
condition=condition,
show_plot=False,
save_plot=True,
plot_dir='plots')
print " ---> done jumeg epocher plot evoked (averaged) :" + fname + "\n"
self.line()
def events_save_events(self,
evt=None,
condition=None,
postfix="evt",
picks=None,
reject=None,
proj=False,
save_condition={
"events": True,
"epochs": True,
"evoked": True
}):
from jumeg.preprocbatch.jumeg_preprocbatch_plot import jumeg_preprocbatch_plot as jplt
jplt.verbose = self.verbose
ep, bc = self.events_apply_epochs_and_baseline(self.raw,
evt=evt,
reject=reject,
proj=proj,
picks=picks)
postfix += '_' + condition
if bc:
postfix += '_bc'
#--- save events to txt file
if save_condition["events"]:
fname = jumeg_base.get_fif_name(raw=self.raw,
postfix=postfix,
extention=".eve",
update_raw_fname=False)
mne.event.write_events(fname, evt['events'])
print " ---> done jumeg epocher save events as => EVENTS :" + fname
#--- save epoch data
if save_condition["epochs"]:
fname = jumeg_base.get_fif_name(raw=self.raw,
postfix=postfix,
extention="-epo.fif",
update_raw_fname=False)
ep.save(fname)
print " ---> done jumeg epocher save events as => EPOCHS :" + fname
#--- save averaged data
if save_condition["evoked"]:
fname = jumeg_base.get_fif_name(raw=self.raw,
postfix=postfix,
extention="-ave.fif",
update_raw_fname=False)
mne.write_evokeds(fname, ep.average())
print " ---> done jumeg epocher save events as => EVOKED (averaged) :" + fname
fname = jumeg_base.get_fif_name(raw=self.raw,
postfix=postfix,
extention="-ave",
update_raw_fname=False)
#--- plot evoked
fname = jplt.plot_evoked(ep,
fname=fname,
condition=condition,
show_plot=False,
save_plot=True,
plot_dir='plots')
print " ---> done jumeg epocher plot evoked (averaged) :" + fname
