def apply_filter_data(fname,raw=None,filter_method="mne",filter_type='bp',fcut1=1.0,fcut2=45.0,notch=None,notch_max=None,order=4,
remove_dcoffset = False,njobs=1,overwrite = False,do_run=True,verbose=False,save=True,picks=None,
fif_postfix=None, fif_extention='-raw.fif'):
'''
Applies the FIR FFT filter [bp,hp,lp,notches] to data array.
filter_method : mne => fft mne-filter
bw => fft butterwoth
ws => fft - windowed sinc
'''
from jumeg.filter import jumeg_filter
#--- define filter
jfilter = jumeg_filter(filter_method=filter_method,filter_type=filter_type,fcut1=fcut1,fcut2=fcut2,njobs=njobs,
remove_dcoffset=True,order=order)
jfilter.verbose = verbose
if do_run :
if raw is None:
if fname is None:
print"ERROR no file foumd!!\n"
return
raw = mne.io.Raw(fname,preload=True)
print"\n"
if picks is None :
picks = jumeg_base.pick_channels_nobads(raw)
#- apply filter for picks, exclude stim,resp,bads
jfilter.sampling_frequency = raw.info['sfreq']
#--- calc notch array 50,100,150 .. max
if notch :
jfilter.calc_notches(notch,notch_max)
jfilter.apply_filter(raw._data,picks=picks )
jfilter.update_info_filter_settings(raw)
#--- make output filename
name_raw = fname.split('-')[0]
fnfilt = name_raw + "," + jfilter.filter_name_postfix + fif_extention
raw.info['filename'] = fnfilt
if save :
fnfilt = jumeg_base.apply_save_mne_data(raw,fname=fnfilt)
else:
#--- calc notch array 50,100,150 .. max
if notch :
jfilter.calc_notches(notch,notch_max)
#--- make output filename
name_raw = fname.split('-')[0]
fnfilt = name_raw + "," + jfilter.filter_name_postfix + fif_extention
return (fnfilt, raw)
def apply_ica(fname_filtered, n_components=0.99, decim=None,
reject={'mag': 5e-12}, ica_method='fastica',
flow=None, fhigh=None, verbose=True):
''' Applies ICA to a list of (filtered) raw files. '''
from mne.preprocessing import ICA
fnfilt = get_files_from_list(fname_filtered)
# loop across all filenames
for fname in fnfilt:
name = os.path.split(fname)[1]
print ">>>> perform ICA signal decomposition on : " + name
# load filtered data
raw = mne.io.Raw(fname, preload=True)
picks = mne.pick_types(raw.info, meg=True, ref_meg=False, exclude='bads')
# check if data to estimate the optimal
# de-mixing matrix should be filtered
if flow or fhigh:
from jumeg.filter import jumeg_filter
# define filter type
if not flow:
filter_type = 'lp'
filter_info = " --> filter parameter : filter type=low pass %dHz" % flow
elif not fhigh:
filter_type = 'hp'
filter_info = " --> filter parameter : filter type=high pass %dHz" % flow
else:
filter_type = 'bp'
filter_info = " --> filter parameter: filter type=band pass %d-%dHz" % (flow, fhigh)
if verbose:
print ">>>> NOTE: Optimal cleaning parameter are estimated from filtered data!"
print filter_info
fi_mne_notch = jumeg_filter(fcut1=flow, fcut2=fhigh, filter_type=filter_type,
remove_dcoffset=False,
sampling_frequency=raw.info['sfreq'])
fi_mne_notch.apply_filter(raw._data, picks=picks)
# ICA decomposition
ica = ICA(method=ica_method, n_components=n_components,
max_pca_components=None)
ica.fit(raw, picks=picks, decim=decim, reject=reject)
# save ICA object
fnica_out = fname[:fname.rfind(ext_raw)] + ext_ica
# fnica_out = fname[0:len(fname)-4]+'-ica.fif'
ica.save(fnica_out)
def apply_ica(fname_filtered, n_components=0.99, decim=None,
reject={'mag': 5e-12}, ica_method='fastica',
flow=None, fhigh=None, verbose=True):
''' Applies ICA to a list of (filtered) raw files. '''
from mne.preprocessing import ICA
fnfilt = get_files_from_list(fname_filtered)
# loop across all filenames
for fname in fnfilt:
name = os.path.split(fname)[1]
print ">>>> perform ICA signal decomposition on : " + name
# load filtered data
raw = mne.io.Raw(fname, preload=True)
picks = mne.pick_types(raw.info, meg=True, ref_meg=False, exclude='bads')
# check if data to estimate the optimal
# de-mixing matrix should be filtered
if flow or fhigh:
from jumeg.filter import jumeg_filter
# define filter type
if not flow:
filter_type = 'lp'
filter_info = " --> filter parameter : filter type=low pass %dHz" % flow
elif not fhigh:
filter_type = 'hp'
filter_info = " --> filter parameter : filter type=high pass %dHz" % flow
else:
filter_type = 'bp'
filter_info = " --> filter parameter: filter type=band pass %d-%dHz" % (flow, fhigh)
if verbose:
print ">>>> NOTE: Optimal cleaning parameter are estimated from filtered data!"
print filter_info
fi_mne_notch = jumeg_filter(fcut1=flow, fcut2=fhigh, filter_type=filter_type,
remove_dcoffset=False,
sampling_frequency=raw.info['sfreq'])
fi_mne_notch.apply_filter(raw._data, picks=picks)
# ICA decomposition
ica = ICA(method=ica_method, n_components=n_components,
max_pca_components=None)
ica.fit(raw, picks=picks, decim=decim, reject=reject)
# save ICA object
fnica_out = fname[:fname.rfind(ext_raw)] + ext_ica
# fnica_out = fname[0:len(fname)-4]+'-ica.fif'
ica.save(fnica_out)
if fi_method_str == 'mne_orig':
raw.filter(opt.fcut1,
opt.fcut2,
n_jobs=opt.njobs,
method='fft',
picks=opt.picks)
plt_legends.append('mne_orig')
else:
#--- create your filter type object
fi[fi_method_str] = jumeg_filter(filter_method=fi_method_str,
filter_type=opt.type,
fcut1=opt.fcut1,
fcut2=opt.fcut2,
remove_dcoffset=opt.remove_dcoffset,
notch=opt.notch,
notch_width=opt.notch_width,
sampling_frequency=sfreq,
order=opt.order)
#sampling_frequency=sfreq)
fi[fi_method_str].apply_filter(raw._data, opt.picks)
plt_legends.append(fi[fi_method_str].filter_info_short)
print fi[fi_method_str].filter_info
raw.plot_psds(picks=opt.picks,
fmin=opt.fmin,
fmax=opt.fmax,
ax=ax,
def apply_ctps(fname_ica, freqs=[(1, 4), (4, 8), (8, 12), (12, 16), (16, 20)],
tmin=-0.2, tmax=0.4, name_stim='STI 014', event_id=None,
baseline=(None, 0), proj=False):
''' Applies CTPS to a list of ICA files. '''
from jumeg.filter import jumeg_filter
fiws = jumeg_filter(filter_method="bw")
fiws.filter_type = 'bp' # bp, lp, hp
fiws.dcoffset = True
fiws.filter_attenuation_factor = 1
nfreq = len(freqs)
print '>>> CTPS calculation on: ', freqs
# Trigger or Response ?
if name_stim == 'STI 014': # trigger
trig_name = 'trigger'
else:
if name_stim == 'STI 013': # response
trig_name = 'response'
else:
trig_name = 'auxillary'
fnlist = get_files_from_list(fname_ica)
# loop across all filenames
for fnica in fnlist:
name = os.path.split(fnica)[1]
#fname = fnica[0:len(fnica)-4]
basename = fnica[:fnica.rfind(ext_ica)]
fnraw = basename + ext_raw
#basename = os.path.splitext(os.path.basename(fnica))[0]
# load cleaned data
raw = mne.io.Raw(fnraw, preload=True)
picks = mne.pick_types(raw.info, meg=True, ref_meg=False, exclude='bads')
# read (second) ICA
print ">>>> working on: " + basename
ica = mne.preprocessing.read_ica(fnica)
ica_picks = np.arange(ica.n_components_)
ncomp = len(ica_picks)
# stim events
stim_events = mne.find_events(raw, stim_channel=name_stim, consecutive=True)
nevents = len(stim_events)
if (nevents > 0):
# for a specific event ID
if event_id:
ix = np.where(stim_events[:, 2] == event_id)[0]
stim_events = stim_events[ix, :]
else:
event_id = stim_events[0, 2]
# create ctps dictionary
dctps = {'fnica': fnica,
'basename': basename,
'stim_channel': name_stim,
'trig_name': trig_name,
'ncomp': ncomp,
'nevent': nevents,
'event_id': event_id,
'nfreq': nfreq,
'freqs': freqs,
}
# loop across all filenames
pkarr = []
ptarr = []
pkmax_arr = []
for ifreq in range(nfreq):
ica_raw = ica.get_sources(raw)
flow, fhigh = freqs[ifreq][0], freqs[ifreq][1]
bp = str(flow) + '_' + str(fhigh)
# filter ICA data and create epochs
#tw=0.1
# ica_raw.filter(l_freq=flow, h_freq=fhigh, picks=ica_picks,
# method='fft',l_trans_bandwidth=tw, h_trans_bandwidth=tw)
# ica_raw.filter(l_freq=flow, h_freq=fhigh, picks=ica_picks,
# method='fft')
# filter ws settings
# later we will make this as a one line call
data_length = raw._data[0, :].size
fiws.sampling_frequency = raw.info['sfreq']
fiws.fcut1 = flow
fiws.fcut2 = fhigh
#fiws.init_filter_kernel(data_length)
#fiws.init_filter(data_length)
for ichan in ica_picks:
fiws.apply_filter(ica_raw._data[ichan, :])
ica_epochs = mne.Epochs(ica_raw, events=stim_events,
event_id=event_id, tmin=tmin,
tmax=tmax, verbose=False,
picks=ica_picks, baseline=baseline,
proj=proj)
# compute CTPS
_, pk, pt = ctps.ctps(ica_epochs.get_data())
pkmax = pk.max(1)
times = ica_epochs.times * 1e3
pkarr.append(pk)
ptarr.append(pt)
pkmax_arr.append(pkmax)
pkarr = np.array(pkarr)
ptarr = np.array(ptarr)
pkmax_arr = np.array(pkmax_arr)
dctps['pk'] = np.float32(pkarr)
dctps['pt'] = np.float32(ptarr)
dctps['pkmax'] = np.float32(pkmax_arr)
dctps['nsamp'] = len(times)
dctps['times'] = np.float32(times)
dctps['tmin'] = np.float32(ica_epochs.tmin)
dctps['tmax'] = np.float32(ica_epochs.tmax)
fnctps = basename + prefix_ctps + trig_name
np.save(fnctps, dctps)
# Note; loading example: dctps = np.load(fnctps).items()
else:
event_id = None
def apply_ica_cleaning(fname_ica, n_pca_components=None,
name_ecg='ECG 001', flow_ecg=10, fhigh_ecg=20,
name_eog_hor='EOG 001', name_eog_ver='EOG 002',
flow_eog=1, fhigh_eog=10, threshold=0.3,
unfiltered=False, notch_filter=True, notch_freq=50,
notch_width=None):
''' Performs artifact rejection based on ICA to a list of (ICA) files. '''
fnlist = get_files_from_list(fname_ica)
# loop across all filenames
for fnica in fnlist:
name = os.path.split(fnica)[1]
#basename = fnica[0:len(fnica)-4]
basename = fnica[:fnica.rfind(ext_ica)]
fnfilt = basename + ext_raw
fnclean = basename + ext_clean
fnica_ar = basename + ext_icap
print ">>>> perform artifact rejection on :"
print ' ' + name
# load filtered data
meg_raw = mne.io.Raw(fnfilt, preload=True)
picks = mne.pick_types(meg_raw.info, meg=True, ref_meg=False, exclude='bads')
# ICA decomposition
ica = mne.preprocessing.read_ica(fnica)
# get ECG and EOG related components
ic_ecg = get_ics_cardiac(meg_raw, ica,
flow=flow_ecg, fhigh=fhigh_ecg, thresh=threshold)
ic_eog = get_ics_ocular(meg_raw, ica,
flow=flow_eog, fhigh=fhigh_eog, thresh=threshold)
ica.exclude += list(ic_ecg) + list(ic_eog)
# ica.plot_topomap(ic_artefacts)
ica.save(fnica) # save again to store excluded
# clean and save MEG data
if n_pca_components:
npca = n_pca_components
else:
npca = picks.size
# check if cleaning should be applied
# to unfiltered data
if unfiltered:
# adjust filenames to unfiltered data
basename = basename[:basename.rfind(',')]
fnfilt = basename + ext_raw
fnclean = basename + ext_clean
fnica_ar = basename + ext_icap
# load raw unfiltered data
meg_raw = mne.io.Raw(fnfilt, preload=True)
# apply notch filter
if notch_filter:
from jumeg.filter import jumeg_filter
# generate and apply filter
# check if array of frequencies is given
if type(notch_freq) in (tuple, list):
notch = np.array(notch_freq)
elif type(np.ndarray) == np.ndarray:
notch = notch_freq
# or a single frequency
else:
notch = np.array([])
fi_mne_notch = jumeg_filter(filter_method="mne", filter_type='notch',
remove_dcoffset=False,
notch=notch, notch_width=notch_width)
# if only a single frequency is given generate optimal
# filter parameter to also remove the harmonics
if not type(notch_freq) in (tuple, list, np.ndarray):
fi_mne_notch.calc_notches(notch_freq)
fi_mne_notch.apply_filter(meg_raw._data, picks=picks)
# apply cleaning
meg_clean = ica.apply(meg_raw, exclude=ica.exclude,
n_pca_components=npca, copy=True)
meg_clean.save(fnclean, overwrite=True)
# plot ECG, EOG averages before and after ICA
print ">>>> create performance image..."
plot_performance_artifact_rejection(meg_raw, ica, fnica_ar,
show=False, verbose=False)
def apply_ctps(fname_ica, freqs=[(1, 4), (4, 8), (8, 12), (12, 16), (16, 20)],
tmin=-0.2, tmax=0.4, name_stim='STI 014', event_id=None,
baseline=(None, 0), proj=False):
''' Applies CTPS to a list of ICA files. '''
from jumeg.filter import jumeg_filter
fiws = jumeg_filter(filter_method="bw")
fiws.filter_type = 'bp' # bp, lp, hp
fiws.dcoffset = True
fiws.filter_attenuation_factor = 1
nfreq = len(freqs)
print '>>> CTPS calculation on: ', freqs
# Trigger or Response ?
if name_stim == 'STI 014': # trigger
trig_name = 'trigger'
else:
if name_stim == 'STI 013': # response
trig_name = 'response'
else:
trig_name = 'auxillary'
fnlist = get_files_from_list(fname_ica)
# loop across all filenames
for fnica in fnlist:
name = os.path.split(fnica)[1]
#fname = fnica[0:len(fnica)-4]
basename = fnica[:fnica.rfind(ext_ica)]
fnraw = basename + ext_raw
#basename = os.path.splitext(os.path.basename(fnica))[0]
# load cleaned data
raw = mne.io.Raw(fnraw, preload=True)
picks = mne.pick_types(raw.info, meg=True, ref_meg=False, exclude='bads')
# read (second) ICA
print ">>>> working on: " + basename
ica = mne.preprocessing.read_ica(fnica)
ica_picks = np.arange(ica.n_components_)
ncomp = len(ica_picks)
# stim events
stim_events = mne.find_events(raw, stim_channel=name_stim, consecutive=True)
nevents = len(stim_events)
if (nevents > 0):
# for a specific event ID
if event_id:
ix = np.where(stim_events[:, 2] == event_id)[0]
stim_events = stim_events[ix, :]
else:
event_id = stim_events[0, 2]
# create ctps dictionary
dctps = {'fnica': fnica,
'basename': basename,
'stim_channel': name_stim,
'trig_name': trig_name,
'ncomp': ncomp,
'nevent': nevents,
'event_id': event_id,
'nfreq': nfreq,
'freqs': freqs,
}
# loop across all filenames
pkarr = []
ptarr = []
pkmax_arr = []
for ifreq in range(nfreq):
ica_raw = ica.get_sources(raw)
flow, fhigh = freqs[ifreq][0], freqs[ifreq][1]
bp = str(flow) + '_' + str(fhigh)
# filter ICA data and create epochs
#tw=0.1
# ica_raw.filter(l_freq=flow, h_freq=fhigh, picks=ica_picks,
# method='fft',l_trans_bandwidth=tw, h_trans_bandwidth=tw)
# ica_raw.filter(l_freq=flow, h_freq=fhigh, picks=ica_picks,
# method='fft')
# filter ws settings
# later we will make this as a one line call
data_length = raw._data[0, :].size
fiws.sampling_frequency = raw.info['sfreq']
fiws.fcut1 = flow
fiws.fcut2 = fhigh
#fiws.init_filter_kernel(data_length)
#fiws.init_filter(data_length)
for ichan in ica_picks:
fiws.apply_filter(ica_raw._data[ichan, :])
ica_epochs = mne.Epochs(ica_raw, events=stim_events,
event_id=event_id, tmin=tmin,
tmax=tmax, verbose=False,
picks=ica_picks, baseline=baseline,
proj=proj)
# compute CTPS
_, pk, pt = ctps.ctps(ica_epochs.get_data())
pkmax = pk.max(1)
times = ica_epochs.times * 1e3
pkarr.append(pk)
ptarr.append(pt)
pkmax_arr.append(pkmax)
pkarr = np.array(pkarr)
ptarr = np.array(ptarr)
pkmax_arr = np.array(pkmax_arr)
dctps['pk'] = np.float32(pkarr)
dctps['pt'] = np.float32(ptarr)
dctps['pkmax'] = np.float32(pkmax_arr)
dctps['nsamp'] = len(times)
dctps['times'] = np.float32(times)
dctps['tmin'] = np.float32(ica_epochs.tmin)
dctps['tmax'] = np.float32(ica_epochs.tmax)
fnctps = basename + prefix_ctps + trig_name
np.save(fnctps, dctps)
# Note; loading example: dctps = np.load(fnctps).items()
else:
event_id = None
def apply_ica_cleaning(fname_ica, n_pca_components=None,
name_ecg='ECG 001', flow_ecg=10, fhigh_ecg=20,
name_eog_hor='EOG 001', name_eog_ver='EOG 002',
flow_eog=1, fhigh_eog=10, threshold=0.3,
unfiltered=False, notch_filter=True, notch_freq=50,
notch_width=None):
''' Performs artifact rejection based on ICA to a list of (ICA) files. '''
fnlist = get_files_from_list(fname_ica)
# loop across all filenames
for fnica in fnlist:
name = os.path.split(fnica)[1]
#basename = fnica[0:len(fnica)-4]
basename = fnica[:fnica.rfind(ext_ica)]
fnfilt = basename + ext_raw
fnclean = basename + ext_clean
fnica_ar = basename + ext_icap
print ">>>> perform artifact rejection on :"
print ' ' + name
# load filtered data
meg_raw = mne.io.Raw(fnfilt, preload=True)
picks = mne.pick_types(meg_raw.info, meg=True, ref_meg=False, exclude='bads')
# ICA decomposition
ica = mne.preprocessing.read_ica(fnica)
# get ECG and EOG related components
ic_ecg = get_ics_cardiac(meg_raw, ica,
flow=flow_ecg, fhigh=fhigh_ecg, thresh=threshold)
ic_eog = get_ics_ocular(meg_raw, ica,
flow=flow_eog, fhigh=fhigh_eog, thresh=threshold)
ica.exclude += list(ic_ecg) + list(ic_eog)
# ica.plot_topomap(ic_artefacts)
ica.save(fnica) # save again to store excluded
# clean and save MEG data
if n_pca_components:
npca = n_pca_components
else:
npca = picks.size
# check if cleaning should be applied
# to unfiltered data
if unfiltered:
# adjust filenames to unfiltered data
# FIXME breaks when noise reduced file is used with basename + ',nr'
# temporarily fixed by checking for fibp filter suffix
if basename.find(',fibp') != -1:
basename = basename[:basename.rfind(',')]
else:
basename = basename
fnfilt = basename + ext_raw
fnclean = basename + ext_clean
fnica_ar = basename + ext_icap
# load raw unfiltered data
meg_raw = mne.io.Raw(fnfilt, preload=True)
# apply notch filter
if notch_filter:
from jumeg.filter import jumeg_filter
# generate and apply filter
# check if array of frequencies is given
if type(notch_freq) in (tuple, list):
notch = np.array(notch_freq)
elif type(np.ndarray) == np.ndarray:
notch = notch_freq
# or a single frequency
else:
notch = np.array([])
fi_mne_notch = jumeg_filter(filter_method="mne", filter_type='notch',
remove_dcoffset=False,
notch=notch, notch_width=notch_width)
# if only a single frequency is given generate optimal
# filter parameter to also remove the harmonics
if not type(notch_freq) in (tuple, list, np.ndarray):
fi_mne_notch.calc_notches(notch_freq)
fi_mne_notch.apply_filter(meg_raw._data, picks=picks)
# apply cleaning
meg_clean = ica.apply(meg_raw.copy(), exclude=ica.exclude,
n_pca_components=npca)
meg_clean.save(fnclean, overwrite=True)
# plot ECG, EOG averages before and after ICA
print ">>>> create performance image..."
plot_performance_artifact_rejection(meg_raw, ica, fnica_ar,
show=False, verbose=False,
name_ecg=name_ecg,
name_eog=name_eog_ver)
h_trans_bandwidth='auto',
n_jobs=4,
method='iir',
iir_params=iir_params,
phase='zero',
fir_window='hamming',
verbose='ERROR')
# apply the jumeg filter using mne
filt_ju_mne = jumeg_filter(filter_method='mne',
filter_type='bp',
fcut1=l_freq,
fcut2=h_freq,
remove_dcoffset=True,
sampling_frequency=raw.info['sfreq'],
filter_window='hamming',
notch=np.array([50., 60.]),
notch_width=1.0,
order=4,
njobs=4,
mne_filter_method='fft',
mne_filter_length='10s',
trans_bandwith=0.5)
ju_mne_filt = raw.copy() # make a copy
ju_mne_filt._data = filt_ju_mne.apply_filter(ju_mne_filt._data, picks)
# apply the jumeg filter using bw
filt_ju_bw = jumeg_filter(filter_method='bw',
filter_type='bp',
fcut1=l_freq,
fcut2=h_freq,
remove_dcoffset=True,
plt_legends = []
fi = {}
for fi_method in opt.methods:
fi_method_str = str(fi_method)
if fi_method_str == 'mne_orig' :
raw.filter(opt.fcut1, opt.fcut2, n_jobs=opt.njobs, method='fft',picks=opt.picks)
plt_legends.append('mne_orig')
else:
#--- create your filter type object
fi[ fi_method_str ] = jumeg_filter(filter_method=fi_method_str,filter_type=opt.type,fcut1=opt.fcut1,fcut2=opt.fcut2,remove_dcoffset=opt.remove_dcoffset,
notch=opt.notch,
notch_width=opt.notch_width,
sampling_frequency=sfreq,
order=opt.order)
#sampling_frequency=sfreq)
fi[fi_method_str].apply_filter(raw._data,opt.picks)
plt_legends.append( fi[fi_method_str].filter_info_short)
print fi[fi_method_str].filter_info
raw.plot_psds(picks=opt.picks,fmin=opt.fmin,fmax=opt.fmax,ax=ax,n_fft=opt.nfft,color=color[fi_method])
#--- copy data back for next filter method
raw._data[opt.picks,:] = data_meg_save