def preprocessing_raw(fpath, subj):
# load data and read event channels
rawlist = []
evelist = []
# extracting bdf filenames into bdfs
bdfs = fnmatch.filter(os.listdir(fpath), subj + '_ITD*.bdf')
if len(bdfs) >= 1:
for k, bdf in enumerate(bdfs):
rawtemp, evestemp = bs.importbdf(fpath + bdf)
rawlist += [rawtemp,]
evelist += [evestemp,]
else:
RuntimeError("No bdf files found!")
raw, eves = mne.concatenate_raws(rawlist, events_list = evelist)
# Things to try if data are too noisy
# raw.info['bads'] += ['A20'] # if A7, A6, A24 have been rejected a lot, they can be added manually
# raw.set_eeg_reference(ref_channels = ['Average']) # referencing to the average of all channels, do this if channels are too noisy
# the returned eves has three columns: sample number, value of onset, value of offset.
eves2 = eves.copy()
eves2[:, 1] = np.mod(eves2[:, 1], 256)
eves2[:, 2] = np.mod(eves2[:, 2], 256)
# raw.plot(events=eves2)
raw.filter(l_freq = 1, h_freq=50) # if channels are noisy, adjust the filter parameters, high-pass frequency is better set at 1 compared to 0.5 Hz, to get rid of low-frequency drift
# SSP for blinks
blinks = find_blinks(raw, ch_name = ['A1',], l_trans_bandwidth=0.5,
l_freq=1.0) # A1 is the closest electrode to eyebrow, raw.plot(events=blinks) shows the lines at eye blinks, 998 is default event_id
epochs_blinks = mne.Epochs(raw, blinks, 998, tmin = -0.25, tmax = 0.25,
proj = False, baseline = (-0.25, 0),
reject=dict(eeg=500e-6))
n_eeg = 4
blink_projs = compute_proj_epochs(epochs_blinks, n_grad=0,
n_mag=0, n_eeg=n_eeg,
verbose='DEBUG')
if subj in ['S025', 'S031', 'S046', 'S117', 'S123', 'S127', 'S128', 'S132', 'S133', 'S143', 'S149', 'S051', 'S183', 'S185', 'S187', 'S189', 'S193', 'S195', 'S196', 'S043', 'S072', 'S078', 'S191', 'S129', 'S141', 'S190', 'S194']:
#raw.add_proj(blink_projs) # adding all projections
#raw.del_proj()
#raw.plot_projs_topomap()
raw.add_proj([blink_projs[0], blink_projs[2]])
elif subj in ['S135', 'S192', 'S197', 'S199', 'S216', 'S218']:
raw.add_proj([blink_projs[0], blink_projs[1]])
elif subj in ['S084', 'S075']:
raw.add_proj([blink_projs[0]])
return raw, eves2
'MEG2033', 'MEG0442', 'MEG2343', 'MEG1643', 'MEG1211',
'MEG2522', 'MEG0731'
]
if eeg:
# raw.info['bads'] += ['EEG004', 'EEG038', 'EEG040', 'EEG067']
raw.info['bads'] += ['EEG009', 'EEG027', 'EEG067']
# Filter the data for SSRs
raw.filter(l_freq=l_freq,
h_freq=h_freq,
l_trans_bandwidth=0.15,
picks=np.arange(0, 306, 1))
if not SSSR:
# raw.resample(sfreq=1000.0, n_jobs=4, verbose='DEBUG')
# SSP for blinks
blinks = find_blinks(raw, ch_name='EOG062')
epochs_blinks = mne.Epochs(raw,
blinks,
998,
tmin=-0.25,
tmax=0.25,
proj=True,
baseline=(-0.25, 0),
reject=dict(grad=8000e-13, mag=8e-12))
blink_projs = compute_proj_epochs(epochs_blinks,
n_grad=2,
n_mag=2,
n_eeg=0,
verbose='DEBUG')
raw.add_proj(blink_projs)
#%% Load data
# with open(os.path.join(data_loc, Subject+'_DynBin.pickle'),'rb') as f:
# IAC_epochs, ITD_epochs = pickle.load(f)
IAC_eeg, IAC_evnt = EEGconcatenateFolder(direct_IAC + Subject + '/', nchans,
refchans, exclude)
ITD_eeg, ITD_evnt = EEGconcatenateFolder(direct_ITD + Subject + '/', nchans,
refchans, exclude)
IAC_eeg.filter(1, 1000)
ITD_eeg.filter(1, 1000)
#%% Blink removal
blinks_IAC = find_blinks(IAC_eeg,
ch_name=['A1'],
thresh=100e-6,
l_trans_bandwidth=0.5,
l_freq=1.0)
blinks_ITD = find_blinks(ITD_eeg,
ch_name=['A1'],
thresh=100e-6,
l_trans_bandwidth=0.5,
l_freq=1.0)
scalings = dict(eeg=40e-6)
blinkIAC_epochs = mne.Epochs(IAC_eeg,
blinks_IAC,
998,
tmin=-0.25,
tmax=0.25,
proj=False,
fifs[k] = fpath + fifname
# Load data and read event channel
raw = mne.io.Raw(fifs, preload=True)
eves = mne.find_events(raw, stim_channel='STI101', shortest_event=1)
# Filter the data for ERPs
raw.filter(l_freq=0.5,
h_freq=144.0,
l_trans_bandwidth=0.15,
picks=np.arange(0, 306, 1))
# raw.apply_proj()
fs = raw.info['sfreq']
# SSP for blinks
blinks = find_blinks(raw, ch_name='EOG062')
blinkname = fpath + subj + '_' + para + '_blinks' + ssstag + '-eve.fif'
mne.write_events(blinkname, blinks)
epochs_blinks = mne.Epochs(raw,
blinks,
998,
tmin=-0.25,
tmax=0.25,
proj=True,
baseline=(-0.25, 0),
reject=dict(grad=8000e-13, mag=8e-12))
blink_projs = compute_proj_epochs(epochs_blinks,
n_grad=2,
n_mag=2,
n_eeg=0,
verbose='DEBUG')
print 'Viola! Data files found!'
edfname = bdfs[0]
# Load data and read event channel
(raw, eves) = bs.importbdf(fpath + edfname)
# Filter the data for ERPs
raw.filter(l_freq=0.5,
h_freq=200,
l_trans_bandwidth=0.15,
picks=np.arange(0, 32, 1))
#raw.apply_function(detrend, picks=np.arange(0, 32, 1), dtype=None,
# n_jobs=1, verbose=True)
# SSP for blinks
blinks = find_blinks(raw)
epochs_blinks = mne.Epochs(raw,
blinks,
998,
tmin=-0.25,
tmax=0.25,
proj=True,
baseline=(-0.25, 0),
reject=dict(eeg=500e-6))
blink_projs = compute_proj_epochs(epochs_blinks,
n_grad=0,
n_mag=0,
n_eeg=2,
verbose='DEBUG')
raw.add_proj(blink_projs)
#don't need these extra external channels that are saved
for s in range(0, len(Subjects)):
Subject = Subjects[s]
print('\n\n\n\n' + Subject + '\n\n\n\n')
IAC_eeg, IAC_evnt = EEGconcatenateFolder(direct_IAC + Subject + '/',
nchans, refchans, exclude)
ITD_eeg, ITD_evnt = EEGconcatenateFolder(direct_ITD + Subject + '/',
nchans, refchans, exclude)
IAC_eeg.filter(1, 40)
ITD_eeg.filter(1, 40)
## blink removal
blinks_IAC = find_blinks(IAC_eeg,
ch_name=['A1'],
l_freq=1,
l_trans_bandwidth=0.5)
blinks_ITD = find_blinks(ITD_eeg,
ch_name=['A1'],
l_freq=1,
l_trans_bandwidth=0.5)
scalings = dict(eeg=40e-6)
blinkIAC_epochs = mne.Epochs(IAC_eeg,
blinks_IAC,
998,
tmin=-0.25,
tmax=0.25,
proj=False,
baseline=(-0.25, 0),
'MEG1013', 'MEG1623', 'MEG2342', 'MEG2513', 'MEG2542'
]
# Filter the data for ERPs
raw.filter(l_freq=0.5,
h_freq=144,
l_trans_bandwidth=0.15,
picks=np.arange(0, 306, 1))
# raw.apply_proj()
fs = raw.info['sfreq']
removeblinks = True
if removeblinks:
# SSP for blinks
blinks = find_blinks(raw, ch_name=[
'EOG062',
])
blinkname = (fpath + subj + '_' + para + '_blinks_erp' + ssstag +
'-eve.fif')
mne.write_events(blinkname, blinks)
epochs_blinks = mne.Epochs(raw,
blinks,
998,
tmin=-0.25,
tmax=0.25,
proj=True,
baseline=(-0.25, 0),
reject=dict(grad=8000e-13, mag=8e-12))
blink_projs = compute_proj_epochs(epochs_blinks,
n_grad=2,
n_mag=2,
data_loc = '/media/ravinderjit/Data_Drive/Data/EEGdata/Sentences/'
subject = 'S211'
refchans = ['EXG1', 'EXG2']
nchans = 34
exclude = ['EXG3', 'EXG4', 'EXG5', 'EXG6', 'EXG7', 'EXG8']
sentEEG, sent_evnt = EEGconcatenateFolder(data_loc + subject + '/', nchans,
refchans, exclude)
sentEEG.filter(1, 75)
blinks = find_blinks(sentEEG,
ch_name=['A1'],
thresh=100e-6,
l_trans_bandwidth=0.5,
l_freq=1.0)
blink_epochs = mne.Epochs(sentEEG,
blinks,
998,
tmin=-0.25,
tmax=0.25,
proj=False,
baseline=(-0.25, 0),
reject=dict(eeg=500e-6))
_Projs = compute_proj_epochs(blink_epochs,
n_grad=0,
n_mag=0,
epochs = mne.read_epochs(fpath + save_raw_name, verbose='DEBUG')
Fs = epochs.info['sfreq']
x = epochs.get_data()
times = epochs.times
else:
preEpoched = False
fifs = fnmatch.filter(os.listdir(fpath), subj + '_CABR_raw.fif')
print 'No pre-epoched data found, looking for raw files'
print 'Viola!', len(fifs), 'files found!'
for k, fif in enumerate(fifs):
fifs[k] = fpath + fif
# Load data and read event channel
raw = mne.io.Raw(fifs, preload=True, add_eeg_ref=False)
raw.set_channel_types({'EMG061': 'eeg'})
raw.info['bads'] += ['MEG0223', 'MEG1623']
# Filter the data for SSRs
raw.filter(l_freq=70., h_freq=None, picks=None)
eves = find_blinks(raw, event_id=1, thresh=0.1, l_freq=70.,
h_freq=2499., ch_name=['MISC001'])
cond = 1
# Epoching events of type
epochs = mne.Epochs(raw, eves, cond, tmin=-0.005, proj=False,
tmax=0.050, baseline=(-0.005, 0.005),
reject=dict(grad=1000e-12, mag=10e-12),
verbose='WARNING')
abr = epochs.average()
avename = subj + '_ABR-ave.fif'
abr.save(fpath + avename)
refchans=None, exclude=exclude)
rawlist += [rawtemp, ]
evelist += [evestemp, ]
raw, eves = mne.concatenate_raws(rawlist, events_list=evelist)
# raw.set_channel_types({'EXG3': 'eeg', 'EXG4': 'eeg'})
raw.info['bads'] += []
# Filter the data
raw.filter(l_freq=1.5, h_freq=40.)
removeblinks = True
if removeblinks:
# SSP for blinks
blinks = find_blinks(raw, ch_name=['A1', ], l_trans_bandwidth=0.4)
blinkname = (fpath + subj + '_inh_blinks_erp' + '-eve.fif')
mne.write_events(blinkname, blinks)
epochs_blinks = mne.Epochs(raw, blinks, 998, tmin=-0.25,
tmax=0.25, proj=True,
baseline=(-0.25, 0),
reject=dict(eeg=500e-6))
blink_projs = compute_proj_epochs(epochs_blinks, n_grad=0,
n_mag=0, n_eeg=2,
verbose='DEBUG')
raw.add_proj(blink_projs)
for c, cond in enumerate(condlist):
condname = condnames[c]
# Epoching events of type
eventlistRS = [eventsRS_1, eventsRS_2, eventsRS_3]
rawHB, evesHB = mne.concatenate_raws(rawlistHB, events_list=eventlistHB)
rawHB.filter(1.0, 100) #filter the eeg
rawRS, evesRS = mne.concatenate_raws(rawlistRS, events_list=eventlistRS)
rawRS.filter(1.0, 100)
rawHB._data[36, :] = rawHB._data[34, :] - rawHB._data[
35, :] #replacing an empty channel with the difference of the two eog channels to use for blink detection. This is gonna be channel EXG5
rawRS._data[36, :] = rawRS._data[34, :] - rawRS._data[
35, :] #replacing an empty channel with the difference of the two eog channels to use for blink detection. This is gonna be channel EXG5
from anlffr.preproc import find_blinks
blinksHB = find_blinks(rawHB, ch_name=['EXG5'], thresh=100e-6)
blinksRS = find_blinks(rawRS, ch_name=['EXG5'], thresh=100e-6)
scalings = dict(mag=1e-12,
grad=4e-11,
eeg=20e-6,
eog=150e-6,
ecg=5e-4,
emg=1e-3,
ref_meg=1e-12,
misc=1e-3,
stim=1,
resp=1,
chpi=1e-4)
scalings['eeg'] = 40e-6
scalings['misc'] = 150e-6
#raw.plot(events =blinks, scalings = scalings, proj = False)
# raw.info['bads'] += []
# Filter the data for ERPs
raw.filter(l_freq=0.5,
h_freq=100,
l_trans_bandwidth=0.15,
picks=np.arange(0, 32, 1))
# raw.apply_proj()
fs = raw.info['sfreq']
removeblinks = True
if removeblinks:
# SSP for blinks
blinks = find_blinks(raw, ch_name=[
'A1',
])
blinkname = (fpath + subj + '_' + para + '_blinks_erp' + '-eve.fif')
mne.write_events(blinkname, blinks)
epochs_blinks = mne.Epochs(raw,
blinks,
998,
tmin=-0.25,
tmax=0.25,
proj=True,
baseline=(-0.25, 0),
reject=dict(eeg=500e-6))
blink_projs = compute_proj_epochs(epochs_blinks,
n_grad=0,
n_mag=0,
n_eeg=2,
'/media/ravinderjit/Data_Drive/EEGdata/Summer17/SubjRS_Binding+001.bdf')
rawlist = [raw1, raw2]
eventlist = [events1, events2]
raw, eves = mne.concatenate_raws(rawlist, events_list=eventlist)
raw.filter(1.0, 150) #filter the eeg
raw._data[36, :] = raw._data[34, :] - raw._data[
35, :] #replacing an empty channel with the difference of the two eog channels to use for blink detection. This is gonna be channel EXG5
from anlffr.preproc import find_blinks
#The below line was blinks = find_blinks(raw, ch_name = ['EXG5'], thresh = 100e-6) ... changed to below on 06/12/18
blinks = find_blinks(raw,
ch_name=['EXG5'],
thresh=100e-6,
l_trans_bandwidth=0.5,
l_freq=1.0)
scalings = dict(mag=1e-12,
grad=4e-11,
eeg=20e-6,
eog=150e-6,
ecg=5e-4,
emg=1e-3,
ref_meg=1e-12,
misc=1e-3,
stim=1,
resp=1,
chpi=1e-4)
scalings['eeg'] = 40e-6
scalings['misc'] = 150e-6
raw.info['bads'] += [
'MEG1013', 'MEG1623', 'MEG2342', 'MEG2513', 'MEG2542', 'MEG1031',
'MEG1041', 'MEG2022'
]
# Filter the data for ERPs
raw.filter(l_freq=1.0, h_freq=90, picks=np.arange(0, 306, 1))
# raw.apply_proj()
fs = raw.info['sfreq']
removeblinks = True
if removeblinks:
# SSP for blinks
blinks = find_blinks(raw,
ch_name=[
'EOG062',
],
l_trans_bandwidth='auto')
blinkname = (fpath + subj + '_' + para + '_blinks_erp' + ssstag +
'-eve.fif')
mne.write_events(blinkname, blinks)
epochs_blinks = mne.Epochs(raw,
blinks,
998,
tmin=-0.25,
tmax=0.25,
proj=True,
baseline=(-0.25, 0),
reject=dict(grad=8000e-13, mag=8e-12))
blink_projs = compute_proj_epochs(epochs_blinks,
n_grad=2,
def processing(stimulus, subj, fpath):
# load data and read event channels
rawlist = []
evelist = []
# extracting bdf filenames into bdfs
if stimulus == 'ITD':
bdfs = fnmatch.filter(os.listdir(fpath), subj + '_ITD*.bdf')
elif stimulus == 'Atten':
bdfs = fnmatch.filter(os.listdir(fpath), subj + '_Atten*.bdf')
if len(bdfs) >= 1:
for k, bdf in enumerate(bdfs):
rawtemp, evestemp = bs.importbdf(fpath + bdf)
rawlist += [rawtemp,]
evelist += [evestemp,]
else:
RuntimeError("No bdf files found!")
# the returned eves has three columns: sample number, value of onset, value of offset.
raw, eves = mne.concatenate_raws(rawlist, events_list = evelist)
# Things to try if data are too noisy
# raw.info['bads'] += ['A7', 'A6', 'A24'] # if A7, A6, A24 have been rejected a lot, they can be added manually
# raw.set_eeg_reference(ref_channels=['Average']) # referencing to the average of all channels, do this if channels are too noisy
eves2 = eves.copy()
# the eves are in 16 bits originally, so this operation only looks at the lower 8 bits. higher 8 bits are always
# high, representing button box condition. If interested in what's happening at button press, do np.floor(eves2[:,1]/256)
eves2[:, 1] = np.mod(eves2[:, 1], 256)
eves2[:, 2] = np.mod(eves2[:, 2], 256)
if stimulus == 'ITD' or stimulus == 'Atten':
raw.filter(l_freq = 1, h_freq=50) # if channels are noisy, adjust the filter parameters
# raw.plot(events=eves2)
# SSP for blinks
blinks = find_blinks(raw, ch_name = ['A1',], l_trans_bandwidth=0.5,
l_freq=1.0) # A1 is the closest electrode to eyebrow
# blink and eves2 triggers can be combined using np.concatenate((eves2, blinks), axis = 0)
# raw.plot(events=blinks) shows the lines at eye blinks
# the trigger for blinks can be chosen to be starting from 1000, just to make sure it doesn't collide with the triggers for conditions
epochs_blinks = mne.Epochs(raw, blinks, 998, tmin = -0.25, tmax = 0.25,
proj = False, baseline = (-0.25, 0),
reject=dict(eeg=500e-6))
# evoked_blinks = epochs_blinks.average()
# evoked_blinks_data = evoked_blinks.data[np.arange(32),:]
# PCA is only applied to the epochs around eye blinks. Since the eye blinks are
# contributing the most to the variance within this chunk of window,
# the first PCA (first eigenvector) is going to be due to the eye blink
# for sure and removed. If the PCA was performed on the whole samples, we wouldn't
# know which vector is going to be linked to the eye blink
# for "n_eeg", it's recommended to remove only the biggest projection, which is eye blinks in this case
# greater n_eeg removes more nueral data, which is not favorable
n_eeg = 4
blink_projs = compute_proj_epochs(epochs_blinks, n_grad=0,
n_mag=0, n_eeg=n_eeg,
verbose='DEBUG')
projN = np.zeros(2)
if subj in ['S025', 'S031', 'S046', 'S117', 'S123', 'S127', 'S128', 'S132', 'S133', 'S143', 'S149', 'S051', 'S183', 'S185', 'S187', 'S189', 'S193', 'S195', 'S196', 'S043', 'S072', 'S075', 'S078', 'S191', 'S129', 'S141', 'S190', 'S021', 'S024', 'S028', 'S083', 'S119', 'S121', 'S122', 'S139', 'S140', 'S142', 'S144', 'S145']:
#raw.add_proj(blink_projs) # adding all projections
raw.add_proj([blink_projs[0], blink_projs[2]]) # raw.del_proj()
projN = [1, 3]
elif subj in ['S135', 'S192', 'S194', 'S197', 'S199', 'S216', 'S218']:
raw.add_proj([blink_projs[0], blink_projs[1]]) # raw.del_proj()
projN = [1, 2]
elif subj in ['S084']:
raw.add_proj([blink_projs[0]]) # raw.del_proj()
projN = [1, 0]
# raw.plot_projs_topomap() shows the 4 max PCAs (eigenvectors)
# raw.plot(events = blinks, show_options = True) could show the options for applying different projections
# if channels are too noisy, play with n_eeg, if the second variance is acting more than
# the first, that means the channels are contaminated not just by the eye blinks, but also
# from other sources
# MANUALLY SELECT PROJECTIONS BY PLOTTING raw.plot_projs_topomap
# REMOVE EXTRA PROJS USING raw.del_proj -- Remember index starts at 0
return raw, eves2, projN
#%% Load data and filter
datapath = os.path.join(data_loc, subject)
data_eeg, data_evnt = EEGconcatenateFolder(datapath + '/', nchans,
refchans, exclude)
data_eeg.filter(l_freq=1, h_freq=500)
#%% Remove bad channels
if subject == 'S207':
data_eeg.info['bads'].append('A15')
#%% Blink Removal
blinks = find_blinks(data_eeg,
ch_name=['A1'],
thresh=100e-6,
l_trans_bandwidth=0.5,
l_freq=1.0)
blink_epochs = mne.Epochs(data_eeg,
blinks,
998,
tmin=-0.25,
tmax=0.25,
proj=False,
baseline=(-0.25, 0),
reject=dict(eeg=500e-6))
Projs = compute_proj_epochs(blink_epochs,
n_grad=0,
n_mag=0,
n_eeg=8,
verbose='DEBUG')
#if Mask is none: events 1:4 = 65281-84, button presses are 65280
#rawlist = [raw1, raw2]
#eventlist = [events1, events2]
rawlist = [raw1, raw2, raw3]
eventlist = [events1, events2, events3]
raw, eves = mne.concatenate_raws(rawlist, events_list=eventlist)
raw.filter(1.0, 40) #filter the eeg
raw._data[36, :] = raw._data[34, :] - raw._data[
35, :] #replacing an empty channel with the difference of the two eog channels to use for blink detection. This is gonna be channel EXG5
from anlffr.preproc import find_blinks
blinks = find_blinks(raw, ch_name=['EXG5'], thresh=100e-6)
scalings = dict(mag=1e-12,
grad=4e-11,
eeg=20e-6,
eog=150e-6,
ecg=5e-4,
emg=1e-3,
ref_meg=1e-12,
misc=1e-3,
stim=1,
resp=1,
chpi=1e-4)
scalings['eeg'] = 40e-6
scalings['misc'] = 150e-6
#raw.plot(events =blinks, scalings = scalings, proj = False)