def preprocess(data, events, matlab=False, spectral=True):
'''
preprocessing pipeline for EEG data from the
buffer
Parameters
----------
data : a list of datapoints (numpy arrays)
events : a list of fieldtrip events or a numpy array
spectral: Whether you want to compute a spectrogram (default False)
matlab: Whether the data comes from a .mat file (default False)
Examples
--------
>>> data, events = ftc.getData(0,100)
>>> data, events = preprocess(data, events)
>>> data, events = ftc.getData(0,100)
>>> data, events = preprocess(data, events, spectral=True)
'''
if matlab:
data, events = reformat(data, events)
data = preproc.detrend(data, dim=0)
data, badch = preproc.badchannelremoval(data)
data = preproc.spatialfilter(data, type='whiten')
if spectral:
data = preproc.spectralfilter(data, (1, 40), fSample)
data, events, badtrials = preproc.badtrailremoval(data, events)
return data, events
#------------------------------------------------------------------- # Run the standard pre-processing and analysis pipeline # 1: detrend X = preproc.detrend(X) updatePlots() # 2: bad-channel removal, channels in dim=0 goodch, badch = preproc.outlierdetection(X) X = X[goodch, :, :] Cnames = Cnames[goodch] Cpos = Cpos[:, goodch] updatePlots() # 3: apply spatial filter X = preproc.spatialfilter(X, type='car') updatePlots() # 4 & 5: map to frequencies and select frequencies of interest X = preproc.fftfilter(X, 1, [8, 10, 28, 30], fs) updatePlots() # 6 : bad-trial removal, trials in dim=2 goodtr, badtr = preproc.outlierdetection(X, dim=2) X = X[:, :, goodtr] Y = Y[goodtr] updatePlots() # 7: train linear least squares classifier, with cross-validation from sklearn.linear_model import LinearRegression clsfr = sklearn.linear_model.RidgeCV()
f = pickle.load(open(dname + '.pk', 'rb'))
data = f['data']
events = f['events']
hdr = f['hdr']
# -------------------------------------------------------------------
# Run the standard pre-processing and analysis pipeline
# 1: detrend
data = [d[:, :10] for d in data]
data = preproc.detrend(data)
# 2: bad-channel removal
data, bad_channels = preproc.badchannelremoval(data)
print(f'Removed channels: {bad_channels}')
# 3: apply spatial filter
data = preproc.spatialfilter(data, type='car')
# 4 & 5: map to frequencies and select frequencies of interest
data = preproc.spectralfilter(data, (5, 6, 31, 32), hdr.fSample)
# 6 : bad-trial removal
data, events, bad_trials = preproc.badtrialremoval(data, events)
print(f'Removed trials: {bad_trials}')
with open('processed_data.pkl', 'wb') as f:
pickle.dump({'X': data, 'events': events, 'hdr': hdr}, f)
# Reduce the number of features per channel
freqs = np.linspace(0, hdr.fSample/2, len(data[0]))
data = [d[(5 <= freqs) & (freqs <= 32)] for d in data]
data = [util.subsample_frequencies(d, 4) for d in data]
# 7: train classifier
# convert to dictionary
valuedict = { val:i for i,val in enumerate(valuedict) }
# use the dict to map from values to numbers
y = np.array([ valuedict[val] for val in y ])
# 1: detrend
data = preproc.detrend(data)
# 2: bad-channel removal
goodch, badch = preproc.outlierdetection(data);
data = data[goodch,:,:];
# 3: apply spatial filter
spatialfilter='car'
data = preproc.spatialfilter(data,type=spatialfilter)
# 4: map to frequencies
data,freqs = preproc.powerspectrum(data,dim=1,fSample=fs)
# 5 : select the frequency bins we want
freqbands =[8,10,28,30]
data,freqIdx=preproc.selectbands(data,dim=1,band=freqbands,bins=freqs)
# 6 : bad-trial removal
goodtr, badtr = preproc.outlierdetection(data,dim=2)
data = data[:,:,goodtr]
y = y[goodtr]
# 7: train classifier, default is a linear-least-squares-classifier
clsfr = sklearn.linear_model.LogisticRegressionCV()
print ("Waiting for startPhase.cmd event.")
while run:
e = bufhelp.waitforevent("startPhase.cmd",1000, True)
print("Got startPhase event: %s"%e)
if e is not None:
if e.value == "calibration":
print("Calibration phase")
data, events, stopevents = bufhelp.gatherdata("stimulus.tgtFlash",trlen_ms,("stimulus.training","end"), milliseconds=True)
pickle.dump({"events":events,"data":data}, open("subject_data", "w"))
elif e.value == "train":
print("Training classifier")
data = preproc.detrend(data)
data, badch = preproc.badchannelremoval(data)
data = preproc.spatialfilter(data)
data = preproc.spectralfilter(data, (0, .1, 10, 12), bufhelp.fSample)
data, events, badtrials = preproc.badtrailremoval(data, events)
mapping = {('stimulus.tgtFlash', '0'): 0, ('stimulus.tgtFlash', '1'): 1}
linear.fit(data,events,mapping)
bufhelp.update()
bufhelp.sendevent("sigproc.training","done")
elif e.value =="testing":
print("Feedback phase")
while True:
data, events, stopevents = bufhelp.gatherdata(["stimulus.columnFlash","stimulus.rowFlash"],trlen_ms,[("stimulus.feedback","end"), ("stimulus.sequence","end")], milliseconds=True)
if isinstance(stopevents, list):
if any(["stimulus.feedback" in x.type for x in stopevents]):
break
if e.value == "calibration":
print("Calibration phase")
data, events, stopevents = bufhelp.gatherdata(
"stimulus.tgtFlash",
trlen_ms, ("stimulus.training", "end"),
milliseconds=True)
pickle.dump({
"events": events,
"data": data
}, open("subject_data", "w"))
elif e.value == "train":
print("Training classifier")
data = preproc.detrend(data)
data, badch = preproc.badchannelremoval(data)
data = preproc.spatialfilter(data)
data = preproc.spectralfilter(data, (0, .1, 10, 12),
bufhelp.fSample)
data, events, badtrials = preproc.badtrailremoval(data, events)
mapping = {
('stimulus.tgtFlash', '0'): 0,
('stimulus.tgtFlash', '1'): 1
}
linear.fit(data, events, mapping)
bufhelp.update()
bufhelp.sendevent("sigproc.training", "done")
elif e.value == "testing":
print("Feedback phase")
while True:
data, events, stopevents = bufhelp.gatherdata(
event_types = [e.type[0] for e in events]
event_letters = [e.value[0] for e in events]
# stop processing if needed
if "stimulus.feedback" in event_types:
break
# get data in correct format
data = np.transpose(data)
# 1: detrend
data = preproc.detrend(data)
# 2: bad-channel removal (as identifed in classifier training)
data = data[goodch, :, :]
# 3: apply spatial filter (as in classifier training)
data = preproc.spatialfilter(data, type=spatialfilter)
# 4: map to frequencies (TODO: check fs matches!!)
data, freqs = preproc.powerspectrum(data, dim=1, fSample=fs)
# 5: select frequency bins we want
data, freqIdx = preproc.selectbands(data,
dim=1,
band=freqbands,
bins=freqs)
print(data.shape)
# 6: apply the classifier, get raw predictions
X2d = np.reshape(
data,
(-1,
data.shape[2])).T # sklearn needs data to be [nTrials x nFeatures]
fraw = classifier.predict(X2d)
# 7: map from fraw to event values
data, events, stopevents = bufhelp.gatherdata(
"stimulus.hybrid",
trlen_ms, ("stimulus.training", "end"),
milliseconds=True)
pickle.dump({
"events": events,
"data": data
}, open("subject_data", "wb"))
elif e.value == "training":
print("Training classifier")
data = preproc.detrend(data)
data, badch = preproc.badchannelremoval(data)
data = preproc.spatialfilter(data)
data = preproc.spectralfilter(data, (1, 10, 15, 25),
bufhelp.fSample)
data, events, badtrials = preproc.badtrailremoval(data, events)
mapping = {
('stimulus.hybrid', 'left'): -1,
('stimulus.hybrid', 'right'): 1
}
classifier = linear.fit(data, events, mapping)
bufhelp.update()
bufhelp.sendEvent("sigproc.training", "done")
elif e.value == "contfeedback":
print("Feedback phase")
while True:
test_data, test_events, stopevents = bufhelp.gatherdata(
for element in original:
if hasattr(element,"deepcopy"):
clone.append(element.deepcopy())
elif hasattr(element,"copy"):
clone.append(element.copy())
else:
clone.append(element)
return clone
if __name__=="main":
#%pdb
import preproc
import numpy
fs = 40
data = numpy.random.randn(3,fs,5) # 1s of data
ppdata=preproc.detrend(data,0)
ppdata=preproc.spatialfilter(data,'car')
ppdata=preproc.spatialfilter(data,'whiten')
ppdata,goodch=preproc.badchannelremoval(data)
ppdata,goodtr=preproc.badtrialremoval(data)
goodtr,badtr=preproc.outlierdetection(data,-1)
Fdata,freqs=preproc.fouriertransform(data, 2, fSamples=1, posFreq=True)
Fdata,freqs=preproc.powerspectrum(data, 1, dim=2)
filt=preproc.mkFilter(10,[0,4,6,7],1)
filt=preproc.mkFilter(numpy.abs(numpy.arange(-10,10,1)),[0,4,6,7])
ppdata=preproc.fftfilter(data,1,[0,3,4,6],fs)
ppdata,keep=preproc.selectbands(data,[4,5,10,15],1); ppdata.shape
print(data[0]) X = np.array(data).T print(X.shape, labels.shape) X = preproc.detrend(X) # 2: bad-channel removal goodch, badch = preproc.outlierdetection(X) print(goodch) X = X[goodch, :, :] # 3: apply spatial filter X = preproc.spatialfilter(X, 'car') # 4 & 5: map to frequencies and select frequencies of interest print(hdr) X, freqs = preproc.powerspectrum(X, dim=1, fSample=hdr.fSample) print(freqs) freqbands = [20, 10, 30, 60] X, freqIdx = preproc.selectbands(X, dim=1, band=freqbands, bins=freqs) freqs = freqs[freqIdx] # 6 : bad-trial removal goodtr, badtr = preproc.outlierdetection(X, dim=2)
# Run the standard pre-processing and analysis pipeline # 1: detrend X = preproc.detrend(X) updatePlots(); # 2: bad-channel removal goodch, badch = preproc.outlierdetection(X); X = X[goodch,:,:]; Cnames=Cnames[goodch]; Cpos=Cpos[:,goodch]; updatePlots() # 3: apply spatial filter X = preproc.spatialfilter(X,type='car') updatePlots(); # Map to frequency domain, only keep the positive frequencies X,freqs = preproc.powerspectrum(X,dim=1,fSample=fs) yvals = freqs; # ensure the plots use the right x-ticks ylabel='freq (Hz)' updatePlots() # 5 : select the frequency bins we want X,freqIdx=preproc.selectbands(X,dim=1,band=[8,10,28,30],bins=freqs) freqs=freqs[freqIdx] yvals=freqs updatePlots()
