def preprocess(data, events, matlab=False, spectral=True):
'''
preprocessing pipeline for EEG data from the
buffer
Parameters
----------
data : a list of datapoints (numpy arrays)
events : a list of fieldtrip events or a numpy array
spectral: Whether you want to compute a spectrogram (default False)
matlab: Whether the data comes from a .mat file (default False)
Examples
--------
>>> data, events = ftc.getData(0,100)
>>> data, events = preprocess(data, events)
>>> data, events = ftc.getData(0,100)
>>> data, events = preprocess(data, events, spectral=True)
'''
if matlab:
data, events = reformat(data, events)
data = preproc.detrend(data, dim=0)
data, badch = preproc.badchannelremoval(data)
data = preproc.spatialfilter(data, type='whiten')
if spectral:
data = preproc.spectralfilter(data, (1, 40), fSample)
data, events, badtrials = preproc.badtrailremoval(data, events)
return data, events
plotTrials(X) # single trial
fig = plt.figure()
plotERP(X, Y) # class averages
# visualize the raw data
# plot a subset of single-trials
plotTrials(X, range(3))
# plot the grand average's per condition
plotERP(X, Y)
#-------------------------------------------------------------------
# Run the standard pre-processing and analysis pipeline
# 1: detrend
X = preproc.detrend(X)
updatePlots()
# 2: bad-channel removal, channels in dim=0
goodch, badch = preproc.outlierdetection(X)
X = X[goodch, :, :]
Cnames = Cnames[goodch]
Cpos = Cpos[:, goodch]
updatePlots()
# 3: apply spatial filter
X = preproc.spatialfilter(X, type='car')
updatePlots()
# 4 & 5: map to frequencies and select frequencies of interest
X = preproc.fftfilter(X, 1, [8, 10, 28, 30], fs)
dname = 'training_data'
cname = 'clsfr'
print("Training classifier")
f = pickle.load(open(dname + '.pk', 'rb'))
data = f['data']
events = f['events']
hdr = f['hdr']
# -------------------------------------------------------------------
# Run the standard pre-processing and analysis pipeline
# 1: detrend
data = [d[:, :10] for d in data]
data = preproc.detrend(data)
# 2: bad-channel removal
data, bad_channels = preproc.badchannelremoval(data)
print(f'Removed channels: {bad_channels}')
# 3: apply spatial filter
data = preproc.spatialfilter(data, type='car')
# 4 & 5: map to frequencies and select frequencies of interest
data = preproc.spectralfilter(data, (5, 6, 31, 32), hdr.fSample)
# 6 : bad-trial removal
data, events, bad_trials = preproc.badtrialremoval(data, events)
print(f'Removed trials: {bad_trials}')
with open('processed_data.pkl', 'wb') as f:
pickle.dump({'X': data, 'events': events, 'hdr': hdr}, f)
# Reduce the number of features per channel
data = np.array(data) # data contains the signal sampled, 750 samples for 32 channels
data_rec = np.copy(data)
if verbose: print(events_im[0].value) # Information in terminal, verbose option, for debugging purposes
if verbose: print(data.shape) # Information in terminal, verbose option, for debugging purposes
data = data[:,:,:]
if verbose: print(data.shape) # Information in terminal, verbose option, for debugging purposes
data = np.array(data) #data support variable
data = np.transpose(data)
data = preproc.detrend(data); # Preprocessing operations, more detailed in the report
data = preproc.spatialfilter(data, type = 'car') # "
data, freqs = preproc.powerspectrum(data,dim = 1,fSample= 250); # " sample information included in the hdr, must be changed if it changes for it to work
data,freqIdx = preproc.selectbands(data,dim=1,band=[6,13,15,32],bins=freqs); # " the bands selected here also should be the same as the chosen one in the classifier file
if verbose: print(data.shape) # Information in terminal, verbose option, for debugging purposes
data = np.reshape(data, (1,-1)) # Data now, is an bunch of array containing the power spectrum of every channel in the selected frequencies (#ch, #freq), in this line all those arrays become one of size (1, #ch * #freq)
prediction_right = classifier_right.predict(data) # The freq informations is now passed to the classifier, each one yielding a float value
prediction_left = classifier_left.predict(data)
pred = [prediction_left, prediction_right] # The predictions are put into an array, for debugging reasons, this isn't what will be passed
prediction = [np.round(prediction_left).astype(int), np.round(prediction_right).astype(int)] # In this array the predictions are rounded, as the expected values are 0,1 for each classifier but the outputs are float, they need to be rounded
# 0: get class labels from events values
y = [e.value[0] for e in events]
# convert to numeric labels
valuedict={} # dict to convert from event.values to numbers
#y = np.array(y) # N.B. Only works with *NUMERIC* event values...
# get the unique values in y
valuedict = set(y)
# convert to dictionary
valuedict = { val:i for i,val in enumerate(valuedict) }
# use the dict to map from values to numbers
y = np.array([ valuedict[val] for val in y ])
# 1: detrend
data = preproc.detrend(data)
# 2: bad-channel removal
goodch, badch = preproc.outlierdetection(data);
data = data[goodch,:,:];
# 3: apply spatial filter
spatialfilter='car'
data = preproc.spatialfilter(data,type=spatialfilter)
# 4: map to frequencies
data,freqs = preproc.powerspectrum(data,dim=1,fSample=fs)
# 5 : select the frequency bins we want
freqbands =[8,10,28,30]
data,freqIdx=preproc.selectbands(data,dim=1,band=freqbands,bins=freqs)
if e.value == "calibrate":
print("Calibration phase")
data, events, stopevents = bufhelp.gatherdata(
"stimulus.hybrid",
trlen_ms, ("stimulus.training", "end"),
milliseconds=True)
pickle.dump({
"events": events,
"data": data
}, open("subject_data", "wb"))
elif e.value == "training":
print("Training classifier")
data = preproc.detrend(data)
data, badch = preproc.badchannelremoval(data)
data = preproc.spatialfilter(data)
data = preproc.spectralfilter(data, (1, 10, 15, 25),
bufhelp.fSample)
data, events, badtrials = preproc.badtrailremoval(data, events)
mapping = {
('stimulus.hybrid', 'left'): -1,
('stimulus.hybrid', 'right'): 1
}
classifier = linear.fit(data, events, mapping)
bufhelp.update()
bufhelp.sendEvent("sigproc.training", "done")
elif e.value == "contfeedback":
print("Feedback phase")
clf = pickle.load(f)
ftc, hdr = bufhelp.connect()
trial_length_ms = 600
trial_length_samples = math.ceil((trial_length_ms / 1000.) * hdr.fSample)
# gather stores events that haven't been long enough ago to collect data
gather = []
print('Waiting for events...')
while True:
events = bufhelp.buffer_newevents('stimulus.type', timeout_ms=1000)
for evt in events:
end_sample = evt.sample + trial_length_samples - 1
gather.append((evt, end_sample))
# Current sample count
nSamples = bufhelp.globalstate[0]
for point in gather:
evt, end_sample = point
if end_sample < nSamples:
# Enough time has passed to collect the data
data = ftc.getData((evt.sample, end_sample))
# Preprocess
data = preproc.detrend([data])
data = preproc.spectralfilter(data, (0, 1, 14, 15),
hdr.fSample)
gather.remove(point)
# Classify
pred = clf.predict(np.array(data).reshape(1, -1))[0]
send_event('classifier.prediction', pred, evt.sample)
for element in original:
if hasattr(element,"deepcopy"):
clone.append(element.deepcopy())
elif hasattr(element,"copy"):
clone.append(element.copy())
else:
clone.append(element)
return clone
if __name__=="main":
#%pdb
import preproc
import numpy
fs = 40
data = numpy.random.randn(3,fs,5) # 1s of data
ppdata=preproc.detrend(data,0)
ppdata=preproc.spatialfilter(data,'car')
ppdata=preproc.spatialfilter(data,'whiten')
ppdata,goodch=preproc.badchannelremoval(data)
ppdata,goodtr=preproc.badtrialremoval(data)
goodtr,badtr=preproc.outlierdetection(data,-1)
Fdata,freqs=preproc.fouriertransform(data, 2, fSamples=1, posFreq=True)
Fdata,freqs=preproc.powerspectrum(data, 1, dim=2)
filt=preproc.mkFilter(10,[0,4,6,7],1)
filt=preproc.mkFilter(numpy.abs(numpy.arange(-10,10,1)),[0,4,6,7])
ppdata=preproc.fftfilter(data,1,[0,3,4,6],fs)
ppdata,keep=preproc.selectbands(data,[4,5,10,15],1); ppdata.shape
from image3d import *
# init connection to the buffer
ftc, hdr = bufhelp.connect()
#define variables
trlen_samp = 50
nSymbols = 2
nr_channels = 4 # in debug mode
erp = np.zeros((nr_channels, trlen_samp, nSymbols))
nTarget = np.zeros((nSymbols, 1))
# read in the capfile
Cname, latlong, xy, xyz, capfile = readCapInf.readCapInf('sigproxy_with_TRG')
fig = plt.figure()
# grab data after every t:'stimulus' event until we get a {t:'stimulus.training' v:'end'} event
data, events, stopevents = bufhelp.gatherdata(["stimulus", "experiment"],
trlen_samp, ("sequence", "end"),
milliseconds=False)
# loop through all recorded events (last to first)
for ei in np.arange(len(events) - 1, -1, -1):
# detrend erp, so we can see stuff
erp = preproc.detrend(erp)
image3d(erp) # plot the ERPs
plt.show()
# init connection to the buffer
ftc,hdr=bufhelp.connect();
#define variables
trlen_samp = 50
nSymbols = 2
nr_channels = 4 # in debug mode
erp = np.zeros((nr_channels,trlen_samp,nSymbols))
nTarget = np.zeros((nSymbols,1))
# read in the capfile
Cname,latlong,xy,xyz,capfile= readCapInf.readCapInf('sigproxy_with_TRG')
fig=plt.figure()
# grab data after every t:'stimulus' event until we get a {t:'stimulus.training' v:'end'} event
data, events, stopevents = bufhelp.gatherdata(["stimulus","experiment"],trlen_samp,("sequence","end"), milliseconds=False)
# loop through all recorded events (last to first)
for ei in np.arange(len(events)-1,-1,-1):
# detrend erp, so we can see stuff
erp = preproc.detrend(erp)
image3d(erp) # plot the ERPs
plt.show()
fig=plt.figure()
plotERP(X,Y) # class averages
# plot a subset of single-trials
plotTrials(X,range(3));
# plot the grand average's per condition
plotERP(X,Y);
#-------------------------------------------------------------------
# Run the standard pre-processing and analysis pipeline
# 1: detrend
X = preproc.detrend(X)
updatePlots();
# 2: bad-channel removal
goodch, badch = preproc.outlierdetection(X);
X = X[goodch,:,:];
Cnames=Cnames[goodch];
Cpos=Cpos[:,goodch];
updatePlots()
# 3: apply spatial filter
X = preproc.spatialfilter(X,type='car')
updatePlots();
