def process(filename):
data_dir = os.path.join("../Datasets/", filename)
data_path = os.path.join(data_dir, filename + '_cnt.txt')
label_path = os.path.join(data_dir, filename + '_mrk.txt')
data_df = pd.read_table(data_path, header=None)
label_df = pd.read_table(label_path, header=None)
## data overview
print("data shape", data_df.shape)
print("label shape", label_df.shape)
## data
label_array = label_df.dropna().values
train_markers = []
for events in label_array:
if events[1] != 0:
for i in range(0, 400, 50):
train_markers.append((float(events[0]) + i, str(int(events[1]))))
markers_subject1_class_1 = [(float(events[0]),str(int(events[1]))) for events in train_markers if events[1]== '1']
markers_subject1_class_2 = [(float(events[0]),str(int(events[1]))) for events in train_markers if events[1]== '2']
data_array = data_df.values
cnt1 = convert_mushu_data(data_array, markers_subject1_class_1, 50, channels)
cnt2 = convert_mushu_data(data_array, markers_subject1_class_2, 50, channels)
epoch_subject1_class1 = segment_dat(cnt1, md, [0, 1000]) # 640x50x118
epoch_subject1_class2 = segment_dat(cnt2, md, [0, 1000]) # 704x50x118
final_epoch = append_epo(epoch_subject1_class1,epoch_subject1_class2) #1344x50x118
targets = final_epoch.axes[0]
methods = ['_csp', '_bandpowers', '_dct', '_wavelet']
for i, func in enumerate(['_csp', utils.bandpowers, utils.dct_features, utils.wavelet_features]):
if func == '_csp':
from mne.decoding import CSP
csp = CSP(n_components=50, reg=None, log=True, norm_trace=True)
dictionary = csp.fit_transform(final_epoch.data, targets)
else:
dictionary = feature_transform(final_epoch, func)
## save the data
res = np.concatenate([dictionary, targets.reshape(-1, 1)], axis=1)
res_df = pd.DataFrame(res)
save_path = os.path.join(data_dir, filename + methods[i] + '.csv')
res_df.to_csv(save_path, index=False)
print("==> saved data at {}".format(save_path))
def test_convert_mushu_data_copy(self):
"""convert_mushu_data should make a copy of its arguments."""
data = self.data.copy()
channels = self.channels[:]
markers = self.markers[:]
cnt = convert_mushu_data(data, self.markers, FS, channels)
# data
data[0, 0] = -1
np.testing.assert_array_equal(cnt.data, self.data)
# axes
channels[0] = 'FOO'
np.testing.assert_array_equal(cnt.axes[-1], self.channels)
# markers
markers[0][0] = markers[0][0] - 1
self.assertEqual(cnt.markers, self.markers)
def test_convert_mushu_data_copy(self):
"""convert_mushu_data should make a copy of its arguments."""
data = self.data.copy()
channels = self.channels[:]
markers = self.markers[:]
cnt = convert_mushu_data(data, self.markers, FS, channels)
# data
data[0, 0] = -1
np.testing.assert_array_equal(cnt.data, self.data)
# axes
channels[0] = "FOO"
np.testing.assert_array_equal(cnt.axes[-1], self.channels)
# markers
markers[0][0] = markers[0][0] - 1
self.assertEqual(cnt.markers, self.markers)
def test_convert_mushu_data(self):
"""Convert mushu data."""
cnt = convert_mushu_data(self.data, self.markers, FS, self.channels)
# data
np.testing.assert_array_equal(cnt.data, self.data)
self.assertEqual(cnt.data.shape, (SAMPLES, CHANNELS))
# axes
timeaxis = np.linspace(0, SAMPLES / FS * 1000, SAMPLES, endpoint=False)
np.testing.assert_array_equal(cnt.axes[0], timeaxis)
np.testing.assert_array_equal(cnt.axes[1], self.channels)
# names and units
self.assertEqual(cnt.names, ['time', 'channel'])
self.assertEqual(cnt.units, ['uV', '#'])
# fs
self.assertEqual(cnt.fs, FS)
# markers
self.assertEqual(cnt.markers, self.markers)
def test_convert_mushu_data(self):
"""Convert mushu data."""
cnt = convert_mushu_data(self.data, self.markers, FS, self.channels)
# data
np.testing.assert_array_equal(cnt.data, self.data)
self.assertEqual(cnt.data.shape, (SAMPLES, CHANNELS))
# axes
timeaxis = np.linspace(0, SAMPLES / FS * 1000, SAMPLES, endpoint=False)
np.testing.assert_array_equal(cnt.axes[0], timeaxis)
np.testing.assert_array_equal(cnt.axes[1], self.channels)
# names and units
self.assertEqual(cnt.names, ["time", "channel"])
self.assertEqual(cnt.units, ["uV", "#"])
# fs
self.assertEqual(cnt.fs, FS)
# markers
self.assertEqual(cnt.markers, self.markers)
def load_epo_data(data_cat, n_before=-3, n_len=100, subjects=None):
# loading 'data_cat' data
data, channels, markers = load_data(FS, folder_name, data_cat, subjects)
# converting plain data to continuous Data object
cnt = convert_mushu_data(data, markers, FS, channels)
# Define the markers belonging to class 1 and 2
markers_definitions = None
if data_cat == 'train':
markers_definitions = {'class 1': (train_labels.query('Prediction == 0', engine='python')['IdFeedBack']).tolist(),
'class 2': (train_labels.query('Prediction == 1', engine='python')['IdFeedBack']).tolist()}
else:
# marker classes doesn't matter for test data
markers_definitions = {'class 1': [m[1] for m in markers], 'class 2': []}
# segmenting continuous Data object into epoched data
# Epoch the data -25ms(5 rows) and +500ms(100 rows) around the markers defined in markers_definitions
return segment_dat(cnt, markers_definitions, [n_before*5, (n_before + n_len)*5])
def online_experiment(amp, clf):
amp_fs = amp.get_sampling_frequency()
amp_channels = amp.get_channels()
buf = BlockBuffer(4)
rb = RingBuffer(5000)
fn = amp.get_sampling_frequency() / 2
b_low, a_low = proc.signal.butter(16, [30 / fn], btype='low')
b_high, a_high = proc.signal.butter(5, [.4 / fn], btype='high')
zi_low = proc.lfilter_zi(b_low, a_low, len(amp_channels))
zi_high = proc.lfilter_zi(b_high, a_high, len(amp_channels))
amp.start()
markers_processed = 0
current_letter_idx = 0
current_letter = TRUE_LABELS[current_letter_idx].lower()
letter_prob = {i: 0 for i in 'abcdefghijklmnopqrstuvwxyz123456789_'}
endresult = []
while True:
# turn on for 'real time'
#time.sleep(0.01)
# get fresh data from the amp
data, markers = amp.get_data()
# we should rather wait for a specific end-of-experiment marker
if len(data) == 0:
break
# convert to cnt
cnt = io.convert_mushu_data(data, markers, amp_fs, amp_channels)
# enter the block buffer
buf.append(cnt)
cnt = buf.get()
if not cnt:
continue
# band-pass and subsample
cnt, zi_low = proc.lfilter(cnt, b_low, a_low, zi=zi_low)
cnt, zi_high = proc.lfilter(cnt, b_high, a_high, zi=zi_high)
cnt = proc.subsample(cnt, 60)
newsamples = cnt.data.shape[0]
# enter the ringbuffer
rb.append(cnt)
cnt = rb.get()
# segment
epo = proc.segment_dat(cnt,
MARKER_DEF_TEST,
SEG_IVAL,
newsamples=newsamples)
if not epo:
continue
fv = proc.jumping_means(epo, JUMPING_MEANS_IVALS)
fv = proc.create_feature_vectors(fv)
logger.debug(markers_processed)
lda_out = proc.lda_apply(fv, clf)
markers = [fv.class_names[cls_idx] for cls_idx in fv.axes[0]]
result = zip(markers, lda_out)
for s, score in result:
if markers_processed == 180:
endresult.append(
sorted(letter_prob.items(), key=lambda x: x[1])[-1][0])
letter_prob = {
i: 0
for i in 'abcdefghijklmnopqrstuvwxyz123456789_'
}
markers_processed = 0
current_letter_idx += 1
current_letter = TRUE_LABELS[current_letter_idx].lower()
for letter in s:
letter_prob[letter] += score
markers_processed += 1
logger.debug("".join([
i[0] for i in sorted(
letter_prob.items(), key=lambda x: x[1], reverse=True)
]).replace(current_letter, " %s " % current_letter))
logger.debug(TRUE_LABELS)
logger.debug("".join(endresult))
# calculate the current accuracy
if len(endresult) > 0:
acc = np.count_nonzero(
np.array(endresult) == np.array(
list(TRUE_LABELS.lower()[:len(endresult)]))) / len(
endresult)
print "Current accuracy:", acc * 100
if len(endresult) == len(TRUE_LABELS):
break
#logger.debug("Result: %s" % result)
acc = np.count_nonzero(
np.array(endresult) == np.array(
list(TRUE_LABELS.lower()[:len(endresult)]))) / len(endresult)
print "Accuracy:", acc * 100
amp.stop()
def runLoop():
alld = dynarray.DynamicArray(
(None, len(amp.get_channels()))) # the growing numpy data matrix
allm = [] # markers
sfreq = amp.get_sampling_frequency() # sampling frequency
ch_names = amp.get_channels() # channel names
rb = RingBuffer(
buffSize * 1000
) # the buffer containing the last X seconds of data - declared in MILISECONDS
totalTime = seed_d.shape[0] / raw_fromfile.info['sfreq']
fig = plt.figure() # plotting...
th = fig.suptitle('')
ah1 = fig.add_subplot(131)
ah2 = fig.add_subplot(132)
ah3 = fig.add_subplot(133)
l1, = ah1.plot(sx, sy1)
l2, = ah2.plot(sx, sy2)
l3, = ah3.plot(sx2, sy3)
# l=LoopState(); l.start()
markeroffset = 0 # needed to store all data in one big mat/vector
t0 = time.time()
curTime = time.time()
markTime = time.time()
st = ''
i = 0
fnames = [] # for making a movie...
while curTime - t0 < totalTime: # l.get_state() != 'Stop':
# keep track of time:
curTime = time.time()
# this is where you get the data
data, marker = amp.get_data()
if data.shape[0] > 0: # this is crucial for remembering filter state.
data2 = hpf.handle(data)
data3 = mr.handle(data2)
data4 = lpf.handle(data3)
data5 = resample.handle(data4)
data6 = cwl.handle(data5)
# something like this:
#filterchain = [HPFilter, MRFilter, LPFilter, ResampleFilter, HPFilter, CWLFilter]
#corr_data = ProcessFilters(chain, (data, marker))
# use case -- first using the MR Corrector
#mr_data = MRFilter.filter(data)
# then -- using the CWL corrector
#cwl_mr_data = CWLFilter.filter(data)
#dataf, rt_zi_bp = signal.lfilter(rt_b_bp, rt_a_bp, data, axis=0, zi=rt_zi_bp) # how to operate directly on the data
cnt = io.convert_mushu_data(data, marker, sfreq, ch_names)
mr_cnt = io.convert_mushu_data(data3, marker, sfreq, ch_names)
cwl_cnt = io.convert_mushu_data(data6, marker, sfreq, ch_names)
# f_cnt, rt_zi_bp = proc.lfilter(cnt, rt_b_bp, rt_a_bp, zi=rt_zi_bp) # real-time data preprocessing...
# plotting...
sy1.extend(cnt.data[:, channel_to_plot]
) # to visualize/plot -- s1 and s2 are deque's
sy2.extend(mr_cnt.data[:, channel_to_plot])
sy3.extend(cwl_cnt.data[:, channel_to_plot])
l1.set_ydata(sy1)
l2.set_ydata(sy2)
l3.set_ydata(sy3)
msy1 = np.mean(sy1)
msy2 = np.mean(sy2)
msy3 = np.mean(sy3)
ah1.set_ylim(-5000 + msy1, 5000 + msy1)
ah2.set_ylim(-250 + msy2, 250 + msy2)
ah3.set_ylim(-250 + msy3, 250 + msy3)
fig.canvas.draw()
fig.canvas.flush_events()
# i+=1; fname = '_tmp%03d.png' % i; plt.savefig(fname); fnames.append(fname)
# currently has no purpose
newsamples = cnt.data.shape[0]
# append to ringbuffer, so we can calculate features later on on the last N secs/samples of data.
rb.append(cwl_cnt)
# append it to the big matrix, for saving later on with pickle.
alld.extend(data6)
for m in marker:
allm.append([m[0] + markeroffset, m[1]])
markeroffset += newsamples / float(sfreq) * 1000.
# do the following every 0.1 msec - with with the ringbuffer:
if curTime - markTime > updateTime:
# do Stuff
markTime = curTime
# 1) obtain last 1-second(s)
d = rb.get()
# clear_output(wait=True) # write some logging information here
# clear_output clear the output of the cell, but if you do that you also remove the figures, it seems
# so don't do it!
str1 = 'Playing Back - time = %f' % (curTime - t0)
str2 = 'Length Markers: %d' % len(allm)
str3 = '%d, %d' % data.shape
#str4 = 'Feature Value: %f' % feature
#str5 = 'Scaled Signal for NF: %f' % signalToSend
#print(str1 + '\n' + str2 + '\n' + str3 + '\n' + str4 + '\n' + str5)
# print('Length Markers: %d' % len(allm))
# print(data.shape)
th.set_text(str1 + '\n' + str2 + '\n' + str3)
def online_experiment(amp, cfy):
amp_fs = amp.get_sampling_frequency()
amp_channels = amp.get_channels()
# buf = BlockBuffer(4)
rb = RingBuffer(5000)
fn = amp_fs / 2
b_low, a_low = proc.signal.butter(5, [38 / fn], btype='low')
b_high, a_high = proc.signal.butter(5, [.1 / fn], btype='high')
zi_low = proc.lfilter_zi(b_low, a_low, len(amp_channels))
zi_high = proc.lfilter_zi(b_high, a_high, len(amp_channels))
amp.start()
print("Iniciando simulacao em 5s...")
for x in xrange(4, 0, -1):
time.sleep(1)
print("%ds" % x)
pass
markers_processed = 0
current_letter_idx = 0
current_letter = TRUE_LABELS[current_letter_idx].lower()
letter_prob = {i: 0 for i in 'abcdefghijklmnopqrstuvwxyz123456789_'}
endresult = []
t0 = time.time()
while True:
t0 = time.time()
# get fresh data from the amp
data, markers = amp.get_data()
if len(data) == 0:
continue
# we should rather wait for a specific end-of-experiment marker
if len(data) == 0:
break
# convert to cnt
cnt = io.convert_mushu_data(data, markers, amp_fs, amp_channels)
# enter the block buffer
# buf.append(cnt)
# cnt = buf.get()
# if not cnt:
# continue
# band-pass and subsample
cnt, zi_low = proc.lfilter(cnt, b_low, a_low, zi=zi_low)
cnt, zi_high = proc.lfilter(cnt, b_high, a_high, zi=zi_high)
cnt = proc.subsample(cnt, 60)
newsamples = cnt.data.shape[0]
# enter the ringbuffer
rb.append(cnt)
cnt = rb.get()
# segment
epo = proc.segment_dat(cnt,
MARKER_DEF_TEST,
SEG_IVAL,
newsamples=newsamples)
if not epo:
continue
fv = proc.jumping_means(epo, JUMPING_MEANS_IVALS)
fv = proc.create_feature_vectors(fv)
print("\n")
logger.info('Step : %d' % markers_processed)
lda_out = proc.lda_apply(fv, cfy)
markers = [fv.class_names[cls_idx] for cls_idx in fv.axes[0]]
result = zip(markers, lda_out)
for s, score in result:
if markers_processed == 180:
endresult.append(
sorted(letter_prob.items(), key=lambda x: x[1])[-1][0])
letter_prob = {
i: 0
for i in 'abcdefghijklmnopqrstuvwxyz123456789_'
}
markers_processed = 0
current_letter_idx += 1
current_letter = TRUE_LABELS[current_letter_idx].lower()
for letter in s:
letter_prob[letter] += score
markers_processed += 1
print('Letra Atual Correta-: %s ' % current_letter)
print("Letra provavel--: %s" % "".join([
i[0] for i in sorted(
letter_prob.items(), key=lambda x: x[1], reverse=True)
]).replace(current_letter, " '%s' " % current_letter))
print('Letras Corretas----: %s' % TRUE_LABELS)
# discovered = BuildDiscoveredString(endresult)
# print('Letras Encontradas-: %s' % discovered)
print('Letras Encontradas-: %s' % "".join(endresult))
# calculate the current accuracy
if len(endresult) > 0:
acc = np.count_nonzero(
np.array(endresult) == np.array(
list(TRUE_LABELS.lower()[:len(endresult)]))) / len(
endresult)
print('Acertividade Atual : %d' % (acc * 100))
if len(endresult) == len(TRUE_LABELS) - 1:
break
# logger.debug('Resultado : %s' % result)
timeValue = 1000 * (time.time() - t0)
print('Tempo consumido por ciclo : %d' % timeValue)
acc = np.count_nonzero(
np.array(endresult) == np.array(
list(TRUE_LABELS.lower()[:len(endresult)]))) / len(endresult)
print("Acertividade Final : %d" % (acc * 100))
amp.stop()
str(events[1]))) #taking only middle elements between 100&200
train_markers1.append(
(float(events[0]) + 200.0, str(events[1]))
) #again filling up the data this time taking two rows for same data point one is starting point other is end point
markers1 = np.array(train_markers1) #markers1 is numpy array of train_markers1
markers_subject1_class_1 = [
(float(events[0]), str(events[1])) for events in markers1
if events[1] == '1\n'
] # separate line index starting and ending for class 1 and class 2
markers_subject1_class_2 = [
(float(events[0]), str(events[1])) for events in markers1
if events[1] == '2\n'
] #markers_subject1_class_1 and like wise other class correspondingly contains the 1 and 2 class marker data points
## ## print type(markers_subject1_class_1),type(markers_subject1_class_2),"markers_subject1_class1&2 data type"
cnt1 = convert_mushu_data(
signal_array1, markers_subject1_class_1, 118,
channels) #convert data into continuous form for 1st and second classs
cnt2 = convert_mushu_data(signal_array1, markers_subject1_class_2, 118,
channels)
cnt_ch1 = convert_mushu_data(
signal_channel1, markers_subject1_class_1, 30, main_channels
) #convert data into continuous form for 1st and second classs
cnt_ch2 = convert_mushu_data(signal_channel1, markers_subject1_class_2, 30,
main_channels)
## ## print cnt1,"cnt1 shape" #What type of marker data should be there should it contain start as well as end point or just start point is required
md = {'class 1': ['1\n'], 'class 2': ['2\n']}
epoch_subject1_class1 = segment_dat(
cnt1, md, [0, 1000]) #epoch is a 3-d data set class*time*channel
epoch_subject1_class2 = segment_dat(cnt2, md, [0, 1000])
def online_experiment(amp, cfy):
amp_fs = amp.get_sampling_frequency()
amp_channels = amp.get_channels()
#buf = BlockBuffer(4)
rb = RingBuffer(5000)
fn = amp_fs / 2
b_low, a_low = proc.signal.butter(5, [30 / fn], btype='low')
b_high, a_high = proc.signal.butter(5, [.4 / fn], btype='high')
zi_low = proc.lfilter_zi(b_low, a_low, len(amp_channels))
zi_high = proc.lfilter_zi(b_high, a_high, len(amp_channels))
amp.start()
markers_processed = 0
current_letter_idx = 0
current_letter = TRUE_LABELS[current_letter_idx].lower()
letter_prob = {i : 0 for i in 'abcdefghijklmnopqrstuvwxyz123456789_'}
endresult = []
t0 = time.time()
while True:
t0 = time.time()
# get fresh data from the amp
data, markers = amp.get_data()
if len(data) == 0:
continue
# we should rather wait for a specific end-of-experiment marker
if len(data) == 0:
break
# convert to cnt
cnt = io.convert_mushu_data(data, markers, amp_fs, amp_channels)
## enter the block buffer
#buf.append(cnt)
#cnt = buf.get()
#if not cnt:
# continue
# band-pass and subsample
cnt, zi_low = proc.lfilter(cnt, b_low, a_low, zi=zi_low)
cnt, zi_high = proc.lfilter(cnt, b_high, a_high, zi=zi_high)
cnt = proc.subsample(cnt, 60)
newsamples = cnt.data.shape[0]
# enter the ringbuffer
rb.append(cnt)
cnt = rb.get()
# segment
epo = proc.segment_dat(cnt, MARKER_DEF_TEST, SEG_IVAL, newsamples=newsamples)
if not epo:
continue
fv = proc.jumping_means(epo, JUMPING_MEANS_IVALS)
fv = proc.create_feature_vectors(fv)
logger.debug(markers_processed)
lda_out = proc.lda_apply(fv, cfy)
markers = [fv.class_names[cls_idx] for cls_idx in fv.axes[0]]
result = zip(markers, lda_out)
for s, score in result:
if markers_processed == 180:
endresult.append(sorted(letter_prob.items(), key=lambda x: x[1])[-1][0])
letter_prob = {i : 0 for i in 'abcdefghijklmnopqrstuvwxyz123456789_'}
markers_processed = 0
current_letter_idx += 1
current_letter = TRUE_LABELS[current_letter_idx].lower()
for letter in s:
letter_prob[letter] += score
markers_processed += 1
logger.debug("".join([i[0] for i in sorted(letter_prob.items(), key=lambda x: x[1], reverse=True)]).replace(current_letter, " %s " % current_letter))
logger.debug(TRUE_LABELS)
logger.debug("".join(endresult))
# calculate the current accuracy
if len(endresult) > 0:
acc = np.count_nonzero(np.array(endresult) == np.array(list(TRUE_LABELS.lower()[:len(endresult)]))) / len(endresult)
print "Current accuracy:", acc * 100
if len(endresult) == len(TRUE_LABELS):
break
#logger.debug("Result: %s" % result)
print 1000 * (time.time() - t0)
acc = np.count_nonzero(np.array(endresult) == np.array(list(TRUE_LABELS.lower()[:len(endresult)]))) / len(endresult)
print "Accuracy:", acc * 100
amp.stop()
str(events[1]))) #taking only middle elements between 100&200
train_markers1.append(
(float(events[0]) + 200.0, str(events[1]))
) #again filling up the data this time taking two rows for same data point one is starting point other is end point
markers1 = np.array(train_markers1) #markers1 is numpy array of train_markers1
markers_subject1_class_1 = [
(float(events[0]), str(events[1])) for events in markers1
if events[1] == '1\n'
] # separate line index starting and ending for class 1 and class 2
markers_subject1_class_2 = [
(float(events[0]), str(events[1])) for events in markers1
if events[1] == '2\n'
] #markers_subject1_class_1 and like wise other class correspondingly contains the 1 and 2 class marker data points
#print "cerberus",markers_subject1_class_1
cnt1 = convert_mushu_data(
signal_array, markers_subject1_class_1, 100,
channels) #convert data into continuous form for 1st and second classs
cnt2 = convert_mushu_data(signal_array, markers_subject1_class_2, 100,
channels)
#print cnt1,"cnt1 shape" #What type of marker data should be there should it contain start as well as end point or just start point is required
md = {'class 1': ['1\n'], 'class 2': ['2\n']}
epoch_subject1_class1 = segment_dat(
cnt1, md, [0, 1000]) #epoch is a 3-d data set class*time*channel
epoch_subject1_class2 = segment_dat(cnt2, md, [0, 1000])
#print "epoch data",epoch_subject1_class1
def bandpowers(segment):
features = []