def process(filename): data_dir = os.path.join("../Datasets/", filename) data_path = os.path.join(data_dir, filename + '_cnt.txt') label_path = os.path.join(data_dir, filename + '_mrk.txt') data_df = pd.read_table(data_path, header=None) label_df = pd.read_table(label_path, header=None) ## data overview print("data shape", data_df.shape) print("label shape", label_df.shape) ## data label_array = label_df.dropna().values train_markers = [] for events in label_array: if events[1] != 0: for i in range(0, 400, 50): train_markers.append((float(events[0]) + i, str(int(events[1])))) markers_subject1_class_1 = [(float(events[0]),str(int(events[1]))) for events in train_markers if events[1]== '1'] markers_subject1_class_2 = [(float(events[0]),str(int(events[1]))) for events in train_markers if events[1]== '2'] data_array = data_df.values cnt1 = convert_mushu_data(data_array, markers_subject1_class_1, 50, channels) cnt2 = convert_mushu_data(data_array, markers_subject1_class_2, 50, channels) epoch_subject1_class1 = segment_dat(cnt1, md, [0, 1000]) # 640x50x118 epoch_subject1_class2 = segment_dat(cnt2, md, [0, 1000]) # 704x50x118 final_epoch = append_epo(epoch_subject1_class1,epoch_subject1_class2) #1344x50x118 targets = final_epoch.axes[0] methods = ['_csp', '_bandpowers', '_dct', '_wavelet'] for i, func in enumerate(['_csp', utils.bandpowers, utils.dct_features, utils.wavelet_features]): if func == '_csp': from mne.decoding import CSP csp = CSP(n_components=50, reg=None, log=True, norm_trace=True) dictionary = csp.fit_transform(final_epoch.data, targets) else: dictionary = feature_transform(final_epoch, func) ## save the data res = np.concatenate([dictionary, targets.reshape(-1, 1)], axis=1) res_df = pd.DataFrame(res) save_path = os.path.join(data_dir, filename + methods[i] + '.csv') res_df.to_csv(save_path, index=False) print("==> saved data at {}".format(save_path))
def test_convert_mushu_data_copy(self): """convert_mushu_data should make a copy of its arguments.""" data = self.data.copy() channels = self.channels[:] markers = self.markers[:] cnt = convert_mushu_data(data, self.markers, FS, channels) # data data[0, 0] = -1 np.testing.assert_array_equal(cnt.data, self.data) # axes channels[0] = 'FOO' np.testing.assert_array_equal(cnt.axes[-1], self.channels) # markers markers[0][0] = markers[0][0] - 1 self.assertEqual(cnt.markers, self.markers)
def test_convert_mushu_data_copy(self): """convert_mushu_data should make a copy of its arguments.""" data = self.data.copy() channels = self.channels[:] markers = self.markers[:] cnt = convert_mushu_data(data, self.markers, FS, channels) # data data[0, 0] = -1 np.testing.assert_array_equal(cnt.data, self.data) # axes channels[0] = "FOO" np.testing.assert_array_equal(cnt.axes[-1], self.channels) # markers markers[0][0] = markers[0][0] - 1 self.assertEqual(cnt.markers, self.markers)
def test_convert_mushu_data(self): """Convert mushu data.""" cnt = convert_mushu_data(self.data, self.markers, FS, self.channels) # data np.testing.assert_array_equal(cnt.data, self.data) self.assertEqual(cnt.data.shape, (SAMPLES, CHANNELS)) # axes timeaxis = np.linspace(0, SAMPLES / FS * 1000, SAMPLES, endpoint=False) np.testing.assert_array_equal(cnt.axes[0], timeaxis) np.testing.assert_array_equal(cnt.axes[1], self.channels) # names and units self.assertEqual(cnt.names, ['time', 'channel']) self.assertEqual(cnt.units, ['uV', '#']) # fs self.assertEqual(cnt.fs, FS) # markers self.assertEqual(cnt.markers, self.markers)
def test_convert_mushu_data(self): """Convert mushu data.""" cnt = convert_mushu_data(self.data, self.markers, FS, self.channels) # data np.testing.assert_array_equal(cnt.data, self.data) self.assertEqual(cnt.data.shape, (SAMPLES, CHANNELS)) # axes timeaxis = np.linspace(0, SAMPLES / FS * 1000, SAMPLES, endpoint=False) np.testing.assert_array_equal(cnt.axes[0], timeaxis) np.testing.assert_array_equal(cnt.axes[1], self.channels) # names and units self.assertEqual(cnt.names, ["time", "channel"]) self.assertEqual(cnt.units, ["uV", "#"]) # fs self.assertEqual(cnt.fs, FS) # markers self.assertEqual(cnt.markers, self.markers)
def load_epo_data(data_cat, n_before=-3, n_len=100, subjects=None): # loading 'data_cat' data data, channels, markers = load_data(FS, folder_name, data_cat, subjects) # converting plain data to continuous Data object cnt = convert_mushu_data(data, markers, FS, channels) # Define the markers belonging to class 1 and 2 markers_definitions = None if data_cat == 'train': markers_definitions = {'class 1': (train_labels.query('Prediction == 0', engine='python')['IdFeedBack']).tolist(), 'class 2': (train_labels.query('Prediction == 1', engine='python')['IdFeedBack']).tolist()} else: # marker classes doesn't matter for test data markers_definitions = {'class 1': [m[1] for m in markers], 'class 2': []} # segmenting continuous Data object into epoched data # Epoch the data -25ms(5 rows) and +500ms(100 rows) around the markers defined in markers_definitions return segment_dat(cnt, markers_definitions, [n_before*5, (n_before + n_len)*5])
def online_experiment(amp, clf): amp_fs = amp.get_sampling_frequency() amp_channels = amp.get_channels() buf = BlockBuffer(4) rb = RingBuffer(5000) fn = amp.get_sampling_frequency() / 2 b_low, a_low = proc.signal.butter(16, [30 / fn], btype='low') b_high, a_high = proc.signal.butter(5, [.4 / fn], btype='high') zi_low = proc.lfilter_zi(b_low, a_low, len(amp_channels)) zi_high = proc.lfilter_zi(b_high, a_high, len(amp_channels)) amp.start() markers_processed = 0 current_letter_idx = 0 current_letter = TRUE_LABELS[current_letter_idx].lower() letter_prob = {i: 0 for i in 'abcdefghijklmnopqrstuvwxyz123456789_'} endresult = [] while True: # turn on for 'real time' #time.sleep(0.01) # get fresh data from the amp data, markers = amp.get_data() # we should rather wait for a specific end-of-experiment marker if len(data) == 0: break # convert to cnt cnt = io.convert_mushu_data(data, markers, amp_fs, amp_channels) # enter the block buffer buf.append(cnt) cnt = buf.get() if not cnt: continue # band-pass and subsample cnt, zi_low = proc.lfilter(cnt, b_low, a_low, zi=zi_low) cnt, zi_high = proc.lfilter(cnt, b_high, a_high, zi=zi_high) cnt = proc.subsample(cnt, 60) newsamples = cnt.data.shape[0] # enter the ringbuffer rb.append(cnt) cnt = rb.get() # segment epo = proc.segment_dat(cnt, MARKER_DEF_TEST, SEG_IVAL, newsamples=newsamples) if not epo: continue fv = proc.jumping_means(epo, JUMPING_MEANS_IVALS) fv = proc.create_feature_vectors(fv) logger.debug(markers_processed) lda_out = proc.lda_apply(fv, clf) markers = [fv.class_names[cls_idx] for cls_idx in fv.axes[0]] result = zip(markers, lda_out) for s, score in result: if markers_processed == 180: endresult.append( sorted(letter_prob.items(), key=lambda x: x[1])[-1][0]) letter_prob = { i: 0 for i in 'abcdefghijklmnopqrstuvwxyz123456789_' } markers_processed = 0 current_letter_idx += 1 current_letter = TRUE_LABELS[current_letter_idx].lower() for letter in s: letter_prob[letter] += score markers_processed += 1 logger.debug("".join([ i[0] for i in sorted( letter_prob.items(), key=lambda x: x[1], reverse=True) ]).replace(current_letter, " %s " % current_letter)) logger.debug(TRUE_LABELS) logger.debug("".join(endresult)) # calculate the current accuracy if len(endresult) > 0: acc = np.count_nonzero( np.array(endresult) == np.array( list(TRUE_LABELS.lower()[:len(endresult)]))) / len( endresult) print "Current accuracy:", acc * 100 if len(endresult) == len(TRUE_LABELS): break #logger.debug("Result: %s" % result) acc = np.count_nonzero( np.array(endresult) == np.array( list(TRUE_LABELS.lower()[:len(endresult)]))) / len(endresult) print "Accuracy:", acc * 100 amp.stop()
def runLoop(): alld = dynarray.DynamicArray( (None, len(amp.get_channels()))) # the growing numpy data matrix allm = [] # markers sfreq = amp.get_sampling_frequency() # sampling frequency ch_names = amp.get_channels() # channel names rb = RingBuffer( buffSize * 1000 ) # the buffer containing the last X seconds of data - declared in MILISECONDS totalTime = seed_d.shape[0] / raw_fromfile.info['sfreq'] fig = plt.figure() # plotting... th = fig.suptitle('') ah1 = fig.add_subplot(131) ah2 = fig.add_subplot(132) ah3 = fig.add_subplot(133) l1, = ah1.plot(sx, sy1) l2, = ah2.plot(sx, sy2) l3, = ah3.plot(sx2, sy3) # l=LoopState(); l.start() markeroffset = 0 # needed to store all data in one big mat/vector t0 = time.time() curTime = time.time() markTime = time.time() st = '' i = 0 fnames = [] # for making a movie... while curTime - t0 < totalTime: # l.get_state() != 'Stop': # keep track of time: curTime = time.time() # this is where you get the data data, marker = amp.get_data() if data.shape[0] > 0: # this is crucial for remembering filter state. data2 = hpf.handle(data) data3 = mr.handle(data2) data4 = lpf.handle(data3) data5 = resample.handle(data4) data6 = cwl.handle(data5) # something like this: #filterchain = [HPFilter, MRFilter, LPFilter, ResampleFilter, HPFilter, CWLFilter] #corr_data = ProcessFilters(chain, (data, marker)) # use case -- first using the MR Corrector #mr_data = MRFilter.filter(data) # then -- using the CWL corrector #cwl_mr_data = CWLFilter.filter(data) #dataf, rt_zi_bp = signal.lfilter(rt_b_bp, rt_a_bp, data, axis=0, zi=rt_zi_bp) # how to operate directly on the data cnt = io.convert_mushu_data(data, marker, sfreq, ch_names) mr_cnt = io.convert_mushu_data(data3, marker, sfreq, ch_names) cwl_cnt = io.convert_mushu_data(data6, marker, sfreq, ch_names) # f_cnt, rt_zi_bp = proc.lfilter(cnt, rt_b_bp, rt_a_bp, zi=rt_zi_bp) # real-time data preprocessing... # plotting... sy1.extend(cnt.data[:, channel_to_plot] ) # to visualize/plot -- s1 and s2 are deque's sy2.extend(mr_cnt.data[:, channel_to_plot]) sy3.extend(cwl_cnt.data[:, channel_to_plot]) l1.set_ydata(sy1) l2.set_ydata(sy2) l3.set_ydata(sy3) msy1 = np.mean(sy1) msy2 = np.mean(sy2) msy3 = np.mean(sy3) ah1.set_ylim(-5000 + msy1, 5000 + msy1) ah2.set_ylim(-250 + msy2, 250 + msy2) ah3.set_ylim(-250 + msy3, 250 + msy3) fig.canvas.draw() fig.canvas.flush_events() # i+=1; fname = '_tmp%03d.png' % i; plt.savefig(fname); fnames.append(fname) # currently has no purpose newsamples = cnt.data.shape[0] # append to ringbuffer, so we can calculate features later on on the last N secs/samples of data. rb.append(cwl_cnt) # append it to the big matrix, for saving later on with pickle. alld.extend(data6) for m in marker: allm.append([m[0] + markeroffset, m[1]]) markeroffset += newsamples / float(sfreq) * 1000. # do the following every 0.1 msec - with with the ringbuffer: if curTime - markTime > updateTime: # do Stuff markTime = curTime # 1) obtain last 1-second(s) d = rb.get() # clear_output(wait=True) # write some logging information here # clear_output clear the output of the cell, but if you do that you also remove the figures, it seems # so don't do it! str1 = 'Playing Back - time = %f' % (curTime - t0) str2 = 'Length Markers: %d' % len(allm) str3 = '%d, %d' % data.shape #str4 = 'Feature Value: %f' % feature #str5 = 'Scaled Signal for NF: %f' % signalToSend #print(str1 + '\n' + str2 + '\n' + str3 + '\n' + str4 + '\n' + str5) # print('Length Markers: %d' % len(allm)) # print(data.shape) th.set_text(str1 + '\n' + str2 + '\n' + str3)
def online_experiment(amp, cfy): amp_fs = amp.get_sampling_frequency() amp_channels = amp.get_channels() # buf = BlockBuffer(4) rb = RingBuffer(5000) fn = amp_fs / 2 b_low, a_low = proc.signal.butter(5, [38 / fn], btype='low') b_high, a_high = proc.signal.butter(5, [.1 / fn], btype='high') zi_low = proc.lfilter_zi(b_low, a_low, len(amp_channels)) zi_high = proc.lfilter_zi(b_high, a_high, len(amp_channels)) amp.start() print("Iniciando simulacao em 5s...") for x in xrange(4, 0, -1): time.sleep(1) print("%ds" % x) pass markers_processed = 0 current_letter_idx = 0 current_letter = TRUE_LABELS[current_letter_idx].lower() letter_prob = {i: 0 for i in 'abcdefghijklmnopqrstuvwxyz123456789_'} endresult = [] t0 = time.time() while True: t0 = time.time() # get fresh data from the amp data, markers = amp.get_data() if len(data) == 0: continue # we should rather wait for a specific end-of-experiment marker if len(data) == 0: break # convert to cnt cnt = io.convert_mushu_data(data, markers, amp_fs, amp_channels) # enter the block buffer # buf.append(cnt) # cnt = buf.get() # if not cnt: # continue # band-pass and subsample cnt, zi_low = proc.lfilter(cnt, b_low, a_low, zi=zi_low) cnt, zi_high = proc.lfilter(cnt, b_high, a_high, zi=zi_high) cnt = proc.subsample(cnt, 60) newsamples = cnt.data.shape[0] # enter the ringbuffer rb.append(cnt) cnt = rb.get() # segment epo = proc.segment_dat(cnt, MARKER_DEF_TEST, SEG_IVAL, newsamples=newsamples) if not epo: continue fv = proc.jumping_means(epo, JUMPING_MEANS_IVALS) fv = proc.create_feature_vectors(fv) print("\n") logger.info('Step : %d' % markers_processed) lda_out = proc.lda_apply(fv, cfy) markers = [fv.class_names[cls_idx] for cls_idx in fv.axes[0]] result = zip(markers, lda_out) for s, score in result: if markers_processed == 180: endresult.append( sorted(letter_prob.items(), key=lambda x: x[1])[-1][0]) letter_prob = { i: 0 for i in 'abcdefghijklmnopqrstuvwxyz123456789_' } markers_processed = 0 current_letter_idx += 1 current_letter = TRUE_LABELS[current_letter_idx].lower() for letter in s: letter_prob[letter] += score markers_processed += 1 print('Letra Atual Correta-: %s ' % current_letter) print("Letra provavel--: %s" % "".join([ i[0] for i in sorted( letter_prob.items(), key=lambda x: x[1], reverse=True) ]).replace(current_letter, " '%s' " % current_letter)) print('Letras Corretas----: %s' % TRUE_LABELS) # discovered = BuildDiscoveredString(endresult) # print('Letras Encontradas-: %s' % discovered) print('Letras Encontradas-: %s' % "".join(endresult)) # calculate the current accuracy if len(endresult) > 0: acc = np.count_nonzero( np.array(endresult) == np.array( list(TRUE_LABELS.lower()[:len(endresult)]))) / len( endresult) print('Acertividade Atual : %d' % (acc * 100)) if len(endresult) == len(TRUE_LABELS) - 1: break # logger.debug('Resultado : %s' % result) timeValue = 1000 * (time.time() - t0) print('Tempo consumido por ciclo : %d' % timeValue) acc = np.count_nonzero( np.array(endresult) == np.array( list(TRUE_LABELS.lower()[:len(endresult)]))) / len(endresult) print("Acertividade Final : %d" % (acc * 100)) amp.stop()
str(events[1]))) #taking only middle elements between 100&200 train_markers1.append( (float(events[0]) + 200.0, str(events[1])) ) #again filling up the data this time taking two rows for same data point one is starting point other is end point markers1 = np.array(train_markers1) #markers1 is numpy array of train_markers1 markers_subject1_class_1 = [ (float(events[0]), str(events[1])) for events in markers1 if events[1] == '1\n' ] # separate line index starting and ending for class 1 and class 2 markers_subject1_class_2 = [ (float(events[0]), str(events[1])) for events in markers1 if events[1] == '2\n' ] #markers_subject1_class_1 and like wise other class correspondingly contains the 1 and 2 class marker data points ## ## print type(markers_subject1_class_1),type(markers_subject1_class_2),"markers_subject1_class1&2 data type" cnt1 = convert_mushu_data( signal_array1, markers_subject1_class_1, 118, channels) #convert data into continuous form for 1st and second classs cnt2 = convert_mushu_data(signal_array1, markers_subject1_class_2, 118, channels) cnt_ch1 = convert_mushu_data( signal_channel1, markers_subject1_class_1, 30, main_channels ) #convert data into continuous form for 1st and second classs cnt_ch2 = convert_mushu_data(signal_channel1, markers_subject1_class_2, 30, main_channels) ## ## print cnt1,"cnt1 shape" #What type of marker data should be there should it contain start as well as end point or just start point is required md = {'class 1': ['1\n'], 'class 2': ['2\n']} epoch_subject1_class1 = segment_dat( cnt1, md, [0, 1000]) #epoch is a 3-d data set class*time*channel epoch_subject1_class2 = segment_dat(cnt2, md, [0, 1000])
def online_experiment(amp, cfy): amp_fs = amp.get_sampling_frequency() amp_channels = amp.get_channels() #buf = BlockBuffer(4) rb = RingBuffer(5000) fn = amp_fs / 2 b_low, a_low = proc.signal.butter(5, [30 / fn], btype='low') b_high, a_high = proc.signal.butter(5, [.4 / fn], btype='high') zi_low = proc.lfilter_zi(b_low, a_low, len(amp_channels)) zi_high = proc.lfilter_zi(b_high, a_high, len(amp_channels)) amp.start() markers_processed = 0 current_letter_idx = 0 current_letter = TRUE_LABELS[current_letter_idx].lower() letter_prob = {i : 0 for i in 'abcdefghijklmnopqrstuvwxyz123456789_'} endresult = [] t0 = time.time() while True: t0 = time.time() # get fresh data from the amp data, markers = amp.get_data() if len(data) == 0: continue # we should rather wait for a specific end-of-experiment marker if len(data) == 0: break # convert to cnt cnt = io.convert_mushu_data(data, markers, amp_fs, amp_channels) ## enter the block buffer #buf.append(cnt) #cnt = buf.get() #if not cnt: # continue # band-pass and subsample cnt, zi_low = proc.lfilter(cnt, b_low, a_low, zi=zi_low) cnt, zi_high = proc.lfilter(cnt, b_high, a_high, zi=zi_high) cnt = proc.subsample(cnt, 60) newsamples = cnt.data.shape[0] # enter the ringbuffer rb.append(cnt) cnt = rb.get() # segment epo = proc.segment_dat(cnt, MARKER_DEF_TEST, SEG_IVAL, newsamples=newsamples) if not epo: continue fv = proc.jumping_means(epo, JUMPING_MEANS_IVALS) fv = proc.create_feature_vectors(fv) logger.debug(markers_processed) lda_out = proc.lda_apply(fv, cfy) markers = [fv.class_names[cls_idx] for cls_idx in fv.axes[0]] result = zip(markers, lda_out) for s, score in result: if markers_processed == 180: endresult.append(sorted(letter_prob.items(), key=lambda x: x[1])[-1][0]) letter_prob = {i : 0 for i in 'abcdefghijklmnopqrstuvwxyz123456789_'} markers_processed = 0 current_letter_idx += 1 current_letter = TRUE_LABELS[current_letter_idx].lower() for letter in s: letter_prob[letter] += score markers_processed += 1 logger.debug("".join([i[0] for i in sorted(letter_prob.items(), key=lambda x: x[1], reverse=True)]).replace(current_letter, " %s " % current_letter)) logger.debug(TRUE_LABELS) logger.debug("".join(endresult)) # calculate the current accuracy if len(endresult) > 0: acc = np.count_nonzero(np.array(endresult) == np.array(list(TRUE_LABELS.lower()[:len(endresult)]))) / len(endresult) print "Current accuracy:", acc * 100 if len(endresult) == len(TRUE_LABELS): break #logger.debug("Result: %s" % result) print 1000 * (time.time() - t0) acc = np.count_nonzero(np.array(endresult) == np.array(list(TRUE_LABELS.lower()[:len(endresult)]))) / len(endresult) print "Accuracy:", acc * 100 amp.stop()
str(events[1]))) #taking only middle elements between 100&200 train_markers1.append( (float(events[0]) + 200.0, str(events[1])) ) #again filling up the data this time taking two rows for same data point one is starting point other is end point markers1 = np.array(train_markers1) #markers1 is numpy array of train_markers1 markers_subject1_class_1 = [ (float(events[0]), str(events[1])) for events in markers1 if events[1] == '1\n' ] # separate line index starting and ending for class 1 and class 2 markers_subject1_class_2 = [ (float(events[0]), str(events[1])) for events in markers1 if events[1] == '2\n' ] #markers_subject1_class_1 and like wise other class correspondingly contains the 1 and 2 class marker data points #print "cerberus",markers_subject1_class_1 cnt1 = convert_mushu_data( signal_array, markers_subject1_class_1, 100, channels) #convert data into continuous form for 1st and second classs cnt2 = convert_mushu_data(signal_array, markers_subject1_class_2, 100, channels) #print cnt1,"cnt1 shape" #What type of marker data should be there should it contain start as well as end point or just start point is required md = {'class 1': ['1\n'], 'class 2': ['2\n']} epoch_subject1_class1 = segment_dat( cnt1, md, [0, 1000]) #epoch is a 3-d data set class*time*channel epoch_subject1_class2 = segment_dat(cnt2, md, [0, 1000]) #print "epoch data",epoch_subject1_class1 def bandpowers(segment): features = []