def sample_decoding(decoder):
"""
Decoding example
"""
# load trigger definitions for labeling
labels = decoder.get_label_names()
tm_watchdog = qc.Timer(autoreset=True)
tm_cls = qc.Timer()
while True:
praw = decoder.get_prob_unread()
psmooth = decoder.get_prob_smooth()
if praw is None:
# watch dog
if tm_cls.sec() > 5:
logger.warning('No classification was done in the last 5 seconds. Are you receiving data streams?')
tm_cls.reset()
tm_watchdog.sleep_atleast(0.001)
continue
txt = '[%8.1f msec]' % (tm_cls.msec())
for i, label in enumerate(labels):
txt += ' %s %.3f (raw %.3f)' % (label, psmooth[i], praw[i])
maxi = qc.get_index_max(psmooth)
txt += ' %s' % labels[maxi]
print(txt)
tm_cls.reset()
def classify(self,
decoder,
true_label,
title_text,
bar_dirs,
state='start',
prob_history=None):
"""
Run a single trial
"""
true_label_index = bar_dirs.index(true_label)
self.tm_trigger.reset()
if self.bar_bias is not None:
bias_idx = bar_dirs.index(self.bar_bias[0])
if self.logf is not None:
self.logf.write('True label: %s\n' % true_label)
tm_classify = qc.Timer(autoreset=True)
self.stimo_timer = qc.Timer()
while True:
self.tm_display.sleep_atleast(self.refresh_delay)
self.tm_display.reset()
if state == 'start' and self.tm_trigger.sec(
) > self.cfg.TIMINGS['INIT']:
state = 'gap_s'
if self.cfg.TRIALS_PAUSE:
self.viz.put_text('Press any key')
self.viz.update()
key = cv2.waitKeyEx()
if key == KEYS['esc'] or not self.protocol_state.value:
return
self.viz.fill()
self.tm_trigger.reset()
self.trigger.signal(self.tdef.INIT)
elif state == 'gap_s':
if self.cfg.TIMINGS['GAP'] > 0:
self.viz.put_text(title_text)
state = 'gap'
self.tm_trigger.reset()
elif state == 'gap' and self.tm_trigger.sec(
) > self.cfg.TIMINGS['GAP']:
state = 'cue'
self.viz.fill()
self.viz.draw_cue()
self.viz.glass_draw_cue()
self.trigger.signal(self.tdef.CUE)
self.tm_trigger.reset()
elif state == 'cue' and self.tm_trigger.sec(
) > self.cfg.TIMINGS['READY']:
state = 'dir_r'
if self.cfg.SHOW_CUE is True:
if self.cfg.FEEDBACK_TYPE == 'BAR':
self.viz.move(true_label,
100,
overlay=False,
barcolor='G')
elif self.cfg.FEEDBACK_TYPE == 'BODY':
self.viz.put_text(DIRS[true_label], 'R')
if true_label == 'L': # left
self.trigger.signal(self.tdef.LEFT_READY)
elif true_label == 'R': # right
self.trigger.signal(self.tdef.RIGHT_READY)
elif true_label == 'U': # up
self.trigger.signal(self.tdef.UP_READY)
elif true_label == 'D': # down
self.trigger.signal(self.tdef.DOWN_READY)
elif true_label == 'B': # both hands
self.trigger.signal(self.tdef.BOTH_READY)
else:
raise RuntimeError('Unknown direction %s' % true_label)
self.tm_trigger.reset()
'''
if self.cfg.FEEDBACK_TYPE == 'BODY':
self.viz.set_pc_feedback(False)
self.viz.move(true_label, 100, overlay=False, barcolor='G')
if self.cfg.FEEDBACK_TYPE == 'BODY':
self.viz.set_pc_feedback(True)
if self.cfg.SHOW_CUE is True:
self.viz.put_text(dirs[true_label], 'R')
if true_label == 'L': # left
self.trigger.signal(self.tdef.LEFREADY)
elif true_label == 'R': # right
self.trigger.signal(self.tdef.RIGHT_READY)
elif true_label == 'U': # up
self.trigger.signal(self.tdef.UP_READY)
elif true_label == 'D': # down
self.trigger.signal(self.tdef.DOWN_READY)
elif true_label == 'B': # both hands
self.trigger.signal(self.tdef.BOTH_READY)
else:
raise RuntimeError('Unknown direction %s' % true_label)
self.tm_trigger.reset()
'''
elif state == 'dir_r' and self.tm_trigger.sec(
) > self.cfg.TIMINGS['DIR_CUE']:
self.viz.fill()
self.viz.draw_cue()
self.viz.glass_draw_cue()
state = 'dir'
# initialize bar scores
bar_label = bar_dirs[0]
bar_score = 0
probs = [1.0 / len(bar_dirs)] * len(bar_dirs)
self.viz.move(bar_label, bar_score, overlay=False)
probs_acc = np.zeros(len(probs))
if true_label == 'L': # left
self.trigger.signal(self.tdef.LEFT_GO)
elif true_label == 'R': # right
self.trigger.signal(self.tdef.RIGHT_GO)
elif true_label == 'U': # up
self.trigger.signal(self.tdef.UP_GO)
elif true_label == 'D': # down
self.trigger.signal(self.tdef.DOWN_GO)
elif true_label == 'B': # both
self.trigger.signal(self.tdef.BOTH_GO)
else:
raise RuntimeError('Unknown truedirection %s' % true_label)
self.tm_watchdog.reset()
self.tm_trigger.reset()
elif state == 'dir':
if self.tm_trigger.sec() > self.cfg.TIMINGS['CLASSIFY'] or (
self.premature_end and bar_score >= 100):
if not hasattr(
self.cfg,
'SHOW_RESULT') or self.cfg.SHOW_RESULT is True:
# show classfication result
if self.cfg.WITH_STIMO is True:
if self.cfg.STIMO_FULLGAIT_CYCLE is not None and bar_label == 'U':
res_color = 'G'
elif self.cfg.TRIALS_RETRY is False or bar_label == true_label:
res_color = 'G'
else:
res_color = 'Y'
else:
res_color = 'Y'
if self.cfg.FEEDBACK_TYPE == 'BODY':
self.viz.move(bar_label,
bar_score,
overlay=False,
barcolor=res_color,
caption=DIRS[bar_label],
caption_color=res_color)
else:
self.viz.move(bar_label,
100,
overlay=False,
barcolor=res_color)
else:
res_color = 'Y'
if self.cfg.FEEDBACK_TYPE == 'BODY':
self.viz.move(bar_label,
bar_score,
overlay=False,
barcolor=res_color,
caption='TRIAL END',
caption_color=res_color)
else:
self.viz.move(bar_label,
0,
overlay=False,
barcolor=res_color)
self.trigger.signal(self.tdef.FEEDBACK)
# STIMO
if self.cfg.WITH_STIMO is True and self.cfg.STIMO_CONTINUOUS is False:
if self.cfg.STIMO_FULLGAIT_CYCLE is not None:
if bar_label == 'U':
self.ser.write(
self.cfg.STIMO_FULLGAIT_PATTERN[0])
logger.info('STIMO: Sent 1')
time.sleep(self.cfg.STIMO_FULLGAIT_CYCLE)
self.ser.write(
self.cfg.STIMO_FULLGAIT_PATTERN[1])
logger.info('STIMO: Sent 2')
time.sleep(self.cfg.STIMO_FULLGAIT_CYCLE)
elif self.cfg.TRIALS_RETRY is False or bar_label == true_label:
if bar_label == 'L':
self.ser.write(b'1')
logger.info('STIMO: Sent 1')
elif bar_label == 'R':
self.ser.write(b'2')
logger.info('STIMO: Sent 2')
if self.cfg.DEBUG_PROBS:
msg = 'DEBUG: Accumulated probabilities = %s' % qc.list2string(
probs_acc, '%.3f')
logger.info(msg)
if self.logf is not None:
self.logf.write(msg + '\n')
if self.logf is not None:
self.logf.write('%s detected as %s (%d)\n\n' %
(true_label, bar_label, bar_score))
self.logf.flush()
# end of trial
state = 'feedback'
self.tm_trigger.reset()
else:
# classify
probs_new = decoder.get_prob_smooth_unread()
if probs_new is None:
if self.tm_watchdog.sec() > 3:
logger.warning(
'No classification being done. Are you receiving data streams?'
)
self.tm_watchdog.reset()
else:
self.tm_watchdog.reset()
if prob_history is not None:
prob_history[true_label].append(
probs_new[true_label_index])
probs_acc += np.array(probs_new)
'''
New decoder: already smoothed by the decoder so bias after.
'''
probs = list(probs_new)
if self.bar_bias is not None:
probs[bias_idx] += self.bar_bias[1]
newsum = sum(probs)
probs = [p / newsum for p in probs]
'''
# Method 2: bias and smoothen
if self.bar_bias is not None:
# print('BEFORE: %.3f %.3f'% (probs_new[0], probs_new[1]) )
probs_new[bias_idx] += self.bar_bias[1]
newsum = sum(probs_new)
probs_new = [p / newsum for p in probs_new]
# print('AFTER: %.3f %.3f'% (probs_new[0], probs_new[1]) )
for i in range(len(probs_new)):
probs[i] = probs[i] * self.alpha_old + probs_new[i] * self.alpha_new
'''
''' Original method
# Method 1: smoothen and bias
for i in range( len(probs_new) ):
probs[i] = probs[i] * self.alpha_old + probs_new[i] * self.alpha_new
# bias bar
if self.bar_bias is not None:
probs[bias_idx] += self.bar_bias[1]
newsum = sum(probs)
probs = [p/newsum for p in probs]
'''
# determine the direction
# TODO: np.argmax(probs)
max_pidx = qc.get_index_max(probs)
max_label = bar_dirs[max_pidx]
if self.cfg.POSITIVE_FEEDBACK is False or \
(self.cfg.POSITIVE_FEEDBACK and true_label == max_label):
dx = probs[max_pidx]
if max_label == 'R':
dx *= self.bar_step_right
elif max_label == 'L':
dx *= self.bar_step_left
elif max_label == 'U':
dx *= self.bar_step_up
elif max_label == 'D':
dx *= self.bar_step_down
elif max_label == 'B':
dx *= self.bar_step_both
else:
logger.debug('Direction %s using bar step %d' %
(max_label, self.bar_step_left))
dx *= self.bar_step_left
# slow start
selected = self.cfg.BAR_SLOW_START['selected']
if self.cfg.BAR_SLOW_START[
selected] and self.tm_trigger.sec(
) < self.cfg.BAR_SLOW_START[selected]:
dx *= self.tm_trigger.sec(
) / self.cfg.BAR_SLOW_START[selected][0]
# add likelihoods
if max_label == bar_label:
bar_score += dx
else:
bar_score -= dx
# change of direction
if bar_score < 0:
bar_score = -bar_score
bar_label = max_label
bar_score = int(bar_score)
if bar_score > 100:
bar_score = 100
if self.cfg.FEEDBACK_TYPE == 'BODY':
if self.cfg.SHOW_CUE:
self.viz.move(bar_label,
bar_score,
overlay=False,
caption=DIRS[true_label],
caption_color='G')
else:
self.viz.move(bar_label,
bar_score,
overlay=False)
else:
self.viz.move(bar_label,
bar_score,
overlay=False)
# send the confidence value continuously
if self.cfg.WITH_STIMO and self.cfg.STIMO_CONTINUOUS:
if self.stimo_timer.sec(
) >= self.cfg.STIMO_COOLOFF:
if bar_label == 'U':
stimo_code = bar_score
else:
stimo_code = 0
self.ser.write(bytes([stimo_code]))
logger.info('Sent STIMO code %d' %
stimo_code)
self.stimo_timer.reset()
if self.cfg.DEBUG_PROBS:
if self.bar_bias is not None:
biastxt = '[Bias=%s%.3f] ' % (
self.bar_bias[0], self.bar_bias[1])
else:
biastxt = ''
msg = '%s%s prob %s acc %s bar %s%d (%.1f ms)' % \
(biastxt, bar_dirs, qc.list2string(probs_new, '%.2f'), qc.list2string(probs, '%.2f'),
bar_label, bar_score, tm_classify.msec())
logger.info(msg)
if self.logf is not None:
self.logf.write(msg + '\n')
elif state == 'feedback' and self.tm_trigger.sec(
) > self.cfg.TIMINGS['FEEDBACK']:
self.trigger.signal(self.tdef.BLANK)
if self.cfg.FEEDBACK_TYPE == 'BODY':
state = 'return'
self.tm_trigger.reset()
else:
state = 'gap_s'
self.viz.fill()
self.viz.update()
return bar_label
elif state == 'return':
self.viz.set_glass_feedback(False)
if self.cfg.WITH_STIMO:
self.viz.move(bar_label,
bar_score,
overlay=False,
barcolor='B')
else:
self.viz.move(bar_label,
bar_score,
overlay=False,
barcolor='Y')
self.viz.set_glass_feedback(True)
bar_score -= 5
if bar_score <= 0:
state = 'gap_s'
self.viz.fill()
self.viz.update()
return bar_label
self.viz.update()
key = cv2.waitKeyEx(1)
if key == KEYS['esc'] or not self.protocol_state.value:
return
elif key == KEYS['space']:
dx = 0
bar_score = 0
probs = [1.0 / len(bar_dirs)] * len(bar_dirs)
self.viz.move(bar_dirs[0], bar_score, overlay=False)
self.viz.update()
logger.info('probs and dx reset.')
self.tm_trigger.reset()
elif key in ARROW_KEYS and ARROW_KEYS[key] in bar_dirs:
# change bias on the fly
if self.bar_bias is None:
self.bar_bias = [ARROW_KEYS[key], BIAS_INCREMENT]
else:
if ARROW_KEYS[key] == self.bar_bias[0]:
self.bar_bias[1] += BIAS_INCREMENT
elif self.bar_bias[1] >= BIAS_INCREMENT:
self.bar_bias[1] -= BIAS_INCREMENT
else:
self.bar_bias = [ARROW_KEYS[key], BIAS_INCREMENT]
if self.bar_bias[1] == 0:
self.bar_bias = None
else:
bias_idx = bar_dirs.index(self.bar_bias[0])
def classify(self,
decoder,
true_label,
title_text,
bar_dirs,
state='start'):
"""
Run a single trial
"""
self.tm_trigger.reset()
if self.bar_bias is not None:
bias_idx = bar_dirs.index(self.bar_bias[0])
if self.logf is not None:
self.logf.write('True label: %s\n' % true_label)
tm_classify = qc.Timer()
while True:
self.tm_display.sleep_atleast(self.refresh_delay)
self.tm_display.reset()
if state == 'start' and self.tm_trigger.sec() > self.cfg.T_INIT:
state = 'gap_s'
self.bar.fill()
self.tm_trigger.reset()
self.trigger.signal(self.tdef.INIT)
elif state == 'gap_s':
self.bar.put_text(title_text)
state = 'gap'
self.tm_trigger.reset()
elif state == 'gap' and self.tm_trigger.sec() > self.cfg.T_GAP:
state = 'cue'
self.bar.fill()
self.bar.draw_cue()
self.bar.glass_draw_cue()
self.trigger.signal(self.tdef.CUE)
self.tm_trigger.reset()
elif state == 'cue' and self.tm_trigger.sec() > self.cfg.T_READY:
state = 'dir_r'
if self.cfg.FEEDBACK_TYPE == 'BODY':
self.bar.set_pc_feedback(False)
self.bar.move(true_label, 100, overlay=False, barcolor='G')
if self.cfg.FEEDBACK_TYPE == 'BODY':
self.bar.set_pc_feedback(True)
if self.cfg.SHOW_CUE is True:
self.bar.put_text(dirs[true_label], 'R')
if true_label == 'L': # left
self.trigger.signal(self.tdef.LEFT_READY)
elif true_label == 'R': # right
self.trigger.signal(self.tdef.RIGHT_READY)
elif true_label == 'U': # up
self.trigger.signal(self.tdef.UP_READY)
elif true_label == 'D': # down
self.trigger.signal(self.tdef.DOWN_READY)
elif true_label == 'B': # both hands
self.trigger.signal(self.tdef.BOTH_READY)
else:
raise RuntimeError('Unknown direction %s' % true_label)
self.tm_trigger.reset()
##################################################################
##################################################################
#qc.print_c('Executing Rex action %s' % 'N', 'W')
#os.system('%s/Rex/RexControlSimple.exe %s %s' % (pycnbi.ROOT, 'COM3', 'N'))
##################################################################
##################################################################
elif state == 'dir_r' and self.tm_trigger.sec(
) > self.cfg.T_DIR_CUE:
self.bar.fill()
self.bar.draw_cue()
self.bar.glass_draw_cue()
state = 'dir'
# initialize bar scores
bar_label = bar_dirs[0]
bar_score = 0
probs = [1.0 / len(bar_dirs)] * len(bar_dirs)
self.bar.move(bar_label, bar_score, overlay=False)
probs_acc = np.zeros(len(probs))
if true_label == 'L': # left
self.trigger.signal(self.tdef.LEFT_GO)
elif true_label == 'R': # right
self.trigger.signal(self.tdef.RIGHT_GO)
elif true_label == 'U': # up
self.trigger.signal(self.tdef.UP_GO)
elif true_label == 'D': # down
self.trigger.signal(self.tdef.DOWN_GO)
elif true_label == 'B': # both
self.trigger.signal(self.tdef.BOTH_GO)
else:
raise RuntimeError('Unknown truedirection %s' % true_label)
self.tm_watchdog.reset()
self.tm_trigger.reset()
elif state == 'dir':
if self.tm_trigger.sec() > self.cfg.T_CLASSIFY or (
self.premature_end and bar_score >= 100):
if not hasattr(
self.cfg,
'SHOW_RESULT') or self.cfg.SHOW_RESULT is True:
# show classfication result
if self.cfg.FEEDBACK_TYPE == 'BODY':
self.bar.move(bar_label,
bar_score,
overlay=False,
barcolor='Y',
caption=dirs[bar_label],
caption_color='Y')
else:
self.bar.move(bar_label,
100,
overlay=False,
barcolor='Y')
else:
self.bar.move(bar_label,
bar_score,
overlay=False,
barcolor='Y',
caption='TRIAL END',
caption_color='Y')
########################## TEST WITH BAR #############################
'''
if self.cfg.FEEDBACK_TYPE == 'BODY':
self.bar.move(bar_label, bar_score, overlay=False, barcolor='Y', caption='TRIAL END', caption_color='Y')
else:
self.bar.move(bar_label, 0, overlay=False, barcolor='Y')
'''
self.trigger.signal(self.tdef.FEEDBACK)
probs_acc /= sum(probs_acc)
if self.cfg.DEBUG_PROBS:
msg = 'DEBUG: Accumulated probabilities = %s' % qc.list2string(
probs_acc, '%.3f')
logger.info(msg)
if self.logf is not None:
self.logf.write(msg + '\n')
if self.logf is not None:
self.logf.write('%s detected as %s (%d)\n\n' %
(true_label, bar_label, bar_score))
self.logf.flush()
# end of trial
state = 'feedback'
self.tm_trigger.reset()
else:
# classify
probs_new = decoder.get_prob_unread()
if probs_new is None:
if self.tm_watchdog.sec() > 3:
logger.warning(
'No classification being done. Are you receiving data streams?'
)
self.tm_watchdog.reset()
else:
self.tm_watchdog.reset()
# bias and accumulate
if self.bar_bias is not None:
probs_new[bias_idx] += self.bar_bias[1]
newsum = sum(probs_new)
probs_new = [p / newsum for p in probs_new]
probs_acc += np.array(probs_new)
for i in range(len(probs_new)):
probs[i] = probs[i] * self.alpha1 + probs_new[
i] * self.alpha2
''' Original: accumulate and bias
# accumulate probs
for i in range( len(probs_new) ):
probs[i]= probs[i] * self.alpha1 + probs_new[i] * self.alpha2
# bias bar
if self.bar_bias is not None:
probs[bias_idx] += self.bar_bias[1]
newsum= sum(probs)
probs= [p/newsum for p in probs]
'''
# determine the direction
max_pidx = qc.get_index_max(probs)
max_label = bar_dirs[max_pidx]
if self.cfg.POSITIVE_FEEDBACK is False or \
(self.cfg.POSITIVE_FEEDBACK and true_label == max_label):
dx = probs[max_pidx]
if max_label == 'R':
dx *= self.bar_step_right
elif max_label == 'L':
dx *= self.bar_step_left
elif max_label == 'U':
dx *= self.bar_step_up
elif max_label == 'D':
dx *= self.bar_step_down
elif max_label == 'B':
dx *= self.bar_step_both
else:
logger.debug('Direction %s using bar step %d' %
(max_label, self.bar_step_left))
dx *= self.bar_step_left
################################################
################################################
# slower in the beginning
#if self.tm_trigger.sec() < 2.0:
# dx *= self.tm_trigger.sec() * 0.5
################################################
################################################
# add likelihoods
if max_label == bar_label:
bar_score += dx
else:
bar_score -= dx
# change of direction
if bar_score < 0:
bar_score = -bar_score
bar_label = max_label
bar_score = int(bar_score)
if bar_score > 100:
bar_score = 100
if self.cfg.FEEDBACK_TYPE == 'BODY':
if self.cfg.SHOW_CUE:
self.bar.move(bar_label,
bar_score,
overlay=False,
caption=dirs[true_label],
caption_color='G')
else:
self.bar.move(bar_label,
bar_score,
overlay=False)
else:
self.bar.move(bar_label,
bar_score,
overlay=False)
if self.cfg.DEBUG_PROBS:
if self.bar_bias is not None:
biastxt = '[Bias=%s%.3f] ' % (
self.bar_bias[0], self.bar_bias[1])
else:
biastxt = ''
msg = '%s%s raw %s acc %s bar %s%d (%.1f ms)' % \
(biastxt, bar_dirs, qc.list2string(probs_new, '%.2f'), qc.list2string(probs, '%.2f'),
bar_label, bar_score, tm_classify.msec())
logger.info(msg)
if self.logf is not None:
self.logf.write(msg + '\n')
tm_classify.reset()
elif state == 'feedback' and self.tm_trigger.sec(
) > self.cfg.T_FEEDBACK:
self.trigger.signal(self.tdef.BLANK)
if self.cfg.FEEDBACK_TYPE == 'BODY':
state = 'return'
self.tm_trigger.reset()
else:
state = 'gap_s'
self.bar.fill()
self.bar.update()
return bar_label
elif state == 'return':
self.bar.set_glass_feedback(False)
self.bar.move(bar_label,
bar_score,
overlay=False,
barcolor='Y')
self.bar.set_glass_feedback(True)
bar_score -= 5
if bar_score <= 0:
state = 'gap_s'
self.bar.fill()
self.bar.update()
return bar_label
self.bar.update()
key = 0xFF & cv2.waitKey(1)
if key == keys['esc']:
return None
if key == keys['space']:
dx = 0
bar_score = 0
probs = [1.0 / len(bar_dirs)] * len(bar_dirs)
self.bar.move(bar_dirs[0], bar_score, overlay=False)
self.bar.update()
logger.info('probs and dx reset.')