def _update_good_list(self, state, frame_id):
frame_state_history = self.state_history.get(frame_id, None)
if frame_state_history is None: # this could only happen if the best algorithm doesn't return in order (e.g. multiple instances of it)
return
now = time.time()
for en, en_detected_state in frame_state_history.iteritems():
if en == config.BEST_ENGINE:
continue
if bm.bitmap_same(en_detected_state, state): # the engine did well in this detection
if config.CHECK_ALGORITHM == 'last':
self.is_prev_good += 1
return
found = False
for idx, (check_state, add_time) in enumerate(self.good_list):
if bm.bitmap_same(check_state, state):
self.good_list[idx][1] = now
found = True
break
if not found:
self.good_list.append([state, now])
elif en_detected_state is not None: # the engine did wrong in this detection
if config.CHECK_ALGORITHM == 'last':
self.is_prev_good = 0
return
for idx, good_item in enumerate(self.good_list):
if bm.bitmap_same(good_item[0], en_detected_state):
del self.good_list[idx]
break
def _trust(self, state, engine_id, frame_id, keep_time = 10000):
## slower than the best one...
if frame_id <= self.prev_best_engine_frame_id:
return "too_slow"
## see if other engines have had results for the same frame
frame_state_history = self.state_history.get(frame_id, None)
if frame_state_history is None: # this is the first result
self.state_history[frame_id] = {engine_id : state, 'RETURNED' : False}
else:
self.state_history[frame_id][engine_id] = state
if self.state_history[frame_id]['RETURNED']: # slower than other engines
return "slow"
## the current result is not slow, let's see how good it is...
# clean the list
if engine_id == config.BEST_ENGINE:
return "success"
if config.CHECK_ALGORITHM == 'last':
if self.is_prev_good >= config.CHECK_LAST_TH:
self.state_history[frame_id]['RETURNED'] = True
return "success"
else:
return "no_trust"
now = time.time()
for idx, (check_state, add_time) in enumerate(self.good_list):
if now - add_time > keep_time:
del self.good_list[idx]
for check_state, add_time in self.good_list:
if bm.bitmap_same(state, check_state):
self.state_history[frame_id]['RETURNED'] = True
return "success"
return "no_trust"
def _handle_img(self, img):
if self.is_first_frame and not config.RECOGNIZE_ONLY: # do something special when the task begins
result, img_guidance = self.task.get_first_guidance()
zc.check_and_display('guidance', img_guidance, display_list, wait_time = config.DISPLAY_WAIT_TIME, resize_max = config.DISPLAY_MAX_PIXEL)
self.is_first_frame = False
result['state_index'] = 0 # first step
return json.dumps(result)
result = {'status' : "nothing"} # default
stretch_ratio = float(16) / 9 * img.shape[0] / img.shape[1]
if img.shape != (config.IMAGE_WIDTH, config.IMAGE_HEIGHT, 3):
img = cv2.resize(img, (config.IMAGE_WIDTH, config.IMAGE_HEIGHT), interpolation = cv2.INTER_AREA)
## get bitmap for current image
zc.check_and_display('input', img, display_list, wait_time = config.DISPLAY_WAIT_TIME, resize_max = config.DISPLAY_MAX_PIXEL)
rtn_msg, bitmap = lc.process(img, stretch_ratio, display_list)
if rtn_msg['status'] != 'success':
print rtn_msg['message']
if rtn_msg['message'] == "Not confident about reconstruction, maybe too much noise":
self.counter['not_confident'] += 1
return json.dumps(result)
self.counter['confident'] += 1
## try to commit bitmap
state_change = False
if bm.bitmap_same(self.commited_bitmap, bitmap):
pass
else:
current_time = time.time()
if not bm.bitmap_same(self.temp_bitmap['bitmap'], bitmap):
self.temp_bitmap['bitmap'] = bitmap
self.temp_bitmap['first_time'] = current_time
self.temp_bitmap['count'] = 0
self.counter['diff_from_prev'] += 1
else:
self.counter['same_as_prev'] += 1
self.temp_bitmap['count'] += 1
if current_time - self.temp_bitmap['first_time'] > config.BM_WINDOW_MIN_TIME or self.temp_bitmap['count'] >= config.BM_WINDOW_MIN_COUNT:
self.commited_bitmap = self.temp_bitmap['bitmap']
state_change = True
#print "\n\n\n\n\n%s\n\n\n\n\n" % self.counter
bitmap = self.commited_bitmap
if 'lego_syn' in display_list and bitmap is not None:
img_syn = bm.bitmap2syn_img(bitmap)
zc.display_image('lego_syn', img_syn, wait_time = config.DISPLAY_WAIT_TIME, resize_scale = 50)
if config.RECOGNIZE_ONLY:
return json.dumps(result)
## now user has done something, provide some feedback
img_guidance = None
if state_change:
self.task.update_state(bitmap)
result, img_guidance = self.task.get_guidance()
if self.task.is_final_state():
step_idx = len(self.task.states)
else:
# get current step
step_idx = self.task.state2idx(self.task.current_state)
# make sure step index is always -1 in case of error
# also, differentiate from the default initial step (which we assign a step index 0)
# from the internal step index obtained from the task (which also begins at 0) by
# shifting the index by 1:
step_idx = -1 if step_idx < 0 else step_idx + 1
result['state_index'] = step_idx
if img_guidance is not None:
zc.check_and_display('guidance', img_guidance, display_list, wait_time = config.DISPLAY_WAIT_TIME, resize_max = config.DISPLAY_MAX_PIXEL)
return json.dumps(result)
def is_final_state(self):
return bm.bitmap_same(self.current_state, self.get_state(-1))
def state2idx(self, state):
for idx, s in enumerate(self.states):
if bm.bitmap_same(state, s):
return idx
return -1
def _generate_guidance(self, header, state_info_str, engine_id):
if config.RECOGNIZE_ONLY:
return json.dumps(result)
if self.is_first_frame: # do something special when the task begins
result, img_guidance = self.task.get_first_guidance()
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
self.is_first_frame = False
header['status'] = result.pop('status')
result.pop('animation', None)
return json.dumps(result)
header['status'] = "success"
result = {} # default
state_info = json.loads(state_info_str)
if not state_info['trust']:
header['status'] = "fail"
return json.dumps(result)
state = state_info['state']
if state == "None":
header['status'] = "nothing"
return json.dumps(result)
bitmap = np.array(json.loads(state))
## try to commit bitmap
state_change = False
if bm.bitmap_same(self.commited_bitmap, bitmap):
pass
else:
current_time = time.time()
if not bm.bitmap_same(self.temp_bitmap['bitmap'], bitmap):
self.temp_bitmap['bitmap'] = bitmap
self.temp_bitmap['first_time'] = current_time
self.temp_bitmap['count'] = 0
self.temp_bitmap['count'] += 1
if current_time - self.temp_bitmap[
'first_time'] > config.BM_WINDOW_MIN_TIME or self.temp_bitmap[
'count'] >= config.BM_WINDOW_MIN_COUNT:
self.commited_bitmap = self.temp_bitmap['bitmap']
state_change = True
bitmap = self.commited_bitmap
if 'lego_syn' in display_list and bitmap is not None:
img_syn = bm.bitmap2syn_img(bitmap)
zc.display_image('lego_syn',
img_syn,
wait_time=config.DISPLAY_WAIT_TIME,
resize_scale=50)
## now user has done something, provide some feedback
img_guidance = None
if state_change:
self.task.update_state(bitmap)
result, img_guidance = self.task.get_guidance()
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
header['status'] = result.pop('status')
result.pop('animation', None)
if img_guidance is not None:
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
return json.dumps(result)
def _handle_img(self, img):
if self.is_first_frame and not config.RECOGNIZE_ONLY: # do something special when the task begins
result, img_guidance = self.task.get_first_guidance()
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
result.pop('animation', None)
self.is_first_frame = False
return json.dumps(result)
result = {'status': "nothing"} # default
stretch_ratio = float(16) / 9 * img.shape[0] / img.shape[1]
if img.shape != (config.IMAGE_WIDTH, config.IMAGE_HEIGHT, 3):
img = cv2.resize(img, (config.IMAGE_WIDTH, config.IMAGE_HEIGHT),
interpolation=cv2.INTER_AREA)
## get bitmap for current image
zc.check_and_display('input',
img,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
rtn_msg, bitmap = lc.process(img, stretch_ratio, display_list)
if gabriel.Debug.TIME_MEASUREMENT:
result[gabriel.Protocol_measurement.
JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
if rtn_msg['status'] != 'success':
print rtn_msg['message']
if rtn_msg[
'message'] == "Not confident about reconstruction, maybe too much noise":
self.counter['not_confident'] += 1
return json.dumps(result)
self.counter['confident'] += 1
## try to commit bitmap
state_change = False
if bm.bitmap_same(self.commited_bitmap, bitmap):
pass
else:
current_time = time.time()
if not bm.bitmap_same(self.temp_bitmap['bitmap'], bitmap):
self.temp_bitmap['bitmap'] = bitmap
self.temp_bitmap['first_time'] = current_time
self.temp_bitmap['count'] = 0
self.counter['diff_from_prev'] += 1
else:
self.counter['same_as_prev'] += 1
self.temp_bitmap['count'] += 1
if current_time - self.temp_bitmap[
'first_time'] > config.BM_WINDOW_MIN_TIME or self.temp_bitmap[
'count'] >= config.BM_WINDOW_MIN_COUNT:
self.commited_bitmap = self.temp_bitmap['bitmap']
state_change = True
#print "\n\n\n\n\n%s\n\n\n\n\n" % self.counter
bitmap = self.commited_bitmap
if 'lego_syn' in display_list and bitmap is not None:
img_syn = bm.bitmap2syn_img(bitmap)
zc.display_image('lego_syn',
img_syn,
wait_time=config.DISPLAY_WAIT_TIME,
resize_scale=50)
if config.RECOGNIZE_ONLY:
return json.dumps(result)
## now user has done something, provide some feedback
img_guidance = None
if state_change:
self.task.update_state(bitmap)
sym_time = result[
gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME]
result, img_guidance = self.task.get_guidance()
result[gabriel.Protocol_measurement.
JSON_KEY_APP_SYMBOLIC_TIME] = sym_time
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
result.pop('animation', None)
if img_guidance is not None:
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
return json.dumps(result)