def handle(self, header, data):
LOG.info("received new image")
result = {} # default
header['status'] = 'success'
## preprocessing of input image
img = zc.raw2cv_image(data)
resize_ratio = 1
if max(img.shape) > config.IMAGE_MAX_WH:
resize_ratio = float(config.IMAGE_MAX_WH) / max(img.shape[0], img.shape[1])
img = cv2.resize(img, (0, 0), fx=resize_ratio, fy=resize_ratio, interpolation=cv2.INTER_AREA)
# get current state
t = time.time()
rtn_msg, objects_data = cc.process(img, resize_ratio)
print time.time() - t
# for measurement, when the sysmbolic representation has been got
if gabriel.Debug.TIME_MEASUREMENT:
result[gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
# the object detection result format is, for each line: [x1, y1, x2, y2, confidence, cls_idx]
objects = np.array(json.loads(objects_data))
img_object = zc.vis_detections(img, objects, config.LABELS)
result['image'] = b64encode(zc.cv_image2raw(img_object))
LOG.info("object detection result: %s" % objects)
# for measurement, when the sysmbolic representation has been got
if gabriel.Debug.TIME_MEASUREMENT:
header[gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
return json.dumps(result)
def handle(self, header, data):
# Receive data from control VM
LOG.info("received new image")
header['status'] = "nothing"
result = {}
# Preprocessing of input image
img = zc.raw2cv_image(data, gray_scale=True)
img_with_color = zc.raw2cv_image(data)
img_with_color = cv2.resize(img_with_color,
(config.IM_HEIGHT, config.IM_WIDTH))
b_channel, g_channel, r_channel = cv2.split(img_with_color)
alpha_channel = np.ones(b_channel.shape, dtype=b_channel.dtype) * 50
img_RGBA = cv2.merge((b_channel, g_channel, r_channel, alpha_channel))
zc.check_and_display('input',
img,
display_list,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
# Get image match
match = self.matcher.match(img)
# Send annotation data to mobile client
if match['status'] != 'success':
return json.dumps(result)
header['status'] = 'success'
img_RGBA = cv2.resize(img_RGBA, (320, 240))
result['annotated_img'] = b64encode(zc.cv_image2raw(img_RGBA))
if match['key'] is not None:
if match.get('annotated_text', None) is not None:
result['annotated_text'] = match['annotated_text']
if match.get('annotation_img', None) is not None:
annotation_img = match['annotation_img']
annotation_img = cv2.resize(annotation_img, (320, 240))
annotated_img = cv2.addWeighted(img_RGBA, 1, annotation_img, 1,
0)
result['annotated_img'] = b64encode(
zc.cv_image2raw(annotated_img))
else:
result['annotated_text'] = "No match found"
header[gabriel.Protocol_measurement.
JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
return json.dumps(result)
def handle(self, header, data):
LOG.info("received new image")
header['status'] = "nothing"
result = {} # default
## first image
if self.is_first_image:
self.is_first_image = False
instruction = self.task.get_instruction(np.array([]))
header['status'] = 'success'
if instruction.get('speech', None) is not None:
result['speech'] = instruction['speech']
display_verbal_guidance(result['speech'])
if config.PLAY_SOUND:
data = result['speech']
packet = struct.pack("!I%ds" % len(data), len(data), data)
self.sound_sock.sendall(packet)
if instruction.get('image', None) is not None:
feedback_image = b64encode(zc.cv_image2raw(instruction['image']))
result['image'] = feedback_image
zc.check_and_display('img_guidance', instruction['image'], display_list,
resize_max=config.DISPLAY_MAX_PIXEL, wait_time=config.DISPLAY_WAIT_TIME)
return json.dumps(result)
## preprocessing of input image
img = zc.raw2cv_image(data)
if header.get('holo_capture', None) is not None:
zc.check_and_display('holo', img, display_list, resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
return json.dumps(result)
zc.check_and_display('input', img, display_list, resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
## preprocessing of input image
resize_ratio = 1
if max(img.shape) > config.IMAGE_MAX_WH:
resize_ratio = float(config.IMAGE_MAX_WH) / max(img.shape[0], img.shape[1])
img = cv2.resize(img, (0, 0), fx=resize_ratio, fy=resize_ratio, interpolation=cv2.INTER_AREA)
zc.check_and_display('input', img, display_list, resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
## get current state
t = time.time()
rtn_msg, objects_data = cc.process(img, resize_ratio, display_list)
print time.time() - t
## for measurement, when the sysmbolic representation has been got
if gabriel.Debug.TIME_MEASUREMENT:
result[gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
# the object detection result format is, for each line: [x1, y1, x2, y2, confidence, cls_idx]
objects = np.array(json.loads(objects_data))
objects = reorder_objects(objects)
if "object" in display_list:
img_object = zc.vis_detections(img, objects, config.LABELS)
zc.check_and_display("object", img_object, display_list, resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
LOG.info("object detection result: %s" % objects)
## for measurement, when the sysmbolic representation has been got
if gabriel.Debug.TIME_MEASUREMENT:
header[gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
## get instruction based on state
instruction = self.task.get_instruction(objects)
if instruction['status'] != 'success':
return json.dumps(result)
header['status'] = 'success'
if instruction.get('speech', None) is not None:
result['speech'] = instruction['speech']
display_verbal_guidance(result['speech'])
if config.PLAY_SOUND:
data = result['speech']
packet = struct.pack("!I%ds" % len(data), len(data), data)
self.sound_sock.sendall(packet)
if instruction.get('image', None) is not None:
feedback_image = b64encode(zc.cv_image2raw(instruction['image']))
result['image'] = feedback_image
zc.check_and_display('img_guidance', instruction['image'], display_list,
resize_max=config.DISPLAY_MAX_PIXEL, wait_time=config.DISPLAY_WAIT_TIME)
if instruction.get('holo_object', None) is not None:
result['holo_object'] = instruction['holo_object']
if instruction.get('holo_location', None) is not None:
result['holo_location'] = instruction['holo_location']
return json.dumps(result)
def _generate_guidance(self, header, state_info_str, engine_id):
if config.RECOGNIZE_ONLY:
return json.dumps(result)
if self.is_first_frame: # do something special when the task begins
result, img_guidance = self.task.get_first_guidance()
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
self.is_first_frame = False
header['status'] = result.pop('status')
result.pop('animation', None)
return json.dumps(result)
header['status'] = "success"
result = {} # default
state_info = json.loads(state_info_str)
if not state_info['trust']:
header['status'] = "fail"
return json.dumps(result)
state = state_info['state']
if state == "None":
header['status'] = "nothing"
return json.dumps(result)
bitmap = np.array(json.loads(state))
## try to commit bitmap
state_change = False
if bm.bitmap_same(self.commited_bitmap, bitmap):
pass
else:
current_time = time.time()
if not bm.bitmap_same(self.temp_bitmap['bitmap'], bitmap):
self.temp_bitmap['bitmap'] = bitmap
self.temp_bitmap['first_time'] = current_time
self.temp_bitmap['count'] = 0
self.temp_bitmap['count'] += 1
if current_time - self.temp_bitmap[
'first_time'] > config.BM_WINDOW_MIN_TIME or self.temp_bitmap[
'count'] >= config.BM_WINDOW_MIN_COUNT:
self.commited_bitmap = self.temp_bitmap['bitmap']
state_change = True
bitmap = self.commited_bitmap
if 'lego_syn' in display_list and bitmap is not None:
img_syn = bm.bitmap2syn_img(bitmap)
zc.display_image('lego_syn',
img_syn,
wait_time=config.DISPLAY_WAIT_TIME,
resize_scale=50)
## now user has done something, provide some feedback
img_guidance = None
if state_change:
self.task.update_state(bitmap)
result, img_guidance = self.task.get_guidance()
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
header['status'] = result.pop('status')
result.pop('animation', None)
if img_guidance is not None:
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
return json.dumps(result)
def encode_images(img_animation):
img_animation_ret = []
for cv_img, duration in img_animation:
img_animation_ret.append(
(b64encode(zc.cv_image2raw(cv_img)), duration))
return img_animation_ret
def _handle_img(self, img):
if self.is_first_frame and not config.RECOGNIZE_ONLY: # do something special when the task begins
result, img_guidance = self.task.get_first_guidance()
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
result.pop('animation', None)
self.is_first_frame = False
return json.dumps(result)
result = {'status': "nothing"} # default
stretch_ratio = float(16) / 9 * img.shape[0] / img.shape[1]
if img.shape != (config.IMAGE_WIDTH, config.IMAGE_HEIGHT, 3):
img = cv2.resize(img, (config.IMAGE_WIDTH, config.IMAGE_HEIGHT),
interpolation=cv2.INTER_AREA)
## get bitmap for current image
zc.check_and_display('input',
img,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
rtn_msg, bitmap = lc.process(img, stretch_ratio, display_list)
if gabriel.Debug.TIME_MEASUREMENT:
result[gabriel.Protocol_measurement.
JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
if rtn_msg['status'] != 'success':
print rtn_msg['message']
if rtn_msg[
'message'] == "Not confident about reconstruction, maybe too much noise":
self.counter['not_confident'] += 1
return json.dumps(result)
self.counter['confident'] += 1
## try to commit bitmap
state_change = False
if bm.bitmap_same(self.commited_bitmap, bitmap):
pass
else:
current_time = time.time()
if not bm.bitmap_same(self.temp_bitmap['bitmap'], bitmap):
self.temp_bitmap['bitmap'] = bitmap
self.temp_bitmap['first_time'] = current_time
self.temp_bitmap['count'] = 0
self.counter['diff_from_prev'] += 1
else:
self.counter['same_as_prev'] += 1
self.temp_bitmap['count'] += 1
if current_time - self.temp_bitmap[
'first_time'] > config.BM_WINDOW_MIN_TIME or self.temp_bitmap[
'count'] >= config.BM_WINDOW_MIN_COUNT:
self.commited_bitmap = self.temp_bitmap['bitmap']
state_change = True
#print "\n\n\n\n\n%s\n\n\n\n\n" % self.counter
bitmap = self.commited_bitmap
if 'lego_syn' in display_list and bitmap is not None:
img_syn = bm.bitmap2syn_img(bitmap)
zc.display_image('lego_syn',
img_syn,
wait_time=config.DISPLAY_WAIT_TIME,
resize_scale=50)
if config.RECOGNIZE_ONLY:
return json.dumps(result)
## now user has done something, provide some feedback
img_guidance = None
if state_change:
self.task.update_state(bitmap)
sym_time = result[
gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME]
result, img_guidance = self.task.get_guidance()
result[gabriel.Protocol_measurement.
JSON_KEY_APP_SYMBOLIC_TIME] = sym_time
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
result.pop('animation', None)
if img_guidance is not None:
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
return json.dumps(result)