def handle(self, header, data):
captured_image_size = header['image_size']
captured_image = data[:captured_image_size]
drawn_image = data[captured_image_size:]
img = zc.raw2cv_image(captured_image, gray_scale=True)
img = cv2.resize(img, (config.IM_HEIGHT, config.IM_WIDTH))
hist = cv2.calcHist([img], [0], None, [256], [0, 256]) # Grayscale
kp, des = self.surf.detectAndCompute(img, None)
img_annotation = zc.raw2cv_image(drawn_image)
img_annotation = cv2.resize(img_annotation,
(config.IM_HEIGHT, config.IM_WIDTH))
# Store the keypoints, descriptors, hist, image name, and cv image in database (memory)
new_name = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10)) # TODO: this needs to be more meaningful
self.table.add_annotation(new_name,
kp,
des,
hist,
img,
annotation_img=img_annotation)
# Store the annotation image in persistent storage
with open(os.path.join('db', '%s.jpeg' % new_name), 'w') as f:
f.write(captured_image)
with open(os.path.join('db', '%s.png' % new_name), 'w') as f:
f.write(drawn_image)
return json.dumps({})
def handle(self, header, data):
LOG.info("received new image")
result = {} # default
header['status'] = 'success'
## preprocessing of input image
img = zc.raw2cv_image(data)
resize_ratio = 1
if max(img.shape) > config.IMAGE_MAX_WH:
resize_ratio = float(config.IMAGE_MAX_WH) / max(img.shape[0], img.shape[1])
img = cv2.resize(img, (0, 0), fx=resize_ratio, fy=resize_ratio, interpolation=cv2.INTER_AREA)
# get current state
t = time.time()
rtn_msg, objects_data = cc.process(img, resize_ratio)
print time.time() - t
# for measurement, when the sysmbolic representation has been got
if gabriel.Debug.TIME_MEASUREMENT:
result[gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
# the object detection result format is, for each line: [x1, y1, x2, y2, confidence, cls_idx]
objects = np.array(json.loads(objects_data))
img_object = zc.vis_detections(img, objects, config.LABELS)
result['image'] = b64encode(zc.cv_image2raw(img_object))
LOG.info("object detection result: %s" % objects)
# for measurement, when the sysmbolic representation has been got
if gabriel.Debug.TIME_MEASUREMENT:
header[gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
return json.dumps(result)
def handle(self, header, data):
# Receive data from control VM
LOG.info("received new image")
header['status'] = "nothing"
result = {}
# Preprocessing of input image
img = zc.raw2cv_image(data, gray_scale=True)
img_with_color = zc.raw2cv_image(data)
img_with_color = cv2.resize(img_with_color,
(config.IM_HEIGHT, config.IM_WIDTH))
b_channel, g_channel, r_channel = cv2.split(img_with_color)
alpha_channel = np.ones(b_channel.shape, dtype=b_channel.dtype) * 50
img_RGBA = cv2.merge((b_channel, g_channel, r_channel, alpha_channel))
zc.check_and_display('input',
img,
display_list,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
# Get image match
match = self.matcher.match(img)
# Send annotation data to mobile client
if match['status'] != 'success':
return json.dumps(result)
header['status'] = 'success'
img_RGBA = cv2.resize(img_RGBA, (320, 240))
result['annotated_img'] = b64encode(zc.cv_image2raw(img_RGBA))
if match['key'] is not None:
if match.get('annotated_text', None) is not None:
result['annotated_text'] = match['annotated_text']
if match.get('annotation_img', None) is not None:
annotation_img = match['annotation_img']
annotation_img = cv2.resize(annotation_img, (320, 240))
annotated_img = cv2.addWeighted(img_RGBA, 1, annotation_img, 1,
0)
result['annotated_img'] = b64encode(
zc.cv_image2raw(annotated_img))
else:
result['annotated_text'] = "No match found"
header[gabriel.Protocol_measurement.
JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
return json.dumps(result)
def _handle_input_data(self):
img_size = struct.unpack("!I", self._recv_all(4))[0]
img = self._recv_all(img_size)
cv_img = zc.raw2cv_image(img)
return_data = self._handle_img(cv_img)
packet = struct.pack("!I%ds" % len(return_data), len(return_data), return_data)
self.request.sendall(packet)
self.wfile.flush()
def handle(self, header, data):
#LOG.info("received new image")
result = {'status': "nothing"} # default
## preprocessing of input image
img = zc.raw2cv_image(data)
if max(img.shape) > config.IMAGE_MAX_WH:
resize_ratio = float(config.IMAGE_MAX_WH) / max(
img.shape[0], img.shape[1])
img = cv2.resize(img, (0, 0),
fx=resize_ratio,
fy=resize_ratio,
interpolation=cv2.INTER_AREA)
zc.check_and_display('input',
img,
display_list,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
## process the image
rtn_msg, objects = pc.process(img, display_list)
## for measurement, when the sysmbolic representation has been got
if gabriel.Debug.TIME_MEASUREMENT:
result[gabriel.Protocol_measurement.
JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
if rtn_msg['status'] == 'success':
result['status'] = 'success'
cue, CO_balls, pocket = objects
result['speech'] = pc.get_guidance(img, cue, CO_balls, pocket,
display_list)
header['status'] = result.pop('status')
header[gabriel.Protocol_measurement.
JSON_KEY_APP_SYMBOLIC_TIME] = result.pop(
gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME, -1)
return json.dumps(result)
def _handle_input_data(self):
if self.engine_id == "LEGO_SLOW":
import lego_cv as lc
else:
import lego_cv_fast as lc
# receive data from control VM
header_size = struct.unpack("!I", self._recv_all(4))[0]
data_size = struct.unpack("!I", self._recv_all(4))[0]
header_str = self._recv_all(header_size)
image_data = self._recv_all(data_size)
#header = json.loads(header_str)
## symbolic representation extraction
img = zc.raw2cv_image(image_data)
stretch_ratio = float(16) / 9 * img.shape[0] / img.shape[1]
if img.shape != (config.IMAGE_WIDTH, config.IMAGE_HEIGHT, 3):
img = cv2.resize(img, (config.IMAGE_WIDTH, config.IMAGE_HEIGHT),
interpolation=cv2.INTER_AREA)
zc.check_and_display('input',
img,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
# get bitmap for current image
rtn_msg, bitmap = lc.process(img, stretch_ratio, display_list)
if rtn_msg['status'] != 'success':
print rtn_msg['message']
result_str = "None"
else:
result_str = json.dumps(bitmap.tolist())
packet = struct.pack("!I%dsI%ds" % (len(header_str), len(result_str)),
len(header_str), header_str, len(result_str),
result_str)
self.sock.sendall(packet)
def handle(self, header, data):
LOG.info("received new image")
header['status'] = "nothing"
result = {} # default
## first image
if self.is_first_image:
self.is_first_image = False
instruction = self.task.get_instruction(np.array([]))
header['status'] = 'success'
if instruction.get('speech', None) is not None:
result['speech'] = instruction['speech']
display_verbal_guidance(result['speech'])
if config.PLAY_SOUND:
data = result['speech']
packet = struct.pack("!I%ds" % len(data), len(data), data)
self.sound_sock.sendall(packet)
if instruction.get('image', None) is not None:
feedback_image = b64encode(zc.cv_image2raw(instruction['image']))
result['image'] = feedback_image
zc.check_and_display('img_guidance', instruction['image'], display_list,
resize_max=config.DISPLAY_MAX_PIXEL, wait_time=config.DISPLAY_WAIT_TIME)
return json.dumps(result)
## preprocessing of input image
img = zc.raw2cv_image(data)
if header.get('holo_capture', None) is not None:
zc.check_and_display('holo', img, display_list, resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
return json.dumps(result)
zc.check_and_display('input', img, display_list, resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
## preprocessing of input image
resize_ratio = 1
if max(img.shape) > config.IMAGE_MAX_WH:
resize_ratio = float(config.IMAGE_MAX_WH) / max(img.shape[0], img.shape[1])
img = cv2.resize(img, (0, 0), fx=resize_ratio, fy=resize_ratio, interpolation=cv2.INTER_AREA)
zc.check_and_display('input', img, display_list, resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
## get current state
t = time.time()
rtn_msg, objects_data = cc.process(img, resize_ratio, display_list)
print time.time() - t
## for measurement, when the sysmbolic representation has been got
if gabriel.Debug.TIME_MEASUREMENT:
result[gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
# the object detection result format is, for each line: [x1, y1, x2, y2, confidence, cls_idx]
objects = np.array(json.loads(objects_data))
objects = reorder_objects(objects)
if "object" in display_list:
img_object = zc.vis_detections(img, objects, config.LABELS)
zc.check_and_display("object", img_object, display_list, resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
LOG.info("object detection result: %s" % objects)
## for measurement, when the sysmbolic representation has been got
if gabriel.Debug.TIME_MEASUREMENT:
header[gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
## get instruction based on state
instruction = self.task.get_instruction(objects)
if instruction['status'] != 'success':
return json.dumps(result)
header['status'] = 'success'
if instruction.get('speech', None) is not None:
result['speech'] = instruction['speech']
display_verbal_guidance(result['speech'])
if config.PLAY_SOUND:
data = result['speech']
packet = struct.pack("!I%ds" % len(data), len(data), data)
self.sound_sock.sendall(packet)
if instruction.get('image', None) is not None:
feedback_image = b64encode(zc.cv_image2raw(instruction['image']))
result['image'] = feedback_image
zc.check_and_display('img_guidance', instruction['image'], display_list,
resize_max=config.DISPLAY_MAX_PIXEL, wait_time=config.DISPLAY_WAIT_TIME)
if instruction.get('holo_object', None) is not None:
result['holo_object'] = instruction['holo_object']
if instruction.get('holo_location', None) is not None:
result['holo_location'] = instruction['holo_location']
return json.dumps(result)