def handle(self, from_client):
if from_client.payload_type != gabriel_pb2.PayloadType.IMAGE:
return cognitive_engine.wrong_input_format_error(
from_client.frame_id)
engine_fields = cognitive_engine.unpack_engine_fields(
instruction_pb2.EngineFields, from_client)
img_array = np.asarray(bytearray(from_client.payload), dtype=np.int8)
img = cv2.imdecode(img_array, -1)
if max(img.shape) > IMAGE_MAX_WH:
resize_ratio = float(IMAGE_MAX_WH) / max(img.shape[0],
img.shape[1])
img = cv2.resize(img, (0, 0),
fx=resize_ratio,
fy=resize_ratio,
interpolation=cv2.INTER_AREA)
objects = self._detect_object(img)
if objects is not None:
objects[:, :4] /= resize_ratio
else:
objects = self._detect_object(img)
objects = reorder_objects(objects)
logger.info("object detection result: %s", objects)
result_wrapper = instructions.get_instruction(engine_fields, objects)
result_wrapper.frame_id = from_client.frame_id
result_wrapper.status = gabriel_pb2.ResultWrapper.Status.SUCCESS
return result_wrapper
def handle(self, from_client):
if from_client.payload_type != gabriel_pb2.PayloadType.IMAGE:
return cognitive_engine.wrong_input_format_error(
from_client.frame_id)
engine_fields = cognitive_engine.unpack_engine_fields(
instruction_pb2.EngineFields, from_client)
img_array = np.asarray(bytearray(from_client.payload), dtype=np.int8)
img = cv2.imdecode(img_array, -1)
if max(img.shape) > IMAGE_MAX_WH:
resize_ratio = float(IMAGE_MAX_WH) / max(img.shape[0],
img.shape[1])
img = cv2.resize(img, (0, 0),
fx=resize_ratio,
fy=resize_ratio,
interpolation=cv2.INTER_AREA)
dets_for_class = self._detect_object(img)
for class_idx in dets_for_class:
for i in range(len(dets_for_class[class_idx])):
dets_for_class[class_idx][i][:4] /= resize_ratio
else:
dets_for_class = self._detect_object(img)
result_wrapper = instructions.get_instruction(engine_fields,
dets_for_class)
result_wrapper.frame_id = from_client.frame_id
result_wrapper.status = gabriel_pb2.ResultWrapper.Status.SUCCESS
return result_wrapper
def handle(self, from_client):
if from_client.payload_type != gabriel_pb2.PayloadType.IMAGE:
return cognitive_engine.wrong_input_format_error(
from_client.frame_id)
engine_fields = cognitive_engine.unpack_engine_fields(
instruction_pb2.EngineFields, from_client)
img_array = np.asarray(bytearray(from_client.payload), dtype=np.int8)
img = cv2.imdecode(img_array, -1)
objects = []
if max(img.shape) > config.IMAGE_MAX_WH:
resize_ratio = float(config.IMAGE_MAX_WH) / max(img.shape[0], img.shape[1])
img = cv2.resize(img, (0, 0), fx=resize_ratio, fy=resize_ratio,
interpolation=cv2.INTER_AREA)
objects = self.detector.detect(img)
if objects is not None:
objects[:, :4] /= resize_ratio
else:
objects = self.detector.detect(img)
logger.info("object detection result: %s", objects)
# obtain client headers
headers = {}
if engine_fields.ribloc.gauge_color:
headers['gaugeColor'] = str(engine_fields.ribloc.gauge_color)
logger.debug("received gaugeColor to be: %s", headers['gaugeColor'])
vis_objects, instruction = self.task.get_instruction(objects, header=headers)
result_wrapper = self._serialize_to_pb(headers, instruction, engine_fields)
logger.info("result_wrapper: {}".format(result_wrapper))
return result_wrapper
def handle(self, from_client):
received = time.time()
if from_client.payload_type != gabriel_pb2.PayloadType.IMAGE:
self._logger.error('Wrong input type. LEGO expects inputs to be '
'images, but the received input is of type: '
f'{from_client.payload_type.__name__}')
return cognitive_engine.wrong_input_format_message(
from_client.frame_id)
self._logger.info('Received a frame from client.')
# build the engine state
engine_fields = cognitive_engine.unpack_engine_fields(
instruction_proto.EngineFields, from_client)
old_lego_state = instruction_proto.LEGOState()
old_lego_state.CopyFrom(engine_fields.lego)
try:
c_task = DefaultTaskManager.get_task(old_lego_state.task_id)
except NoSuchTaskError:
self._logger.warning(
f'Invalid task name: {old_lego_state.task_id}')
self._logger.warning(f'Proceeding with default task instead: '
f'{self._DEFAULT_TASK}')
c_task = DefaultTaskManager.get_task(self._DEFAULT_TASK)
old_lego_state.task_id = self._DEFAULT_TASK
current_state_id = old_lego_state.current_state_index
target_state_id = old_lego_state.target_state_index
if old_lego_state.status == \
instruction_proto.LEGOState.STATUS.INIT:
self._logger.info('Sending initial guidance...')
current_state = InitialTaskState(c_task)
# immediately return initial guidance
return LEGOCognitiveEngine._wrap_LEGO_state(
frame_id=from_client.frame_id,
status=gabriel_pb2.ResultWrapper.Status.SUCCESS,
lego_state=LEGOCognitiveEngine._build_LEGO_state(
task_id=old_lego_state.task_id,
recv_time=received,
status=instruction_proto.LEGOState.STATUS.
WAITING_FOR_BOARD,
result=instruction_proto.LEGOState.FRAME_RESULT.SUCCESS,
target_state=0,
current_state=-1,
),
update_cnt=engine_fields.update_count + 1,
img_guidance=current_state.current_image,
txt_guidance=current_state.current_instruction)
elif old_lego_state.status == \
instruction_proto.LEGOState.STATUS.WAITING_FOR_BOARD:
# initial state, waiting for LEGO board to start task
self._logger.info('Checking for initial board presence...')
current_state = InitialTaskState(c_task)
status = instruction_proto.LEGOState.STATUS.WAITING_FOR_BOARD
elif old_lego_state.status == \
instruction_proto.LEGOState.STATUS.NORMAL:
# normal, non-error state
self._logger.info('Engine in normal state.')
current_state = TaskState.generate_correct_state(
c_task, current_state_id)
status = instruction_proto.LEGOState.STATUS.NORMAL
elif old_lego_state.status == \
instruction_proto.LEGOState.STATUS.ERROR:
# task error
self._logger.info('Engine in error state.')
p_board_state = BoardState(
np.asarray(bytearray(old_lego_state.error_prev_board_state),
dtype=np.int8))
current_state = IncorrectTaskState(c_task, target_state_id,
p_board_state)
status = instruction_proto.LEGOState.STATUS.ERROR
elif old_lego_state.status == \
instruction_proto.LEGOState.STATUS.FINISHED:
# finished the task, just return empty success messages ad infinitum
self._logger.info('Engine in finished state.')
return LEGOCognitiveEngine._wrap_LEGO_state(
frame_id=from_client.frame_id,
status=gabriel_pb2.ResultWrapper.Status.SUCCESS,
lego_state=LEGOCognitiveEngine._build_LEGO_state(
task_id=old_lego_state.task_id,
recv_time=received,
status=instruction_proto.LEGOState.STATUS.FINISHED,
result=instruction_proto.LEGOState.FRAME_RESULT.SUCCESS,
target_state=-1,
current_state=-1,
),
update_cnt=engine_fields.update_count + 1,
)
else:
# unimplemented state
# todo, send error message?
raise RuntimeError()
board_state = None
img_guidance = None
txt_guidance = None
try:
# process input
img_array = np.asarray(bytearray(from_client.payload),
dtype=np.int8)
img = cv2.imdecode(img_array, cv2.IMREAD_UNCHANGED)
# process the image into a BoardState
try:
board_state = BoardState(img_util.preprocess_img(img))
except NoLEGODetectedError:
# board is in frame, but it's empty
self._logger.info('Detected empty board.')
board_state = EmptyBoardState()
# *actually* compute next state
next_state = current_state.compute_next_task_state(board_state)
# code after this point will only execute if board state was
# correctly processed (previous statements all shortcut through
# exceptions on failure)
current_state_id = next_state.state_index
target_state_id = next_state.next_state_index
img_guidance = next_state.current_image
txt_guidance = next_state.current_instruction
if next_state.is_correct:
# step was performed correctly!
result = instruction_proto.LEGOState.FRAME_RESULT.SUCCESS
if next_state.is_final:
self._logger.info('Finished task!')
status = instruction_proto.LEGOState.STATUS.FINISHED
else:
self._logger.info(
'Step was performed correctly, providing '
'guidance for next step.')
status = instruction_proto.LEGOState.STATUS.NORMAL
else:
# task error
status = instruction_proto.LEGOState.STATUS.ERROR
result = instruction_proto.LEGOState.FRAME_RESULT.TASK_ERROR
self._logger.info('Input is incorrect! Providing guidance to '
'correct error.')
except LowConfidenceError:
self._logger.warning('Low confidence in LEGO reconstruction.')
result = \
instruction_proto.LEGOState.FRAME_RESULT.LOW_CONFIDENCE_RECON
except NoBoardDetectedError:
# junk frame, no board in frame
self._logger.warning('No board detected in frame.')
result = \
instruction_proto.LEGOState.FRAME_RESULT.JUNK_FRAME
except NoStateChangeError:
self._logger.warning('No change from previous input.')
result = \
instruction_proto.LEGOState.FRAME_RESULT.NO_CHANGE
except Exception as e:
# anything else
self._logger.error('CV processing failed!')
self._logger.error(e)
# immediately return
return LEGOCognitiveEngine._wrap_LEGO_state(
frame_id=from_client.frame_id,
status=gabriel_pb2.ResultWrapper.Status.ENGINE_ERROR,
lego_state=LEGOCognitiveEngine._build_LEGO_state(
task_id=old_lego_state.task_id,
recv_time=received,
status=instruction_proto.LEGOState.STATUS.ERROR,
result=instruction_proto.LEGOState.FRAME_RESULT.
OTHER_CV_ERROR,
target_state=-1,
current_state=-1,
),
update_cnt=engine_fields.update_count + 1,
)
self._logger.info('Sending result to client...')
return LEGOCognitiveEngine._wrap_LEGO_state(
frame_id=from_client.frame_id,
status=gabriel_pb2.ResultWrapper.Status.SUCCESS,
lego_state=LEGOCognitiveEngine._build_LEGO_state(
task_id=old_lego_state.task_id,
recv_time=received,
status=status,
result=result,
target_state=target_state_id,
current_state=current_state_id,
prev_board_state=board_state),
update_cnt=engine_fields.update_count + 1,
img_guidance=img_guidance,
txt_guidance=txt_guidance)
def handle(self, from_client):
if from_client.payload_type != gabriel_pb2.PayloadType.IMAGE:
return cognitive_engine.wrong_input_format_error(
from_client.frame_id)
engine_fields = cognitive_engine.unpack_engine_fields(
instruction_pb2.EngineFields, from_client)
result_wrapper = gabriel_pb2.ResultWrapper()
result_wrapper.frame_id = from_client.frame_id
result_wrapper.status = gabriel_pb2.ResultWrapper.Status.SUCCESS
img_array = np.asarray(bytearray(from_client.payload), dtype=np.int8)
img = cv2.imdecode(img_array, -1)
if max(img.shape) != IMAGE_MAX_WH:
resize_ratio = float(IMAGE_MAX_WH) / max(img.shape[0],
img.shape[1])
img = cv2.resize(img, (0, 0),
fx=resize_ratio,
fy=resize_ratio,
interpolation=cv2.INTER_AREA)
frame_time = current_milli_time()
self.state['is_playing'] = self.ball_trace.is_playing(
frame_time) and self.seen_opponent
## check if two frames are too close
if (self.prev_frame_info is not None
and frame_time - self.prev_frame_info['time'] < 80):
logger.info("two frames too close!")
return complete_result_wrapper(result_wrapper, engine_fields)
## find table
rtn_msg, objects = pingpong_cv.find_table(img, O_IMG_HEIGHT,
O_IMG_WIDTH)
if rtn_msg['status'] != 'success':
logger.info(rtn_msg['message'])
return complete_result_wrapper(result_wrapper, engine_fields)
img_rotated, mask_table, rotation_matrix = objects
current_frame_info = {
'time': frame_time,
'img': img,
'img_rotated': img_rotated,
'mask_ball': None
}
## in case we don't have good "previous" frame, process the current one
# and return
mask_ball = None
ball_stat = None
if (self.prev_frame_info is None
or frame_time - self.prev_frame_info['time'] > 300):
logger.info("previous frame not good")
rtn_msg, objects = pingpong_cv.find_pingpong(
img, None, mask_table, None, rotation_matrix)
if rtn_msg['status'] != 'success':
logger.info(rtn_msg['message'])
else:
mask_ball, ball_stat = objects
self.ball_trace.insert((frame_time, ball_stat))
current_frame_info['mask_ball'] = mask_ball
self.prev_frame_info = current_frame_info
return complete_result_wrapper(result_wrapper, engine_fields)
## now we do have an okay previous frame
rtn_msg, objects = pingpong_cv.find_pingpong(
img, self.prev_frame_info['img'], mask_table,
self.prev_frame_info['mask_ball'], rotation_matrix)
if rtn_msg['status'] != 'success':
logger.info(rtn_msg['message'])
else:
mask_ball, ball_stat = objects
self.ball_trace.insert((frame_time, ball_stat))
current_frame_info['mask_ball'] = mask_ball
## determine where the wall was hit to
self.state['ball_position'] = self.ball_trace.leftOrRight()
## find position (relatively, left or right) of your opponent
rtn_msg, objects = pingpong_cv.find_opponent(
img_rotated, self.prev_frame_info['img_rotated'], O_IMG_HEIGHT)
if rtn_msg['status'] != 'success':
self.seen_opponent = False
logger.info(rtn_msg['message'])
self.prev_frame_info = current_frame_info
return complete_result_wrapper(result_wrapper, engine_fields)
self.seen_opponent = True
opponent_x = objects
# a simple averaging over history
self.opponent_x = self.opponent_x * 0.7 + opponent_x * 0.3
self.state['opponent_position'] = (
"left" if self.opponent_x < O_IMG_WIDTH * 0.58 else "right")
t = time.time()
if self.state['is_playing']:
if self.state['opponent_position'] == "left":
if ((t - self.last_played_t < 3
and self.last_played == "right")
or (t - self.last_played_t < 1)):
return complete_result_wrapper(result_wrapper,
engine_fields)
speech = "right"
self.last_played_t = t
self.last_played = speech
result_with_update(result_wrapper, engine_fields, speech)
elif self.state['opponent_position'] == "right":
if ((t - self.last_played_t < 3 and self.last_played == "left")
or (t - self.last_played_t < 1)):
return complete_result_wrapper(result_wrapper,
engine_fields)
speech = "left"
self.last_played_t = t
self.last_played = speech
result_with_update(result_wrapper, engine_fields, speech)
return complete_result_wrapper(result_wrapper, engine_fields)