def ImageProcessor(self): """ Process each frame > detect left and right hand-gestures > perform mapping to {left token, right token}s > use Finite state machine to generate full instruction see more details in the paper: ieeexplore.ieee.org/document/8543168 """ # get the tokens (and the bounding boxes for vizualization) left_token, left_box, right_token, right_box, success_ = self.gest_rec.Get_gest(self.original, self.use_single_hand) print ("Hand gestures detection success: {2}. token: ({0}, {1})".format(right_token, left_token, success_)) if success_: # ROBO_GEST mode if self.robo_gest_mode: # reverse left and right since camera(left, right) == person(right, left) # then pass it to generate instruction get_token, done_ = self.ins.decode(right_token, left_token) print (get_token, done_) if done_: print print ("*** Decoded Instruction: {0}".format(get_token)) print # For Menue Selection only if self.menue_mode: #Notifies aquamenu that the gesture is waiting. msg = String() msg.data = 'WAITING' self.stat_pub.publish(msg) men_ins_, men_done_ = self.men_sel.decode(right_token, left_token) #print(men_ins_, men_done_) if men_done_: #Notifies aquamenu that the gesture is done. msg.data = 'DONE' self.stat_pub.publish(msg) print print ("Decoded Instruction: {0}".format(men_ins_)) print men_tok = men_ins_.split(' ') if (len(men_tok)>0 and men_tok[1] in self.menue_map.keys()): menue_selected = self.menue_map[men_tok[1]] msg = Tags() tag = Tag() tag.id = menue_selected msg.tags = [tag] self.tags_pub.publish(msg) print ('***** Menue selected :: {0}'.format(menue_selected)) print if (self.bench_test or self.publish_image): localised_objs = [(left_token, left_box), (right_token, right_box)] self.draw_boxes_and_labels(localised_objs) if self.bench_test: self.showFrame(self.original, 'test_viz') if self.publish_image: msg_frame = CvBridge().cv2_to_imgmsg(self.original, encoding="bgr8") self.ProcessedRaw.publish(msg_frame)
def ImageProcessor(self): """ Process each frame > detect left and right hand-gestures > perform mapping to {left token, right token}s > use Finite state machine to generate full instruction see more details in the paper: ieeexplore.ieee.org/document/8543168 """ # get the tokens (and the bounding boxes for vizualization) left_token, left_box, right_token, right_box, success_ = self.gest_rec.Get_gest( self.original, self.use_single_hand) print("Hand gestures detection success: {2}. token: ({0}, {1})".format( right_token, left_token, success_)) if success_: # ROBO_GEST mode if self.robo_gest_mode: # reverse left and right since camera(left, right) == person(right, left) # then pass it to generate instruction get_token, done_ = self.ins.decode(right_token, left_token) print(get_token, done_) if done_: print print("*** Decoded Instruction: {0}".format(get_token)) print # For Menue Selection only if self.menue_mode: men_ins_, men_done_ = self.men_sel.decode( right_token, left_token) #print(men_ins_, men_done_) if men_done_: print print("Decoded Instruction: {0}".format(men_ins_)) print men_tok = men_ins_.split(' ') if (len(men_tok) > 0 and men_tok[1] in self.menue_map.keys()): menue_selected = self.menue_map[men_tok[1]] msg = Tags() tag = Tag() tag.id = menue_selected msg.tags = [tag] self.tags_pub.publish(msg) print('***** Menue selected :: {0}'.format( menue_selected)) print if self.bench_test: self.showFrame(self.original, 'test_viz') if self.publish_image: if left_box != None: output_img = cv2.rectangle(self.original, (left_box[0], left_box[2]), (left_box[1], left_box[3]), (255, 0, 0), 2) else: output_img = self.original msg_frame = CvBridge().cv2_to_imgmsg(output_img, encoding="bgr8") self.ProcessedRaw.publish(msg_frame)