def handle(self, input_frame): to_server_extras = cognitive_engine.unpack_extras( ikea_pb2.ToServerExtras, input_frame) if (to_server_extras.zoom_status == ikea_pb2.ToServerExtras.ZoomStatus.STOP): msg = {'zoom_action': 'stop'} self._engine_conn.send(msg) pipe_output = self._engine_conn.recv() new_state_name = pipe_output.get('state') logger.info('Zoom Stopped. New state: %s', new_state_name) state = cv_rules.State[new_state_name.upper()] return state.update_result_wrapper(update_count=0) # When State contains a oneof field for different tasks, we can see if # none have been set using to_server_extras.WhichOneof('') if to_server_extras.state.step == ikea_pb2.State.Step.DONE: status = gabriel_pb2.ResultWrapper.Status.SUCCESS return cognitive_engine.create_result_wrapper(status) elif to_server_extras.state.step == ikea_pb2.State.Step.START: return cv_rules.State.BASE.update_result_wrapper(update_count=1) state = PROTO_TO_STATE[to_server_extras.state.step] if (to_server_extras.zoom_status == ikea_pb2.ToServerExtras.ZoomStatus.START): msg = {'zoom_action': 'start', 'state': state.name.lower()} self._engine_conn.send(msg) logger.info('Zoom Started') status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) to_client_extras = ikea_pb2.ToClientExtras() to_client_extras.zoom_info.app_key = credentials.ANDROID_KEY to_client_extras.zoom_info.app_secret = credentials.ANDROID_SECRET to_client_extras.zoom_info.meeting_number = ( credentials.MEETING_NUMBER) to_client_extras.zoom_info.meeting_password = ( credentials.MEETING_PASSWORD) result_wrapper.extras.Pack(to_client_extras) return result_wrapper assert len(input_frame.payloads) == 1 if input_frame.payload_type != gabriel_pb2.PayloadType.IMAGE: status = gabriel_pb2.ResultWrapper.Status.WRONG_INPUT_FORMAT return cognitive_engine.create_result_wrapper(status) np_data = np.frombuffer(input_frame.payloads[0], dtype=np.uint8) img = cv2.imdecode(np_data, cv2.IMREAD_COLOR) if max(img.shape) > IMAGE_MAX_WH: raise Exception('Image too large') return self._result_wrapper_from_cv(img, to_server_extras.state)
def handle(self, input_frame): if self._state == State.DONE: status = gabriel_pb2.ResultWrapper.Status.SUCCESS return cognitive_engine.create_result_wrapper(status) to_server = cognitive_engine.unpack_extras(knex_pb2.ToServer, input_frame) assert len(input_frame.payloads) == 1 if input_frame.payload_type != gabriel_pb2.PayloadType.IMAGE: status = gabriel_pb2.ResultWrapper.Status.WRONG_INPUT_FORMAT return cognitive_engine.create_result_wrapper(status) width = to_server.width height = to_server.height if width > IMAGE_MAX_WH or height > IMAGE_MAX_WH: raise Exception('Image too large') yuv = np.frombuffer(input_frame.payloads[0], dtype=np.uint8) yuv = np.reshape(yuv, ((height + (height // 2)), width)) img = cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR_NV21) img = np.rot90(img, 3) if self._state == State.NOTHING: return self._update_state_gen_response(State.NOSE) elif self._state == State.NOSE: if _img_is_label(img, NOSE): return self._update_state_gen_response(State.FUSELAGE) elif self._state == State.FUSELAGE: if _img_is_label(img, FUSELAGE): return self._update_state_gen_response(State.REAR) elif self._state == State.REAR: if _img_is_label(img, REAR): return self._update_state_gen_response(State.TAIL) elif self._state == State.TAIL: if _img_is_label(img, TAIL): return self._update_state_gen_response(State.WINGS) elif self._state == State.WINGS: if _img_is_label(img, WINGS): return self._update_state_gen_response(State.DONE) else: raise Exception('Bad State') status = gabriel_pb2.ResultWrapper.Status.SUCCESS return cognitive_engine.create_result_wrapper(status)
def handle(self, input_frame): np_data = np.frombuffer(input_frame.payloads[0], dtype=np.uint8) frame = cv2.imdecode(np_data, cv2.IMREAD_COLOR) cv2.imshow("Image from source: {}".format(self._source_name), frame) cv2.waitKey(1) status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) return result_wrapper
def handle(self, input_frame): status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.IMAGE result.payload = input_frame.payloads[0] result_wrapper.results.append(result) return result_wrapper
def result_wrapper_without_update(self, update_count): status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) to_client_extras = ikea_pb2.ToClientExtras() to_client_extras.state.update_count = update_count to_client_extras.state.step = self._proto_step result_wrapper.extras.Pack(to_client_extras) return result_wrapper
def _result_wrapper_for_state(state): status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.TEXT result.payload = state.get_speech().encode() result_wrapper.results.append(result) result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.IMAGE result.payload = state.get_image_bytes() result_wrapper.results.append(result) logger.info('sending %s', state.get_speech()) return result_wrapper
async def _heartbeat_helper(self): current_time = time.time() # We cannot directly iterate over items because we delete some entries for address, engine_worker in list(self._engine_workers.items()): if (current_time - engine_worker.get_last_sent()) < self._timeout: continue if ((not engine_worker.get_awaiting_heartbeat_response()) and engine_worker.get_current_input_metadata() is None): await engine_worker.send_heartbeat() continue source_info = engine_worker.get_source_info() logger.info( 'Lost connection to engine worker that consumes items ' 'from source: %s', source_info.get_name()) latest_input = source_info.get_latest_input() current_input_metadata = engine_worker.get_current_input_metadata() if (latest_input is not None and current_input_metadata == latest_input.metadata): # Return token for frame engine was in the middle of processing status = gabriel_pb2.ResultWrapper.Status.ENGINE_ERROR result_wrapper = cognitive_engine.create_result_wrapper(status) await self.send_result_wrapper( current_input_metadata.client_address, source_info.get_name(), current_input_metadata.frame_id, result_wrapper, return_token=True) source_info.remove_engine_worker(engine_worker) del self._engine_workers[address] if source_info.has_no_engine_workers(): source_name = source_info.get_name() logger.info( 'No remaining engines consume input from source: ' '%s', source_name) del self._source_infos[source_name] self.remove_source_consumed(source_name)
def buckle_result(dets_for_class, update_count, frames_with_one_buckle, frames_with_two_buckles): shadetops = dets_for_class[SHADETOP] buckles = dets_for_class[BUCKLE] if (len(shadetops) == 0) or (len(buckles) == 0): return State.BUCKLE.result_wrapper_without_update(update_count) status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) n_buckles = _count_buckles(shadetops, buckles) if n_buckles == 2: frames_with_one_buckle = 0 frames_with_two_buckles += 1 update_count += 1 if frames_with_two_buckles > 3: return State.BLACKCIRCLE.update_result_wrapper(update_count) elif n_buckles == 1: frames_with_one_buckle += 1 frames_with_two_buckles = 0 update_count += 1 # We only give this instruction when frames_with_one_buckle is # exactly 5 so it does not get repeated if frames_with_one_buckle == 5: result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.TEXT result.payload = ONE_WIRE_INSTRUCTION_BYTES result_wrapper.results.append(result) logger.info('sending second wire message') to_client_extras = ikea_pb2.ToClientExtras() to_client_extras.state.update_count = update_count to_client_extras.state.step = State.BUCKLE.get_proto_step() to_client_extras.state.frames_with_one_buckle = frames_with_one_buckle to_client_extras.state.frames_with_two_buckles = frames_with_two_buckles result_wrapper.extras.Pack(to_client_extras) return result_wrapper
def update_result_wrapper(self, update_count): status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.TEXT result.payload = self._speech_bytes result_wrapper.results.append(result) result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.IMAGE result.payload = self._image_bytes result_wrapper.results.append(result) logger.info('Updated State: %s', self.name) to_client_extras = ikea_pb2.ToClientExtras() to_client_extras.state.update_count = update_count to_client_extras.state.step = self.get_proto_step() result_wrapper.extras.Pack(to_client_extras) return result_wrapper
def handle(self, input_frame): yuv = np.frombuffer(input_frame.payloads[0], dtype=np.uint8) to_server = cognitive_engine.unpack_extras(yuv_pb2.ToServer, input_frame) width = to_server.width height = to_server.height rotation = to_server.rotation yuv = np.reshape(yuv, ((height + (height // 2)), width)) img = cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR_NV21) if rotation != 0: # np.rot90(img, 3) would correctly display an image rotated 90 # degress from Android times_to_rotate = (360 - rotation) / 90 img = np.rot90(img, times_to_rotate) cv2.imshow('Image from client', img) cv2.waitKey(1) status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) return result_wrapper
def handle(self, input_frame): if input_frame.payload_type != gabriel_pb2.PayloadType.IMAGE: status = gabriel_pb2.ResultWrapper.Status.WRONG_INPUT_FORMAT return cognitive_engine.create_result_wrapper(status) extras = cognitive_engine.unpack_extras(openscout_pb2.Extras, input_frame) image = self.preprocess_image(input_frame.payloads[0]) status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) result_wrapper.result_producer_name.value = self.ENGINE_NAME if extras.is_training: training_dir = os.getcwd()+"/training/" + extras.name + "/" try: os.mkdir(training_dir) except FileExistsError: logger.info("Directory already exists.") img = Image.open(image) path = training_dir + str(time.time()) + ".png" logger.info("Stored training image: {}".format(path)) img.save(path) self.new_faces = True else: #if we received new images for training and training has ended... if self.new_faces: self.train() self.new_faces = False result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.TEXT result.payload = 'Retraining complete.'.encode(encoding="utf-8") result_wrapper.results.append(result) else: faces_recognized = False response = self.infer(image) if response is not None: identities = json.loads(response) if len(identities) > 0: logger.debug('Detected {} faces. Attempting recognition...'.format(len(identities))) # Identify faces for person in identities: logger.debug(person) if person['confidence'] > self.threshold: faces_recognized = True logger.info('Recognized: {} - Score: {:.3f}'.format(person['name'], person['confidence'])) result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.TEXT result.payload = 'Recognized {} ({:.3f})'.format(person['name'], person['confidence']).encode(encoding="utf-8") result_wrapper.results.append(result) else: logger.debug('Confidence did not exceed threshold.') if faces_recognized: if self.store_detections: bb_img = Image.open(image) draw = ImageDraw.Draw(bb_img) for person in identities: draw.rectangle(self.getRectangle(person), width=4, outline='red') text = '{} ({:.3f})'.format(person['name'], person['confidence']) w, h = draw.textsize(text) xy= ((self.getRectangle(person)[0]), (self.getRectangle(person)[0][0]+w, self.getRectangle(person)[0][1]+h)) draw.rectangle(xy, width=4, outline='red', fill='red') draw.text(self.getRectangle(person)[0], text, fill='black') draw.bitmap((0,0), self.watermark, fill=None) filename = str(time.time()) + ".png" path = self.storage_path + filename logger.info("Stored image: {}".format(path)) bb_img.save(path) bio = BytesIO() bb_img.save(bio, format="JPEG") detection_log.info("{},{},{},{},{:.3f},{}".format(extras.client_id, extras.location.latitude, extras.location.longitude, person['name'], person['confidence'], os.environ["WEBSERVER"]+"/"+filename)) else: detection_log.info("{},{},{},{},{:.3f},".format(extras.client_id, extras.location.latitude, extras.location.longitude, person['name'], person['confidence'])) else: logger.debug('No faces recognized with confidence above {}.'.format(self.threshold)) return result_wrapper
def handle(self, input_frame): if input_frame.payload_type != gabriel_pb2.PayloadType.IMAGE: status = gabriel_pb2.ResultWrapper.Status.WRONG_INPUT_FORMAT return cognitive_engine.create_result_wrapper(status) extras = cognitive_engine.unpack_extras(openscout_pb2.Extras, input_frame) image = self.preprocess_image(input_frame.payloads[0]) status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) result_wrapper.result_producer_name.value = self.ENGINE_NAME if extras.is_training: training_dir = os.getcwd()+"/training/" + extras.name + "/" try: os.mkdir(training_dir) except FileExistsError: logger.info("Directory already exists.") img = Image.open(image) path = training_dir + str(time.time()) + ".png" logger.info("Stored training image: {}".format(path)) img.save(path) self.new_faces = True else: #if we received new images for training and training has ended... if self.new_faces: self.train() self.new_faces = False result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.TEXT result.payload = 'Retraining complete.'.encode(encoding="utf-8") result_wrapper.results.append(result) else: detections = self.detection(image) face_ids = [] for face in detections: face_ids.append(face.face_id) if len(face_ids) > 0: logger.debug('Detected {} faces. Attempting recognition...'.format(len(face_ids))) # Identify faces try: identities = self.recognition(face_ids) except ValidationError as v: logger.error(v.message) for person in identities: logger.debug(person) if len(person.candidates) > 0: if person.candidates[0].confidence > self.threshold: match = self.face_client.person_group_person.get(self.PERSON_GROUP_ID, person.candidates[0].person_id) logger.info('Recognized: {} - Score: {:.3f}'.format(match.name, person.candidates[0].confidence)) # Get topmost confidence score result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.TEXT result.payload = 'Recognized {} ({:.3f})'.format(match.name, person.candidates[0].confidence).encode(encoding="utf-8") result_wrapper.results.append(result) if self.store_detections: bb_img = Image.open(image) draw = ImageDraw.Draw(bb_img) for face in detections: draw.rectangle(self.getRectangle(face), width=4, outline='red') text = '{} ({:.3f})'.format(match.name, person.candidates[0].confidence) w, h = draw.textsize(text) xy= ((self.getRectangle(face)[0]), (self.getRectangle(face)[0][0]+w, self.getRectangle(face)[0][1]+h)) draw.rectangle(xy, width=4, outline='red', fill='red') draw.text(self.getRectangle(face)[0], text, fill='black') draw.bitmap((0,0), self.watermark, fill=None) filename = str(time.time()) + ".png" path = self.storage_path + filename logger.info("Stored image: {}".format(path)) bb_img.save(path) detection_log.info("{},{},{},{},{:.3f},{}".format(extras.client_id, extras.location.latitude, extras.location.longitude, match.name, person.candidates[0].confidence, os.environ["WEBSERVER"]+"/"+filename)) else: detection_log.info("{},{},{},{},{:.3f}".format(extras.client_id, extras.location.latitude, extras.location.longitude, match.name, person.candidates[0].confidence)) else: logger.debug('Confidence did not exceed threshold.') else: logger.debug('No faces recognized from person group.') return result_wrapper
def handle(self, input_frame): status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) return result_wrapper
def handle(self, input_frame): if input_frame.payload_type != gabriel_pb2.PayloadType.IMAGE: status = gabriel_pb2.ResultWrapper.Status.WRONG_INPUT_FORMAT return cognitive_engine.create_result_wrapper(status) extras = cognitive_engine.unpack_extras(openscout_pb2.Extras, input_frame) if extras.model != '' and extras.model != self.model: if not os.path.exists('./model/' + extras.model): logger.error( f"Model named {extras.model} not found. Sticking with previous model." ) else: self.detector = TFPredictor(extras.model) self.model = extras.model output_dict, image_np = self.process_image(input_frame.payloads[0]) status = gabriel_pb2.ResultWrapper.Status.SUCCESS result_wrapper = cognitive_engine.create_result_wrapper(status) result_wrapper.result_producer_name.value = self.ENGINE_NAME if self.store_detections: try: vis_util.visualize_boxes_and_labels_on_image_array( image_np, np.squeeze(output_dict['detection_boxes']), np.squeeze(output_dict['detection_classes']), np.squeeze(output_dict['detection_scores']), self.detector.category_index, use_normalized_coordinates=True, min_score_thresh=self.threshold, line_thickness=4) img = Image.fromarray(image_np) draw = ImageDraw.Draw(img) draw.bitmap((0, 0), self.watermark, fill=None) bio = BytesIO() img.save(bio, format="JPEG") filename = str(time.time()) + ".png" path = self.storage_path + filename except IndexError as e: logger.error(f"IndexError while getting bounding boxes [{e}]") return result_wrapper if output_dict['num_detections'] > 0: #convert numpy arrays to python lists classes = output_dict['detection_classes'].tolist() boxes = output_dict['detection_boxes'].tolist() scores = output_dict['detection_scores'].tolist() result = gabriel_pb2.ResultWrapper.Result() result.payload_type = gabriel_pb2.PayloadType.TEXT detections_above_threshold = False r = "" for i in range(0, len(classes)): if (scores[i] > self.threshold): if self.exclusions is None or classes[ i] not in self.exclusions: detections_above_threshold = True logger.info("Detected : {} - Score: {:.3f}".format( self.detector.category_index[classes[i]]['name'], scores[i])) if i > 0: r += ", " r += "Detected {} ({:.3f})".format( self.detector.category_index[classes[i]]['name'], scores[i]) if self.store_detections: detection_log.info("{},{},{},{},{:.3f},{}".format( extras.client_id, extras.location.latitude, extras.location.longitude, self.detector.category_index[ classes[i]]['name'], scores[i], os.environ["WEBSERVER"] + "/" + filename)) else: detection_log.info("{},{},{},{},{:.3f},".format( extras.client_id, extras.location.latitude, extras.location.longitude, self.detector.category_index[ classes[i]]['name'], scores[i])) if detections_above_threshold: result.payload = r.encode(encoding="utf-8") result_wrapper.results.append(result) if self.store_detections: logger.info("Stored image: {}".format(path)) img.save(path) return result_wrapper