def handle(self, input_frame):
to_server_extras = cognitive_engine.unpack_extras(
ikea_pb2.ToServerExtras, input_frame)
if (to_server_extras.zoom_status ==
ikea_pb2.ToServerExtras.ZoomStatus.STOP):
msg = {'zoom_action': 'stop'}
self._engine_conn.send(msg)
pipe_output = self._engine_conn.recv()
new_state_name = pipe_output.get('state')
logger.info('Zoom Stopped. New state: %s', new_state_name)
state = cv_rules.State[new_state_name.upper()]
return state.update_result_wrapper(update_count=0)
# When State contains a oneof field for different tasks, we can see if
# none have been set using to_server_extras.WhichOneof('')
if to_server_extras.state.step == ikea_pb2.State.Step.DONE:
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
return cognitive_engine.create_result_wrapper(status)
elif to_server_extras.state.step == ikea_pb2.State.Step.START:
return cv_rules.State.BASE.update_result_wrapper(update_count=1)
state = PROTO_TO_STATE[to_server_extras.state.step]
if (to_server_extras.zoom_status ==
ikea_pb2.ToServerExtras.ZoomStatus.START):
msg = {'zoom_action': 'start', 'state': state.name.lower()}
self._engine_conn.send(msg)
logger.info('Zoom Started')
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
to_client_extras = ikea_pb2.ToClientExtras()
to_client_extras.zoom_info.app_key = credentials.ANDROID_KEY
to_client_extras.zoom_info.app_secret = credentials.ANDROID_SECRET
to_client_extras.zoom_info.meeting_number = (
credentials.MEETING_NUMBER)
to_client_extras.zoom_info.meeting_password = (
credentials.MEETING_PASSWORD)
result_wrapper.extras.Pack(to_client_extras)
return result_wrapper
assert len(input_frame.payloads) == 1
if input_frame.payload_type != gabriel_pb2.PayloadType.IMAGE:
status = gabriel_pb2.ResultWrapper.Status.WRONG_INPUT_FORMAT
return cognitive_engine.create_result_wrapper(status)
np_data = np.frombuffer(input_frame.payloads[0], dtype=np.uint8)
img = cv2.imdecode(np_data, cv2.IMREAD_COLOR)
if max(img.shape) > IMAGE_MAX_WH:
raise Exception('Image too large')
return self._result_wrapper_from_cv(img, to_server_extras.state)
def handle(self, input_frame):
if self._state == State.DONE:
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
return cognitive_engine.create_result_wrapper(status)
to_server = cognitive_engine.unpack_extras(knex_pb2.ToServer,
input_frame)
assert len(input_frame.payloads) == 1
if input_frame.payload_type != gabriel_pb2.PayloadType.IMAGE:
status = gabriel_pb2.ResultWrapper.Status.WRONG_INPUT_FORMAT
return cognitive_engine.create_result_wrapper(status)
width = to_server.width
height = to_server.height
if width > IMAGE_MAX_WH or height > IMAGE_MAX_WH:
raise Exception('Image too large')
yuv = np.frombuffer(input_frame.payloads[0], dtype=np.uint8)
yuv = np.reshape(yuv, ((height + (height // 2)), width))
img = cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR_NV21)
img = np.rot90(img, 3)
if self._state == State.NOTHING:
return self._update_state_gen_response(State.NOSE)
elif self._state == State.NOSE:
if _img_is_label(img, NOSE):
return self._update_state_gen_response(State.FUSELAGE)
elif self._state == State.FUSELAGE:
if _img_is_label(img, FUSELAGE):
return self._update_state_gen_response(State.REAR)
elif self._state == State.REAR:
if _img_is_label(img, REAR):
return self._update_state_gen_response(State.TAIL)
elif self._state == State.TAIL:
if _img_is_label(img, TAIL):
return self._update_state_gen_response(State.WINGS)
elif self._state == State.WINGS:
if _img_is_label(img, WINGS):
return self._update_state_gen_response(State.DONE)
else:
raise Exception('Bad State')
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
return cognitive_engine.create_result_wrapper(status)
def handle(self, input_frame):
np_data = np.frombuffer(input_frame.payloads[0], dtype=np.uint8)
frame = cv2.imdecode(np_data, cv2.IMREAD_COLOR)
cv2.imshow("Image from source: {}".format(self._source_name), frame)
cv2.waitKey(1)
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
return result_wrapper
def handle(self, input_frame):
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.IMAGE
result.payload = input_frame.payloads[0]
result_wrapper.results.append(result)
return result_wrapper
def result_wrapper_without_update(self, update_count):
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
to_client_extras = ikea_pb2.ToClientExtras()
to_client_extras.state.update_count = update_count
to_client_extras.state.step = self._proto_step
result_wrapper.extras.Pack(to_client_extras)
return result_wrapper
def _result_wrapper_for_state(state):
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.TEXT
result.payload = state.get_speech().encode()
result_wrapper.results.append(result)
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.IMAGE
result.payload = state.get_image_bytes()
result_wrapper.results.append(result)
logger.info('sending %s', state.get_speech())
return result_wrapper
async def _heartbeat_helper(self):
current_time = time.time()
# We cannot directly iterate over items because we delete some entries
for address, engine_worker in list(self._engine_workers.items()):
if (current_time - engine_worker.get_last_sent()) < self._timeout:
continue
if ((not engine_worker.get_awaiting_heartbeat_response())
and engine_worker.get_current_input_metadata() is None):
await engine_worker.send_heartbeat()
continue
source_info = engine_worker.get_source_info()
logger.info(
'Lost connection to engine worker that consumes items '
'from source: %s', source_info.get_name())
latest_input = source_info.get_latest_input()
current_input_metadata = engine_worker.get_current_input_metadata()
if (latest_input is not None
and current_input_metadata == latest_input.metadata):
# Return token for frame engine was in the middle of processing
status = gabriel_pb2.ResultWrapper.Status.ENGINE_ERROR
result_wrapper = cognitive_engine.create_result_wrapper(status)
await self.send_result_wrapper(
current_input_metadata.client_address,
source_info.get_name(),
current_input_metadata.frame_id,
result_wrapper,
return_token=True)
source_info.remove_engine_worker(engine_worker)
del self._engine_workers[address]
if source_info.has_no_engine_workers():
source_name = source_info.get_name()
logger.info(
'No remaining engines consume input from source: '
'%s', source_name)
del self._source_infos[source_name]
self.remove_source_consumed(source_name)
def buckle_result(dets_for_class, update_count, frames_with_one_buckle,
frames_with_two_buckles):
shadetops = dets_for_class[SHADETOP]
buckles = dets_for_class[BUCKLE]
if (len(shadetops) == 0) or (len(buckles) == 0):
return State.BUCKLE.result_wrapper_without_update(update_count)
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
n_buckles = _count_buckles(shadetops, buckles)
if n_buckles == 2:
frames_with_one_buckle = 0
frames_with_two_buckles += 1
update_count += 1
if frames_with_two_buckles > 3:
return State.BLACKCIRCLE.update_result_wrapper(update_count)
elif n_buckles == 1:
frames_with_one_buckle += 1
frames_with_two_buckles = 0
update_count += 1
# We only give this instruction when frames_with_one_buckle is
# exactly 5 so it does not get repeated
if frames_with_one_buckle == 5:
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.TEXT
result.payload = ONE_WIRE_INSTRUCTION_BYTES
result_wrapper.results.append(result)
logger.info('sending second wire message')
to_client_extras = ikea_pb2.ToClientExtras()
to_client_extras.state.update_count = update_count
to_client_extras.state.step = State.BUCKLE.get_proto_step()
to_client_extras.state.frames_with_one_buckle = frames_with_one_buckle
to_client_extras.state.frames_with_two_buckles = frames_with_two_buckles
result_wrapper.extras.Pack(to_client_extras)
return result_wrapper
def update_result_wrapper(self, update_count):
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.TEXT
result.payload = self._speech_bytes
result_wrapper.results.append(result)
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.IMAGE
result.payload = self._image_bytes
result_wrapper.results.append(result)
logger.info('Updated State: %s', self.name)
to_client_extras = ikea_pb2.ToClientExtras()
to_client_extras.state.update_count = update_count
to_client_extras.state.step = self.get_proto_step()
result_wrapper.extras.Pack(to_client_extras)
return result_wrapper
def handle(self, input_frame):
yuv = np.frombuffer(input_frame.payloads[0], dtype=np.uint8)
to_server = cognitive_engine.unpack_extras(yuv_pb2.ToServer,
input_frame)
width = to_server.width
height = to_server.height
rotation = to_server.rotation
yuv = np.reshape(yuv, ((height + (height // 2)), width))
img = cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR_NV21)
if rotation != 0:
# np.rot90(img, 3) would correctly display an image rotated 90
# degress from Android
times_to_rotate = (360 - rotation) / 90
img = np.rot90(img, times_to_rotate)
cv2.imshow('Image from client', img)
cv2.waitKey(1)
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
return result_wrapper
def handle(self, input_frame):
if input_frame.payload_type != gabriel_pb2.PayloadType.IMAGE:
status = gabriel_pb2.ResultWrapper.Status.WRONG_INPUT_FORMAT
return cognitive_engine.create_result_wrapper(status)
extras = cognitive_engine.unpack_extras(openscout_pb2.Extras, input_frame)
image = self.preprocess_image(input_frame.payloads[0])
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
result_wrapper.result_producer_name.value = self.ENGINE_NAME
if extras.is_training:
training_dir = os.getcwd()+"/training/" + extras.name + "/"
try:
os.mkdir(training_dir)
except FileExistsError:
logger.info("Directory already exists.")
img = Image.open(image)
path = training_dir + str(time.time()) + ".png"
logger.info("Stored training image: {}".format(path))
img.save(path)
self.new_faces = True
else:
#if we received new images for training and training has ended...
if self.new_faces:
self.train()
self.new_faces = False
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.TEXT
result.payload = 'Retraining complete.'.encode(encoding="utf-8")
result_wrapper.results.append(result)
else:
faces_recognized = False
response = self.infer(image)
if response is not None:
identities = json.loads(response)
if len(identities) > 0:
logger.debug('Detected {} faces. Attempting recognition...'.format(len(identities)))
# Identify faces
for person in identities:
logger.debug(person)
if person['confidence'] > self.threshold:
faces_recognized = True
logger.info('Recognized: {} - Score: {:.3f}'.format(person['name'], person['confidence']))
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.TEXT
result.payload = 'Recognized {} ({:.3f})'.format(person['name'], person['confidence']).encode(encoding="utf-8")
result_wrapper.results.append(result)
else:
logger.debug('Confidence did not exceed threshold.')
if faces_recognized:
if self.store_detections:
bb_img = Image.open(image)
draw = ImageDraw.Draw(bb_img)
for person in identities:
draw.rectangle(self.getRectangle(person), width=4, outline='red')
text = '{} ({:.3f})'.format(person['name'], person['confidence'])
w, h = draw.textsize(text)
xy= ((self.getRectangle(person)[0]), (self.getRectangle(person)[0][0]+w, self.getRectangle(person)[0][1]+h))
draw.rectangle(xy, width=4, outline='red', fill='red')
draw.text(self.getRectangle(person)[0], text, fill='black')
draw.bitmap((0,0), self.watermark, fill=None)
filename = str(time.time()) + ".png"
path = self.storage_path + filename
logger.info("Stored image: {}".format(path))
bb_img.save(path)
bio = BytesIO()
bb_img.save(bio, format="JPEG")
detection_log.info("{},{},{},{},{:.3f},{}".format(extras.client_id, extras.location.latitude, extras.location.longitude, person['name'], person['confidence'], os.environ["WEBSERVER"]+"/"+filename))
else:
detection_log.info("{},{},{},{},{:.3f},".format(extras.client_id, extras.location.latitude, extras.location.longitude, person['name'], person['confidence']))
else:
logger.debug('No faces recognized with confidence above {}.'.format(self.threshold))
return result_wrapper
def handle(self, input_frame):
if input_frame.payload_type != gabriel_pb2.PayloadType.IMAGE:
status = gabriel_pb2.ResultWrapper.Status.WRONG_INPUT_FORMAT
return cognitive_engine.create_result_wrapper(status)
extras = cognitive_engine.unpack_extras(openscout_pb2.Extras, input_frame)
image = self.preprocess_image(input_frame.payloads[0])
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
result_wrapper.result_producer_name.value = self.ENGINE_NAME
if extras.is_training:
training_dir = os.getcwd()+"/training/" + extras.name + "/"
try:
os.mkdir(training_dir)
except FileExistsError:
logger.info("Directory already exists.")
img = Image.open(image)
path = training_dir + str(time.time()) + ".png"
logger.info("Stored training image: {}".format(path))
img.save(path)
self.new_faces = True
else:
#if we received new images for training and training has ended...
if self.new_faces:
self.train()
self.new_faces = False
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.TEXT
result.payload = 'Retraining complete.'.encode(encoding="utf-8")
result_wrapper.results.append(result)
else:
detections = self.detection(image)
face_ids = []
for face in detections:
face_ids.append(face.face_id)
if len(face_ids) > 0:
logger.debug('Detected {} faces. Attempting recognition...'.format(len(face_ids)))
# Identify faces
try:
identities = self.recognition(face_ids)
except ValidationError as v:
logger.error(v.message)
for person in identities:
logger.debug(person)
if len(person.candidates) > 0:
if person.candidates[0].confidence > self.threshold:
match = self.face_client.person_group_person.get(self.PERSON_GROUP_ID, person.candidates[0].person_id)
logger.info('Recognized: {} - Score: {:.3f}'.format(match.name, person.candidates[0].confidence)) # Get topmost confidence score
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.TEXT
result.payload = 'Recognized {} ({:.3f})'.format(match.name, person.candidates[0].confidence).encode(encoding="utf-8")
result_wrapper.results.append(result)
if self.store_detections:
bb_img = Image.open(image)
draw = ImageDraw.Draw(bb_img)
for face in detections:
draw.rectangle(self.getRectangle(face), width=4, outline='red')
text = '{} ({:.3f})'.format(match.name, person.candidates[0].confidence)
w, h = draw.textsize(text)
xy= ((self.getRectangle(face)[0]), (self.getRectangle(face)[0][0]+w, self.getRectangle(face)[0][1]+h))
draw.rectangle(xy, width=4, outline='red', fill='red')
draw.text(self.getRectangle(face)[0], text, fill='black')
draw.bitmap((0,0), self.watermark, fill=None)
filename = str(time.time()) + ".png"
path = self.storage_path + filename
logger.info("Stored image: {}".format(path))
bb_img.save(path)
detection_log.info("{},{},{},{},{:.3f},{}".format(extras.client_id, extras.location.latitude, extras.location.longitude, match.name, person.candidates[0].confidence, os.environ["WEBSERVER"]+"/"+filename))
else:
detection_log.info("{},{},{},{},{:.3f}".format(extras.client_id, extras.location.latitude, extras.location.longitude, match.name, person.candidates[0].confidence))
else:
logger.debug('Confidence did not exceed threshold.')
else:
logger.debug('No faces recognized from person group.')
return result_wrapper
def handle(self, input_frame):
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
return result_wrapper
def handle(self, input_frame):
if input_frame.payload_type != gabriel_pb2.PayloadType.IMAGE:
status = gabriel_pb2.ResultWrapper.Status.WRONG_INPUT_FORMAT
return cognitive_engine.create_result_wrapper(status)
extras = cognitive_engine.unpack_extras(openscout_pb2.Extras,
input_frame)
if extras.model != '' and extras.model != self.model:
if not os.path.exists('./model/' + extras.model):
logger.error(
f"Model named {extras.model} not found. Sticking with previous model."
)
else:
self.detector = TFPredictor(extras.model)
self.model = extras.model
output_dict, image_np = self.process_image(input_frame.payloads[0])
status = gabriel_pb2.ResultWrapper.Status.SUCCESS
result_wrapper = cognitive_engine.create_result_wrapper(status)
result_wrapper.result_producer_name.value = self.ENGINE_NAME
if self.store_detections:
try:
vis_util.visualize_boxes_and_labels_on_image_array(
image_np,
np.squeeze(output_dict['detection_boxes']),
np.squeeze(output_dict['detection_classes']),
np.squeeze(output_dict['detection_scores']),
self.detector.category_index,
use_normalized_coordinates=True,
min_score_thresh=self.threshold,
line_thickness=4)
img = Image.fromarray(image_np)
draw = ImageDraw.Draw(img)
draw.bitmap((0, 0), self.watermark, fill=None)
bio = BytesIO()
img.save(bio, format="JPEG")
filename = str(time.time()) + ".png"
path = self.storage_path + filename
except IndexError as e:
logger.error(f"IndexError while getting bounding boxes [{e}]")
return result_wrapper
if output_dict['num_detections'] > 0:
#convert numpy arrays to python lists
classes = output_dict['detection_classes'].tolist()
boxes = output_dict['detection_boxes'].tolist()
scores = output_dict['detection_scores'].tolist()
result = gabriel_pb2.ResultWrapper.Result()
result.payload_type = gabriel_pb2.PayloadType.TEXT
detections_above_threshold = False
r = ""
for i in range(0, len(classes)):
if (scores[i] > self.threshold):
if self.exclusions is None or classes[
i] not in self.exclusions:
detections_above_threshold = True
logger.info("Detected : {} - Score: {:.3f}".format(
self.detector.category_index[classes[i]]['name'],
scores[i]))
if i > 0:
r += ", "
r += "Detected {} ({:.3f})".format(
self.detector.category_index[classes[i]]['name'],
scores[i])
if self.store_detections:
detection_log.info("{},{},{},{},{:.3f},{}".format(
extras.client_id, extras.location.latitude,
extras.location.longitude,
self.detector.category_index[
classes[i]]['name'], scores[i],
os.environ["WEBSERVER"] + "/" + filename))
else:
detection_log.info("{},{},{},{},{:.3f},".format(
extras.client_id, extras.location.latitude,
extras.location.longitude,
self.detector.category_index[
classes[i]]['name'], scores[i]))
if detections_above_threshold:
result.payload = r.encode(encoding="utf-8")
result_wrapper.results.append(result)
if self.store_detections:
logger.info("Stored image: {}".format(path))
img.save(path)
return result_wrapper