def infinite_infer_run(): """ Entry point of the lambda function""" try: # This face detection model is implemented as single shot detector (ssd). model_type = 'ssd' output_map = {1: 'face'} # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format( os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The sample projects come with optimized artifacts, hence only the artifact # path is required. model_path = '/opt/awscam/artifacts/mxnet_deploy_ssd_FP16_FUSED.xml' # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading face detection model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Face detection model loaded') # Set the threshold for detection detection_threshold = 0.25 # The height and width of the training set images input_height = 300 input_width = 300 bucket_name = "citadel2019" s3 = boto3.client('s3') # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) # Compute the scale in order to draw bounding boxes on the full resolution # image. yscale = float(frame.shape[0]) / float(input_height) xscale = float(frame.shape[1]) / float(input_width) # Dictionary to be filled with labels and probabilities for MQTT cloud_output = {} # Get the detected faces and probabilities for obj in parsed_inference_results[model_type]: if obj['prob'] > detection_threshold: # Add bounding boxes to full resolution frame xmin = int(xscale * obj['xmin']) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) ymax = int(yscale * obj['ymax']) crop_img = frame[ymin:ymax, xmin:xmax] cv2.imwrite("/tmp/test.jpg", crop_img) #client.publish(topic=iot_topic, payload="YES, /tmp/test.jpg") s3.upload_file("/tmp/test.jpg", bucket_name, "whitewalker.jpg") #invokeLam = boto3.client("lambda", region_name="us-east-1") #payload = {"message": "Hi, you have been invoked."} #For InvocationType = "RequestResponse" #resp = invokeLam.invoke(FunctionName = "upload_s3", InvocationType = "Event", Payload = json.dumps(payload)) #client.publish(topic=iot_topic, payload="YES, trigger") # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.rectangle method. # Method signature: image, point1, point2, color, and tickness. cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 10) # Amount to offset the label/probability text above the bounding box. text_offset = 15 # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, # and tickness cv2.putText(frame, '{:.2f}%'.format(obj['prob'] * 100), (xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6) # Store label and probability to send to cloud cloud_output[output_map[obj['label']]] = obj['prob'] # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send results to the cloud client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: client.publish(topic=iot_topic, payload='Error in face detection lambda: {}'.format(ex))
def greengrass_infinite_infer_run(): try: modelPath = "/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml" modelType = "ssd" input_width = 300 input_height = 300 results_thread = FIFO_Thread() results_thread.start() print '** Object detection starts now' try: voice.say_phrase('loading') # threading.Thread(target=voice.say_phrase, args=('loading', )).start() except Exception as e: print '** Unable to say loading' mcfg = {"GPU": 1} model = awscam.Model(modelPath, mcfg) print '** Model loaded' ret, frame = awscam.getLastFrame() if ret == False: raise Exception("Failed to get frame from the stream") print '** Got last frame' yscale = float(frame.shape[0] / input_height) xscale = float(frame.shape[1] / input_width) firstGo = True spotted_objects_last_time = {} print '** Inference now starting' doInfer = True while doInfer: spotted_objects_this_time = {} if firstGo: print '** First time - about to play welcome message' try: questions.intro() # threading.Thread(target=questions.intro).start() except Exception as e: print "!! Unable to play intro: " + str(e) print '** First time - finished playing welcome message' firstGo = False # Get a frame from the video stream ret, frame = awscam.getLastFrame() # Raise an exception if failing to get a frame if ret == False: print "!! Failed to get frame from the stream" raise Exception("Failed to get frame from the stream") # Resize frame to fit model input requirement frameResize = cv2.resize(frame, (input_width, input_height)) # Run model inference on the resized frame inferOutput = model.doInference(frameResize) # Output inference result to the fifo file so it can be viewed with mplayer parsed_results = model.parseResult(modelType, inferOutput)['ssd'] # print "Before parsed results..." for obj in parsed_results: object_name = out_map[obj['label']] if object_name in thresholds: if obj['prob'] > thresholds[object_name]: object_name = out_map[obj['label']] spotted_objects_this_time[object_name] = obj['prob'] xmin = int(xscale * obj['xmin']) + int( (obj['xmin'] - input_width / 2) + input_width / 2) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) + int( (obj['xmax'] - input_width / 2) + input_width / 2) ymax = int(yscale * obj['ymax']) cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 4) label_show = "XX:{}: {:.2f}%".format( object_name, obj['prob'] * 100) cv2.putText(frame, label_show, (xmin, ymin - 15), cv2.FONT_HERSHEY_SIMPLEX, 1.2, (255, 165, 20), 4) global jpeg ret, jpeg = cv2.imencode('.jpg', frame) # print "...parsing results finished" try: considerChangeInObjects(spotted_objects_last_time, spotted_objects_this_time) except Exception as e: print '!! Problem considering change in objects: ' + str(e) spotted_objects_last_time = spotted_objects_this_time except IOError as e: print "FATAL I/O error({0}): {1}".format(e.errno, e.strerror) except ValueError as e: print "FATAL ValueError: " + str(e) except 0: print "Got a 0" traceback.print_exc() except Exception as e: print "The non-0 exception was", e print "FException: " + str(e) print "Oh well - now running again:" greengrass_infinite_infer_run() # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run(): """ Entry point of the lambda function""" try: model_type = 'classification' model_name = 'image-classification' output_map = {0: 'weed', 1: 'grass'} # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format( os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # model_path = '/opt/awscam/artifacts/image-classification.xml' # The height and width of the training set images input_height = 224 input_width = 224 error, model_path = mo.optimize(model_name, input_height, input_width, aux_inputs={'--epoch': 10}) # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading image classification model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Image classification model loaded') # Set the threshold for classification classification_threshold = 0.25 # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) # Compute the scale in order to draw bounding boxes on the full resolution # image. yscale = float(frame.shape[0] / input_height) xscale = float(frame.shape[1] / input_width) # Dictionary to be filled with labels and probabilities for MQTT cloud_output = {} # Get the classified images and probabilities for obj in parsed_inference_results[model_type]: if obj['prob'] > classification_threshold: # Store label and probability to send to cloud cloud_output[output_map[obj['label']]] = obj['prob'] # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send results to the cloud client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: client.publish( topic=iot_topic, payload='Error in image classification lambda: {}'.format(ex))
def infinite_infer_run(): """ Entry point of the lambda function""" try: # This cat-dog model is implemented as binary classifier, since the number # of labels is small, create a dictionary that converts the machine # labels to human readable labels. model_type = 'classification' output_map = {0: 'dog', 1: 'cat'} # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The sample projects come with optimized artifacts, hence only the artifact # path is required. model_path = '/opt/awscam/artifacts/mxnet_resnet18-catsvsdogs_FP32_FUSED.xml' # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading action cat-dog model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Cat-Dog model loaded') # Since this is a binary classifier only retrieve 2 classes. num_top_k = 2 # The height and width of the training set images input_height = 224 input_width = 224 # Do inference until the lambda is killed. while True: # inference loop to add. See the next step ... except Exception as ex: client.publish(topic=iot_topic, payload='Error in cat-dog lambda: {}'.format(ex)) # snippet-end:[deeplens.python.deeplens_inference_lambda.inference_loop] # snippet-start:[deeplens.python.deeplens_inference_lambda.inference_step] # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a classification model, # a simple API is provided. parsed_inference_results = model.parseResult(model_type, model.doInference(frame_resize)) # Get top k results with highest probabilities top_k = parsed_inference_results[model_type][0:num_top_k] # Add the label of the top result to the frame used by local display. # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, and tickness cv2.putText(frame, output_map[top_k[0]['label']], (10, 70), cv2.FONT_HERSHEY_SIMPLEX, 3, (255, 165, 20), 8) # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send the top k results to the IoT console via MQTT cloud_output = {} for obj in top_k: cloud_output[output_map[obj['label']]] = obj['prob'] client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
def infinite_infer_run(): global no_person_num, no_person_discontinue, multi_person_num, \ multi_person_discontinue, cellphone_num, cellphone_discontinue, \ book_num, book_discontinue, multi_monitor_num, multi_monitor_discontinue """ Entry point of the lambda function""" try: # This object detection model is implemented as single shot detector (ssd), since # the number of labels is small we create a dictionary that will help us convert # the machine labels to human readable labels. model_type = 'ssd' # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The sample projects come with optimized artifacts, hence only the artifact # path is required. model_path = '/home/aws_cam/public/ssd_mobilenet_v2_coco/FP16/ssd_mobilenet_v2_coco.xml' # Load the model onto the GPU. print('Loading object detection model') start = time.time() model = awscam.Model(model_path, {'GPU': 1}) print('Object detection model loaded, Elapsed: ' + str(time.time() - start)) # The height and width of the training set images input_height = 300 input_width = 300 # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) elapsed = time.time() - start # Compute the scale in order to draw bounding boxes on the full resolution # image. yscale = float(frame.shape[0]) / float(input_height) xscale = float(frame.shape[1]) / float(input_width) persons = [] monitors = [] cellphones = [] books = [] draw_objects = [] display_text = [] # Get the detected objects and probabilities for obj in parsed_inference_results[model_type]: # Add bounding boxes to full resolution frame xmin = int(xscale * obj['xmin']) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) ymax = int(yscale * obj['ymax']) if obj['label'] == PERSON_LABEL and obj[ 'prob'] > PERSON_THRESHOLD: persons.append((xmin, xmax, ymin, ymax, obj['prob'])) elif (obj['label'] == TV_LABEL or obj['label'] == LAPTOP_LABEL) and obj['prob'] > TV_LAPTOP_THRESHOLD: monitors.append((xmin, xmax, ymin, ymax, obj['prob'])) elif obj['label'] == CELLPHONE_LABEL and obj[ 'prob'] > CELLPHONE_THRESHOLD: cellphones.append((xmin, xmax, ymin, ymax, obj['prob'])) elif obj['label'] == BOOK_LABEL and obj[ 'prob'] > BOOK_THRESHOLD: books.append((xmin, xmax, ymin, ymax, obj['prob'])) if len(persons) < 1: no_person_num += 1 no_person_discontinue = 0 elif no_person_num > 0: no_person_discontinue += 1 if no_person_discontinue == no_person_discontinue_max: no_person_num = 0 no_person_discontinue = 0 if len(persons) > 1: multi_person_num += 1 multi_person_discontinue = 0 elif multi_person_num > 0: multi_person_discontinue += 1 if multi_person_discontinue == multi_person_discontinue_max: multi_person_num = 0 multi_person_discontinue = 0 if len(monitors) > 1: multi_monitor_num += 1 multi_monitor_discontinue = 0 elif multi_monitor_num > 0: multi_monitor_discontinue += 1 if multi_monitor_discontinue == multi_monitor_discontinue_max: multi_monitor_num = 0 multi_monitor_discontinue = 0 if len(cellphones) > 0: cellphone_num += 1 cellphone_discontinue = 0 elif cellphone_num > 0: cellphone_discontinue += 1 if cellphone_discontinue == cellphone_discontinue_max: cellphone_num = 0 cellphone_discontinue = 0 if len(books) > 1: book_num += 1 book_discontinue = 0 elif book_num > 0: book_discontinue += 1 if book_discontinue == book_discontinue_max: book_num = 0 book_discontinue = 0 draw_objects.extend(persons) if no_person_num >= no_person_max: display_text.append('Exam taker left') if multi_person_num >= multi_person_max: display_text.append('multiple people detected') if multi_monitor_num >= multi_monitor_max: display_text.append('multiple PC monitors/laptops detected') draw_objects.extend(monitors) if cellphone_num >= cellphone_max: display_text.append('cellphone detected') draw_objects.extend(cellphones) if book_num >= book_max: display_text.append('book detected') draw_objects.extend(books) if len(display_text) > 0: position = 0 for text in display_text: position += 60 cv2.putText(frame, text, (0, position), cv2.FONT_HERSHEY_SIMPLEX, 2.5, (0, 0, 255), 6) else: cv2.putText(frame, 'No problem detected', (0, 60), cv2.FONT_HERSHEY_SIMPLEX, 2.5, (0, 255, 0), 6) draw_color = (0, 0, 255) if len(display_text) > 0 else (0, 255, 0) for box in draw_objects: xmin, xmax, ymin, ymax, score = box # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.rectangle method. # Method signature: image, point1, point2, color, and tickness. cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), draw_color, 10) # Amount to offset the label/probability text above the bounding box. text_offset = 15 # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, # and tickness cv2.putText(frame, "{:.2f}%".format(score * 100), (xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX, 2.5, draw_color, 6) # Set the next frame in the local display stream. local_display.set_frame_data(frame) except Exception as ex: print('Error in object detection lambda: {}'.format(ex))
def greengrass_infinite_infer_run(): try: input_width = 224 input_height = 224 model_name = "image-classification" error, model_path = mo.optimize(model_name, input_width, input_height, aux_inputs={'--epoch': 3}) # Load model to GPU (use {"GPU": 0} for CPU) mcfg = {"GPU": 1} model = awscam.Model(model_path, mcfg) client.publish(topic=iotTopic, payload="Model loaded") model_type = "classification" with open('caltech256_labels.txt', 'r') as f: labels = [l.rstrip() for l in f] topk = 5 results_thread = FIFO_Thread() results_thread.start() # Send a starting message to IoT console client.publish(topic=iotTopic, payload="Inference is starting") doInfer = True while doInfer: # Get a frame from the video stream ret, frame = awscam.getLastFrame() # Raise an exception if failing to get a frame if ret == False: raise Exception("Failed to get frame from the stream") # Resize frame to fit model input requirement frameResize = cv2.resize(frame, (input_width, input_height)) # Run model inference on the resized frame inferOutput = model.doInference(frameResize) # Output inference result to the fifo file so it can be viewed with mplayer parsed_results = model.parseResult(model_type, inferOutput) top_k = parsed_results[model_type][0:topk] msg = '{' prob_num = 0 for obj in top_k: if prob_num == topk - 1: msg += '"{}": {:.2f}'.format(labels[obj["label"]], obj["prob"] * 100) else: msg += '"{}": {:.2f},'.format(labels[obj["label"]], obj["prob"] * 100) prob_num += 1 msg += "}" client.publish(topic=iotTopic, payload=msg) cv2.putText(frame, labels[top_k[0]["label"]], (0, 22), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 165, 20), 4) global jpeg ret, jpeg = cv2.imencode('.jpg', frame) except Exception as e: msg = "myModel Lambda failed: " + str(e) client.publish(topic=iotTopic, payload=msg) # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run(): try: modelPath = "/opt/awscam/artifacts/mxnet_deploy_ssd_FP16_FUSED.xml" modelType = "ssd" input_width = 300 input_height = 300 prob_thresh = 0.1 results_thread = FIFO_Thread() results_thread.start() # Send a starting message to IoT console client.publish(topic=iotTopic, payload="Face detection starts now") # Load model to GPU (use {"GPU": 0} for CPU) mcfg = {"GPU": 1} model = awscam.Model(modelPath, mcfg) client.publish(topic=iotTopic, payload="Model loaded") ret, frame = awscam.getLastFrame() if ret == False: raise Exception("Failed to get frame from the stream") yscale = float(frame.shape[0] / input_height) xscale = float(frame.shape[1] / input_width) font = cv2.FONT_HERSHEY_SIMPLEX rgb_color = (255, 165, 20) #Timers for cooldown and countdown cooldown = datetime.datetime.now() countdown = datetime.datetime.now() doInfer = True onCountdown = False while doInfer: # Get a frame from the video stream ret, frame = awscam.getLastFrame() # Raise an exception if failing to get a frame if ret == False: raise Exception("Failed to get frame from the stream") # Resize frame to fit model input requirement frameResize = cv2.resize(frame, (input_width, input_height)) # Run model inference on the resized frame inferOutput = model.doInference(frameResize) # Output inference result to the fifo file so it can be viewed with mplayer parsed_results = model.parseResult(modelType, inferOutput)['ssd'] label = '{' msg = 'false' time_now = datetime.datetime.now() for obj in parsed_results: if (obj['prob'] < prob_thresh): break xmin = int(xscale * obj['xmin']) + int( (obj['xmin'] - input_width / 2) + input_width / 2) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) + int( (obj['xmax'] - input_width / 2) + input_width / 2) ymax = int(yscale * obj['ymax']) cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), rgb_color, 4) label += '"{}": {:.2f},'.format("prob", obj['prob']) label_show = '{}: {:.2f}'.format(str(obj['label']), obj['prob']) cv2.putText(frame, label_show, (xmin, ymin - 15), font, 0.5, rgb_color, 4) msg = "true" if time_now >= cooldown: # Uploading to Amazon S3 if cooldown and countdown allow it if onCountdown and time_now >= countdown: message = "uploading to s3..." client.publish(topic=iotTopic, payload=message) key = 'images/frame-' + time.strftime( "%Y%m%d-%H%M%S") + '.jpg' session = Session() s3 = session.create_client('s3') _, jpg_data = cv2.imencode('.jpg', frame) result = s3.put_object(Body=jpg_data.tostring(), Bucket=bucket_name, Key=key) message = "uploaded to s3: " + key client.publish(topic=iotTopic, payload=message) cooldown = time_now + datetime.timedelta(seconds=10) onCountdown = False # Starting countdown elif not onCountdown: onCountdown = True countdown = time_now + datetime.timedelta(seconds=4) if not onCountdown: cv2.putText( frame, "Wait for picture: " + str(max(0, int( (cooldown - time_now).total_seconds()))) + " seconds", (950, 100), font, 2, rgb_color, 4) if int((cooldown - time_now).total_seconds()) >= 5: cv2.putText(frame, "Image Uploaded! ", (1150, 200), font, 2, rgb_color, 4) cv2.putText(frame, "Please check the leaderboard", (900, 300), font, 2, rgb_color, 4) else: if int((countdown - time_now).total_seconds()) >= -5: cv2.putText(frame, "Say Cheese!", (1000, 1000), font, 3, rgb_color, 4) cv2.putText( frame, str(max(0, int( (countdown - time_now).total_seconds()))) + "...", (1200, 1100), font, 3, rgb_color, 4) else: onCountdown = False label += '"face": "' + msg + '"' label += '}' client.publish(topic=iotTopic, payload=label) global jpeg ret, jpeg = cv2.imencode('.jpg', frame) except Exception as e: msg = "Test failed: " + str(e) client.publish(topic=iotTopic, payload=msg) # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run(): try: modelPath = "/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml" modelType = "ssd" input_width = 300 input_height = 300 max_threshold = 0.60 # raise/lower this value based on your conditions outMap = { 1: 'aeroplane', 2: 'bicycle', 3: 'bird', 4: 'boat', 5: 'bottle', 6: 'bus', 7: 'car', 8: 'cat', 9: 'chair', 10: 'cow', 11: 'dinning table', 12: 'dog', 13: 'horse', 14: 'motorbike', 15: 'person', 16: 'pottedplant', 17: 'sheep', 18: 'sofa', 19: 'train', 20: 'tvmonitor' } results_thread = FIFO_Thread() results_thread.start() # Load model to GPU mcfg = {"GPU": 1} model = awscam.Model(modelPath, mcfg) # try to get a frame from the camera ret, frame = awscam.getLastFrame() #flip the screen if is_deeplens_upside_down == 'yes': frame = cv2.flip(frame, -1) if ret == False: raise Exception("Failed to get frame from the stream") yscale = float(frame.shape[0] / input_height) xscale = float(frame.shape[1] / input_width) doInfer = True while doInfer: # Get a frame from the video stream ret, frame = awscam.getLastFrame() #Rotate the frame if the camera is mounted upside down if is_deeplens_upside_down == 'yes': frame = cv2.flip(frame, -1) # Raise an exception if failing to get a frame if ret == False: raise Exception("Failed to get frame from the stream") # Resize frame to fit model input requirement frameResize = cv2.resize(frame, (input_width, input_height)) # Run model inference on the resized frame inferOutput = model.doInference(frameResize) # Output inference result to the fifo file so it can be viewed with mplayer parsed_results = model.parseResult(modelType, inferOutput)['ssd'] label = '{' for obj in parsed_results: if obj['prob'] > max_threshold: xmin = int(xscale * obj['xmin']) + int( (obj['xmin'] - input_width / 2) + input_width / 2) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) + int( (obj['xmax'] - input_width / 2) + input_width / 2) ymax = int(yscale * obj['ymax']) # if a person was found, upload the target area to S3 for further inspection if outMap[obj['label']] == 'person': # get the person image person = frame[ymin:ymax, xmin:xmax] # create a nice s3 file key s3_key = datetime.datetime.utcnow().strftime( '%Y-%m-%d_%H_%M_%S.%f') + '.jpg' encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 90] # 90% should be more than enough _, jpg_data = cv2.imencode('.jpg', person, encode_param) filename = "incoming/%s" % s3_key # the guess lambda function is listening here response = s3.put_object(ACL='public-read', Body=jpg_data.tostring(), Bucket=s3_bucket, Key=filename) time.sleep(2) # draw a rectangle around the designated area, and tell what label was found cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 4) label += '"{}": {:.2f},'.format(outMap[obj['label']], obj['prob']) label_show = "{}: {:.2f}%".format( outMap[obj['label']], obj['prob'] * 100) cv2.putText(frame, label_show, (xmin, ymin - 15), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 165, 20), 4) label += '"null": 0.0' label += '}' global jpeg ret, jpeg = cv2.imencode('.jpg', frame) except Exception as e: print "Crap, something failed: %s" % str(e) # Asynchronously schedule this function to be run again in 15 seconds Timer(10, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run(): try: input_width = 224 input_height = 224 model_name = "image-classification" model_type = "classification" client = greengrasssdk.client('iot-data') iotTopic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME']) local_display = LocalDisplay('480p') local_display.start() error, model_path = mo.optimize(model_name,input_width,input_height, aux_inputs={'--epoch': 30}) mcfg = {"GPU": 1} print("model_path: " + model_path) model = awscam.Model(model_path, mcfg) client.publish(topic=iotTopic, payload="Model loaded") with open('pinehead_labels.txt', 'r') as f: labels = [l.rstrip() for l in f] num_top_k = 2 # Send a starting message to IoT console client.publish(topic=iotTopic, payload="Inference is starting") doInfer = True while doInfer: # Get a frame from the video stream ret, frame = awscam.getLastFrame() # Raise an exception if failing to get a frame if ret == False: raise Exception("Failed to get frame from the stream") # Resize frame to fit model input requirement s = frame.shape cropped = frame[0:s[0], int((s[1]-s[0])/2):s[0]+int((s[1]-s[0])/2)] frameResize = cv2.resize(cropped, (input_width, input_height)) # Run model inference on the resized frame inferOutput = model.doInference(frameResize) parsed_inference_results = model.parseResult(model_type, model.doInference(frameResize)) top_k = parsed_inference_results[model_type][0:num_top_k-1] msg = "{" msg += '"{}"'.format(labels[top_k[0]["label"]]) msg += "}" client.publish(topic=iotTopic, payload = msg) font = cv2.FONT_HERSHEY_SIMPLEX cv2.putText(frame, labels[top_k[0]["label"]], (10, 140), font, 5, (174, 235, 52), 10) local_display.set_frame_data(frame) except Exception as e: msg = "myModel Lambda failed: " + str(e) client.publish(topic=iotTopic, payload=msg) # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()
def infinite_infer_run(): """ Entry point of the lambda function""" try: # This object detection model is implemented as single shot detector (ssd), since # the number of labels is small we create a dictionary that will help us convert # the machine labels to human readable labels. model_type = 'ssd' output_map = {15: 'person'} # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format( os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The sample projects come with optimized artifacts, hence only the artifact # path is required. model_path = '/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml' # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading object detection model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Object detection model loaded') # Set the threshold for detection detection_threshold = 0.60 # The height and width of the training set images input_height = 300 input_width = 300 # set pause = 0 pause = 0 pause_counter = 0 #twilio data twilio_sms_url = "https://api.twilio.com/2010-04-01/Accounts/{}/Messages.json" twilio_account_sid = os.environ['TWILIO_ACCOUNT_SID'] twilio_auth_token = os.environ['TWILIO_AUTH_TOKEN'] twilio_my_number = os.environ['MY_DIGITS'] twilio_number = os.environ['TWILIO_DIGITS'] # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) # Compute the scale in order to draw bounding boxes on the full resolution # image. yscale = float(frame.shape[0]) / float(input_height) xscale = float(frame.shape[1]) / float(input_width) # Dictionary to be filled with labels and probabilities for MQTT cloud_output = {} # Get the detected objects and probabilities for obj in parsed_inference_results[model_type]: if obj['label'] == 15: if (obj['prob'] > detection_threshold) and (pause == 0): # Add bounding boxes to full resolution frame xmin = int(xscale * obj['xmin']) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) ymax = int(yscale * obj['ymax']) # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.rectangle method. # Method signature: image, point1, point2, color, and tickness. cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 10) # Amount to offset the label/probability text above the bounding box. text_offset = 15 # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, # and tickness cv2.putText( frame, "{}: {:.2f}%".format(output_map[obj['label']], obj['prob'] * 100), (xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6) # Store label and probability to send to cloud cloud_output[output_map[obj['label']]] = obj['prob'] #send frame to S3 timestamp = datetime.datetime.today().strftime( "%m%d%Y%H%M") cv2.imwrite("/tmp/detected.jpeg", frame) s3.upload_file('/tmp/detected.jpeg', 'deeplens-sagemaker-davidadkins', 'detected-deeplens-' + timestamp + ".jpg", ExtraArgs={ 'ACL': 'public-read', 'ContentType': 'image/jpeg' }) #send Twilio Text Message # insert Twilio Account SID into the REST API URL populated_url = twilio_sms_url.format( twilio_account_sid) post_params = { "To": twilio_my_number, "From": twilio_number, "Body": "Person detected.", "MediaUrl": 'https://deeplens-sagemaker-davidadkins.s3.amazonaws.com/detected-deeplens-' + timestamp + '.jpg' } # encode the parameters for Python's urllib data = urllib.urlencode(post_params).encode() req = urllib2.Request(populated_url) # add authentication header to request based on Account SID + Auth Token authentication = "{}:{}".format( twilio_account_sid, twilio_auth_token) base64string = base64.b64encode( authentication.encode('utf-8')) req.add_header( "Authorization", "Basic %s" % base64string.decode('ascii')) try: # perform HTTP POST request f = urllib2.urlopen(req, data) client.publish(topic=iot_topic, payload="Twilio returned {}".format( str(f.read().decode('utf-8')))) except Exception as e: # something went wrong! client.publish( topic=iot_topic, payload="Twilio error: {}".format(e)) pause = 1 # Set the next frame in the local display stream. local_display.set_frame_data(frame) #handle pauses if pause == 1: pause_counter = pause_counter + 1 if pause_counter > 100: pause = 0 pause_counter = 0 # Send results to the cloud client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: client.publish( topic=iot_topic, payload='Error in object detection lambda: {}'.format(ex))
def infinite_infer_run(): """ Entry point of the lambda function""" try: # This object detection model is implemented as single shot detector (ssd), since # the number of labels is small we create a dictionary that will help us convert # the machine labels to human readable labels. model_type = 'ssd' output_map = {0: 'fish'} # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format( os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The sample projects come with optimized artifacts, hence only the artifact # path is required. input_height = 300 input_width = 300 # optimize the model client.publish(topic=iot_topic, payload='Optimizing model...') ret, model_path = mo.optimize('deploy_ssd_resnet50_300', input_width, input_height) # model_path = '/opt/awscam/artifacts/deploy_ssd_resnet50_300.xml' # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading object detection model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Object detection model loaded') # Set the threshold for detection detection_threshold = 0.5 # Load the model onto the GPU. # The height and width of the training set images # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) # Compute the scale in order to draw bounding boxes on the full resolution # image. yscale = float(frame.shape[0]) / float(input_height) xscale = float(frame.shape[1]) / float(input_width) # Dictionary to be filled with labels and probabilities for MQTT cloud_output = {} # Get the detected objects and probabilities for obj in parsed_inference_results[model_type]: if obj['prob'] > detection_threshold: # Add bounding boxes to full resolution frame xmin = int(xscale * obj['xmin']) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) ymax = int(yscale * obj['ymax']) # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.rectangle method. # Method signature: image, point1, point2, color, and tickness. cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 10) # Amount to offset the label/probability text above the bounding box. text_offset = 10 # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, # and tickness cv2.putText( frame, "{}: {:.2f}%".format(output_map[obj['label']], obj['prob'] * 100), (xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6) # Store label and probability to send to cloud cloud_output[output_map[obj['label']]] = obj['prob'] # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send results to the cloud client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: client.publish( topic=iot_topic, payload='Error in object detection lambda: {}'.format(ex))