def infinite_infer_run(): """ Run the DeepLens inference loop frame by frame""" try: # Number of top classes to output num_top_k = 2 model_type = 'classification' model_name = 'image-classification' with open('labels.txt', 'r') as f: output_map = [l for l in f] # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # Optimize the model error, model_path = mo.optimize(model_name, INPUT_WIDTH, INPUT_HEIGHT) # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Model loaded') while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (INPUT_HEIGHT, INPUT_WIDTH)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a classification model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) # Get top k results with highest probabilities top_k = parsed_inference_results[model_type][0:num_top_k] # Add the label of the top result to the frame used by local display. # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, and thickness output_text = '{} : {:.2f}'.format(output_map[top_k[0]['label']], top_k[0]['prob']) cv2.putText(frame, output_text, (10, 70), cv2.FONT_HERSHEY_SIMPLEX, 3, (255, 165, 20), 8) # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send the top k results to the IoT console via MQTT cloud_output = {} for obj in top_k: cloud_output[output_map[obj['label']]] = obj['prob'] client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: print('Error in lambda {}'.format(ex)) client.publish(topic=iot_topic, payload='Error in lambda: {}'.format(ex))
def greengrass_infinite_infer_run(): try: global clients input_width = 224 input_height = 224 model_name = "image-classification" error, model_path = mo.optimize(model_name, input_width, input_height) if error: print error raise Exception(error) print("Optimized model") model = awscam.Model(model_path, {"GPU": 1}) print("Loaded model") model_type = "classification" labels = ['a','b','c','d','e','f','g','h','i','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','not'] topk = 1 while True: start = datetime.now() ret, frame = awscam.getLastFrame() if ret == False: raise Exception("Failed to get frame from the stream") height, width, channels = frame.shape frame_cropped = frame[0:height, (width-height)/2:width-(width-height)/2] frame_resize = cv2.resize(frame_cropped, (input_width, input_height)) infer_output = model.doInference(frame_resize) parsed_results = model.parseResult(model_type, infer_output) top_k = parsed_results[model_type][0:topk] ret, jpeg = cv2.imencode(".jpg", frame_resize) try: end = datetime.now() top_k[0]["label"] = labels[top_k[0]["label"]]; top_k[0]["jpeg"] = b64encode(jpeg) top_k[0]["delay"] = (end - start).microseconds / 1000 msg = json.dumps(top_k[0]) + u'' for client in clients: client.sendMessage(msg) except: print("error", sys.exc_info()[0]) except Exception as e: print("Lambda failed: " + str(e)) # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run(): try: modelType = "classification" model_name = "image-classification" input_width = 224 input_height = 224 max_threshold = 0.75 error, model_path = mo.optimize(model_name,input_width,input_height) outMap = { 0: "a", 1: "b", 2: "c", 3: "d", 4: "e", 5: "f", 6: "g", 7 : "h", 8 : "i", 9 : "k", 10 : "l", 11 : "m", 12 : "n", 13 : "o", 14 : "p", 15 : "q", 16 : "r", 17 : "s", 18 : "t", 19 : "u", 20 : "v", 21 : "w", 22 : "x", 23 : "y", 24 : "z" } client.publish(topic=iotTopic, payload="Object detection starts now") mcfg = {"GPU": 1} model = awscam.Model(model_path, mcfg) client.publish(topic=iotTopic, payload="Model loaded") ret, frame = awscam.getLastFrame() if ret == False: raise Exception("Failed to get frame from the stream") doInfer = True while doInfer: # Get a frame from the video stream ret, frame = awscam.getLastFrame() # Raise an exception if failing to get a frame if ret == False: raise Exception("Failed to get frame from the stream") margin = (frame.shape[1] - frame.shape[0]) / 2 cropped = frame[0:frame.shape[0], margin:frame.shape[1] - margin] frameResize = cv2.resize(cropped, (input_width, input_height)) inferOutput = model.doInference(frameResize) parsed_results = model.parseResult(modelType, inferOutput)["classification"] client.publish(topic=iotTopic, payload = str(parsed_results)) for obj in parsed_results: if obj["prob"] > max_threshold and detected != outMap[obj["label"]]: detected = outMap[obj["label"]] client.publish(topic=iotTopic, payload = outMap[obj["label"]]) playsound("../letters/" + outMap[obj["label"]] + ".mp3"); except Exception as e: msg = "Test failed: " + str(e) client.publish(topic=iotTopic, payload=msg) # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run(): """ Entry point of the lambda function""" client.publish(topic=iot_topic, payload='Start of run loop...') try: # This object detection model is implemented as single shot detector (ssd), since # the number of labels is small we create a dictionary that will help us convert # the machine labels to human readable labels. model_type = 'ssd' output_map = {} with open('classes.txt') as f: for line in f: (key, val) = line.split() output_map[int(key)] = val client.publish(topic=iot_topic, payload='Classes to be detected: ' + str(output_map)) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The height and width of the training set images input_height = 512 input_width = 512 # Load the model onto the GPU. # optimize the model client.publish(topic=iot_topic, payload='Optimizing model...') ret, model_path = mo.optimize('deploy_ssd_resnet50_512', input_width, input_height) # load the model client.publish(topic=iot_topic, payload='Loading model...') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Custom object detection model loaded') # Set the threshold for detection detection_threshold = 0.40 # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API. Note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. client.publish(topic=iot_topic, payload='Calling inference on next frame...') parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) # Compute the scale in order to draw bounding boxes on the full resolution # image. yscale = float(frame.shape[0]) / float(input_height) xscale = float(frame.shape[1]) / float(input_width) # Dictionary to be filled with labels and probabilities for MQTT cloud_output = {} # Get the detected objects and probabilities default_color = (255, 165, 20) i = 0 for obj in parsed_inference_results[model_type]: if obj['prob'] > detection_threshold: # Add bounding boxes to full resolution frame xmin = int(xscale * obj['xmin']) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) ymax = int(yscale * obj['ymax']) # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.rectangle method. # Method signature: image, point1, point2, color, and tickness. cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), default_color, 10) # Amount to offset the label/probability text above the bounding box. text_offset = 15 # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, # and thickness cv2.putText( frame, "{}: {:.2f}%".format(output_map[obj['label']], obj['prob'] * 100), (xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6) # Store label and probability to send to cloud cloud_output[output_map[obj['label']]] = obj['prob'] # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send results to the cloud client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: client.publish( topic=iot_topic, payload='Error in object detection lambda: {}'.format(ex))
def infinite_infer_run(): """ Entry point of the lambda function""" try: # This bird detection model is implemented as multi classifier. The number of labels # is quite large so we upload them to a list to map the machine labels to human readable # labels. model_type = 'classification' with open('labels.txt', 'r') as labels_file: output_map = [class_label.rstrip() for class_label in labels_file] # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format( os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The height and width of the training set images input_height = 224 input_width = 224 # The sample projects come with optimized artifacts, hence only the artifact # path is required. ret, model_path = mo.optimize('bird_classification_resnet-18', input_width, input_height, 'mx') # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading bird detection model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Bird detection loaded') # The number of top results to stream to IoT. num_top_k = 5 # Define the detection region size. region_size = 800 # Define the inference display region size. This size was decided based on the longest label. label_region_width = 940 label_region_height = 600 # Heading for the inference display. prediction_label = 'Top 5 bird predictions' # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Crop the detection region for inference. frame_crop = frame[int(frame.shape[0]/2-region_size/2):int(frame.shape[0]/2+region_size/2), \ int(frame.shape[1]/2-region_size/2):int(frame.shape[1]/2+region_size/2), :] # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame_crop, (input_height, input_width)) # Model was trained in RGB format but getLastFrame returns image # in BGR format so need to switch. frame_resize = cv2.cvtColor(frame_resize, cv2.COLOR_BGR2RGB) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a classification model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) # Get top k results with highest probabilities top_k = parsed_inference_results[model_type][0:num_top_k] # Create a copy of the original frame. overlay = frame.copy() # Create the rectangle that shows the inference results. cv2.rectangle(overlay, (0, 0), \ (int(label_region_width), int(label_region_height)), (211,211,211), -1) # Blend with the original frame. opacity = 0.7 cv2.addWeighted(overlay, opacity, frame, 1 - opacity, 0, frame) # Add the header for the inference results. cv2.putText(frame, prediction_label, (0, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 4) # Add the label along with the probability of the top result to the frame used by local display. # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, and tickness for i in range(num_top_k): cv2.putText(frame, output_map[top_k[i]['label']] + ' ' + str(round(top_k[i]['prob'], 3) * 100) + '%', \ (0, 100*i+150), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 0, 0), 3) # Display the detection region. cv2.rectangle(frame, (int(frame.shape[1]/2-region_size/2), int(frame.shape[0]/2-region_size/2)), \ (int(frame.shape[1]/2+region_size/2), int(frame.shape[0]/2+region_size/2)), (255,0,0), 5) # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send the top k results to the IoT console via MQTT cloud_output = {} for obj in top_k: cloud_output[output_map[obj['label']]] = obj['prob'] client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: client.publish(topic=iot_topic, payload='Error in bird detection lambda: {}'.format(ex))
Write_To_FIFO = True FIRST_RUN = True PLAY_INTRO = True input_width = 300 input_height = 300 prob_thresh = 0.55 outMap = {0: 'text_block'} model_name = "read-to-me" occursThreshold = 10 client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME']) jpeg = None logger = logging.getLogger() logger.setLevel(logging.INFO) error, model_path = mo.optimize(model_name, input_width, input_height) model = awscam.Model(model_path, {"GPU": 1}, awscam.Runtime.DLDT) def log_message(message): logger.info(message) # client.publish(topic=iot_topic, payload=message) def first_run(): log_message('first run') global FIRST_RUN try: if PLAY_INTRO: log_message('attempting to play audio files for first run') speak.playAudioFile(os.path.join('staticfiles', 'intro.mp3')) sleep(0.5) speak.playAudioFile(os.path.join('staticfiles', 'dir1.mp3'))
def greengrass_infinite_infer_run(): """ Entry point of the lambda function""" try: model_type = 'classification' model_name = 'image-classification' output_map = {0: 'weed', 1: 'grass'} # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format( os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # model_path = '/opt/awscam/artifacts/image-classification.xml' # The height and width of the training set images input_height = 224 input_width = 224 error, model_path = mo.optimize(model_name, input_height, input_width, aux_inputs={'--epoch': 10}) # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading image classification model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Image classification model loaded') # Set the threshold for classification classification_threshold = 0.25 # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) # Compute the scale in order to draw bounding boxes on the full resolution # image. yscale = float(frame.shape[0] / input_height) xscale = float(frame.shape[1] / input_width) # Dictionary to be filled with labels and probabilities for MQTT cloud_output = {} # Get the classified images and probabilities for obj in parsed_inference_results[model_type]: if obj['prob'] > classification_threshold: # Store label and probability to send to cloud cloud_output[output_map[obj['label']]] = obj['prob'] # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send results to the cloud client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: client.publish( topic=iot_topic, payload='Error in image classification lambda: {}'.format(ex))
def infinite_infer_run(): """ Entry point of the lambda function""" try: model_type = 'classification' output_map = { 0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer', 5: 'dog', 6: 'frog', 7: 'horse', 8: 'ship', 9: 'truck' } # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format( os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() error, model_path = mo.optimize('model', 32, 32) #model_path = '/opt/awscam/artifacts/mxnet_resnet18-catsvsdogs_FP32_FUSED.xml' # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading cifar model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='cifar model loaded') num_top_k = 10 # The height and width of the training set images input_height = 32 input_width = 32 # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a classification model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) # Get top k results with highest probabilities top_k = parsed_inference_results[model_type][0:num_top_k] # Add the label of the top result to the frame used by local display. # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, and tickness cv2.putText(frame, output_map[top_k[0]['label']], (10, 70), cv2.FONT_HERSHEY_SIMPLEX, 3, (255, 165, 20), 8) # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send the top k results to the IoT console via MQTT cloud_output = {} for obj in top_k: cloud_output[output_map[obj['label']]] = obj['prob'] client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: client.publish(topic=iot_topic, payload='Error in cat-dog lambda: {}'.format(ex))
def greengrass_infinite_infer_run(): try: # Optimizing model (you have to update MxNet with: pip3 install mxnet --upgrade) input_width = 224 input_height = 224 model_name = "deeplens-asl" error, model_path = mo.optimize(model_name, input_width, input_height) if error: raise Exception(error) # Loading model model = awscam.Model(model_path, {"GPU": 1}) client.publish(topic=iot_topic, payload="Model loaded") # Model variables model_type = "classification" labels = ['a','b','c','d','e','f','g','h','i','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','spok','like','f**k'] topk = 1 # Streaming results results_thread = FIFO_Thread() results_thread.start() client.publish(topic=iot_topic, payload="Inference is starting") # Preparing sentence display actual_letter = '' actual_frame_count = 0 message = '' minimum_count = 20 space_count = 30 reset_count = 70 total_frame_count = 0 # Inference loop while True: # Retrieving last frame total_frame_count = total_frame_count + 1 ret, frame = awscam.getLastFrame() if ret == False: raise Exception("Failed to get frame from the stream") # Resizing frame height, width, channels = frame.shape frame_cropped = frame[0:height, (width-height)/2:width-(width-height)/2] frame_resize = cv2.resize(frame_cropped, (input_width, input_height)) infer_output = model.doInference(frame_resize) parsed_results = model.parseResult(model_type, infer_output) top_k = parsed_results[model_type][0:topk] output_frame = frame_cropped # Filtering 40% confidence if (top_k[0]["prob"] > 0.4): # Writing inference results letter = labels[top_k[0]["label"]] if (actual_letter == letter): actual_frame_count = actual_frame_count + 1 if (actual_frame_count == minimum_count): message = message + letter else: actual_frame_count = 1 actual_letter = labels[top_k[0]["label"]] cv2.putText(output_frame, '{}: {:.2f}'.format(labels[top_k[0]["label"]], top_k[0]["prob"]), (0,20), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 165, 20), 4) # Publishing results to IoT topic #msg = "{" #prob_num = 0 #for obj in top_k: # if prob_num == topk-1: # msg += '"{}": {:.2f}'.format(labels[obj["label"]], obj["prob"]) # else: # msg += '"{}": {:.2f},'.format(labels[obj["label"]], obj["prob"]) # prob_num += 1 #msg += "}" #client.publish(topic=iot_topic, payload=msg) else: # Handling whitespaces if (actual_letter != ''): actual_frame_count = 0 actual_letter = '' else : actual_frame_count = actual_frame_count + 1 if (actual_frame_count == space_count and not message.endswith(" ")): message = message + " " if (actual_frame_count == reset_count): message = "" # Handling caret total_message = message if (total_frame_count % 4 < 2) : total_message = message + "_" # Writing sentence cv2.putText(output_frame, total_message, (0,height-50), cv2.FONT_HERSHEY_SIMPLEX, 4, (45, 145, 236), 4) # Streaming frame global jpeg ret, jpeg = cv2.imencode('.jpg', output_frame) except Exception as e: msg = "Lambda failed: " + str(e) client.publish(topic=iot_topic, payload=msg) # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run(): try: input_width = 224 input_height = 224 model_name = "image-classification" error, model_path = mo.optimize(model_name,input_width,input_height, aux_inputs={'--epoch': 10}) # The aux_inputs is equal to the number of epochs and in this case, it is 10 # Load model to GPU (use {"GPU": 0} for CPU) mcfg = {"GPU": 1} model = awscam.Model(model_path, mcfg) client.publish(topic=iotTopic, payload="Model loaded") model_type = "classification" with open('caltech256_labels.txt', 'r') as f: labels = [l.rstrip() for l in f] topk = 5 results_thread = FIFO_Thread() results_thread.start() # Send a starting message to IoT console client.publish(topic=iotTopic, payload="Inference is starting") doInfer = True while doInfer: # Get a frame from the video stream ret, frame = awscam.getLastFrame() # Raise an exception if failing to get a frame if ret == False: raise Exception("Failed to get frame from the stream") # Resize frame to fit model input requirement frameResize = cv2.resize(frame, (input_width, input_height)) # Run model inference on the resized frame inferOutput = model.doInference(frameResize) # Output inference result to the fifo file so it can be viewed with mplayer parsed_results = model.parseResult(model_type, inferOutput) top_k = parsed_results[model_type][0:topk] msg = '{' prob_num = 0 for obj in top_k: if prob_num == topk-1: msg += '"{}": {:.2f}'.format(labels[obj["label"]], obj["prob"]*100) else: msg += '"{}": {:.2f},'.format(labels[obj["label"]], obj["prob"]*100) prob_num += 1 msg += "}" client.publish(topic=iotTopic, payload = msg) cv2.putText(frame, labels[top_k[0]["label"]], (0,22), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 165, 20), 4) global jpeg ret,jpeg = cv2.imencode('.jpg', frame) except Exception as e: msg = "caltech256 Lambda failed: " + str(e) client.publish(topic=iotTopic, payload=msg) # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()
def infinite_infer_run(): """ Entry point of the lambda function""" try: # This object detection model is implemented as single shot detector (ssd), since # the number of labels is small we create a dictionary that will help us convert # the machine labels to human readable labels. model_type = 'ssd' output_map = {0: '1', 1: '2', 2: '3', 3: '4'} # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format( os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('square') local_display.start() # The height and width of the training set images input_height = 300 input_width = 300 # The sample projects come with optimized artifacts, hence only the artifact # path is required. model_path = '/opt/awscam/artifacts/deploy_ssd_resnet50_300.xml' if not os.path.exists(model_path): # Optimization needed client.publish( topic=iot_topic, payload='Model does not exist in "{}", optimizing...'.format( model_path)) res, model_path = mo.optimize('deploy_ssd_resnet50_300', input_height, input_width) if res == 0: client.publish( topic=iot_topic, payload= 'Model optimized successful. The optimized model is "{}"'. format(model_path)) # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading object detection model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Object detection model loaded') # Set the threshold for detection detection_threshold = 0.30 # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') frame = crop_frame_square(frame) # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) # Compute the scale in order to draw bounding boxes on the full resolution # image. yscale = float(frame.shape[0]) / float(input_height) xscale = float(frame.shape[1]) / float(input_width) # Dictionary to be filled with labels and probabilities for MQTT cloud_output = {} # Get the detected objects and probabilities for obj in parsed_inference_results[model_type]: if obj['prob'] > detection_threshold: # Add bounding boxes to full resolution frame xmin = int(xscale * obj['xmin']) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) ymax = int(yscale * obj['ymax']) # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.rectangle method. # Method signature: image, point1, point2, color, and tickness. cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 10) # Amount to offset the label/probability text above the bounding box. text_offset = 15 # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, # and tickness cv2.putText( frame, "{}: {:.2f}%".format(output_map[obj['label']], obj['prob'] * 100), (xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6) # Store label and probability to send to cloud cloud_output[output_map[obj['label']]] = obj['prob'] # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send results to the cloud client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: client.publish( topic=iot_topic, payload='Error in object detection lambda: {}. Stack trace: {}'. format(ex, traceback.format_exc().splitlines()))
def greengrass_infinite_infer_run(): try: model_type = "ssd" input_width = 224 input_height = 224 prob_thresh = 0.25 results_thread = FIFO_Thread() results_thread.start() haar_face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_alt.xml') # Send a starting message to IoT console client.publish(topic=iotTopic, payload="Face detection starts now") # Load model to GPU (use {"GPU": 0} for CPU) mcfg = {"GPU": 1} error, modelPath = mo.optimize('smile-net', input_width, input_height) model = awscam.Model(modelPath, mcfg) client.publish(topic=iotTopic, payload="Model loaded") ret, frame = awscam.getLastFrame() if ret == False: raise Exception("Failed to get frame from the stream") doInfer = True while doInfer: # Get a frame from the video stream ret, frame = awscam.getLastFrame() # Raise an exception if failing to get a frame if ret == False: raise Exception("Failed to get frame from the stream") img_copy = np.copy(frame) # convert the test image to gray image as opencv face detector expects gray images gray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) # let's detect multiscale (some images may be closer to camera than others) images faces = haar_face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5); # go over list of faces and draw them as rectangles on original colored img for face in faces: (x, y, w, h) = face face_img = img_copy[y:y + h, x:x + w] resize_face = cv2.resize(face_img, (224, 224)) infer_output = model.doInference(resize_face) parsed_results = model.parseResult(model_type, infer_output) prob = parsed_results[model_type][0]['prob'] if prob > prob_thresh: # smiling, show blueish cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 165, 20), 4) else: # Neutral, show orangish cv2.rectangle(frame, (x, y), (x + w, y + h), (20, 165, 255), 4) msg = "{" msg += '"{}":{:.2f},'.format('smile', prob) msg += "}" client.publish(topic=iotTopic, payload=msg) global jpeg ret, jpeg = cv2.imencode('.jpg', frame) except Exception as e: msg = "Test failed: " + str(e) client.publish(topic=iotTopic, payload=msg) # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run(): """ Entry point of the lambda function""" client.publish(topic=iot_topic, payload='Start of run loop...') try: # This object detection model model_type = "classification" # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The height and width of the training set images input_height = 512 input_width = 512 # optimize the model client.publish(topic=iot_topic, payload='Optimizing model...') model_name = "image-classification" error, model_path = mo.optimize(model_name, input_width, input_height, aux_inputs={'--epoch': 15}) # Load model to GPU (use {"GPU": 0} for CPU) mcfg = {"GPU": 1} model = awscam.Model(model_path, mcfg) client.publish(topic=iot_topic, payload='Custom object detection model loaded') # Load labels from text file with open('sock_labels.txt', 'r') as f: labels = [l.rstrip() for l in f] topk = 2 # Send a starting message to IoT console client.publish(topic=iot_topic, payload="Inference is starting") doInfer = True while doInfer: # Get a frame from the video stream ret, frame = awscam.getLastFrame() # Raise an exception if failing to get a frame if ret == False: raise Exception("Failed to get frame from the stream") # Resize frame to fit model input requirement frameResize = cv2.resize(frame, (input_width, input_height)) # Run model inference on the resized frame inferOutput = model.doInference(frameResize) # Output inference result to the fifo file so it can be viewed with mplayer parsed_results = model.parseResult(model_type, inferOutput) top_k = parsed_results[model_type][0:topk] sock_label = labels[top_k[0]["label"]] sock_prob = top_k[0]["prob"] * 100 # Write to MQTT json_payload = {"image": sock_label, "probability": sock_prob} client.publish(topic=iot_topic, payload=json.dumps(json_payload)) client2.publish(topic=mqttconfig.mqtt_topic, payload=json.dumps(json_payload), qos=0, retain=False) # Write to image buffer; screen display msg_screen = '{} {:.0f}%'.format(sock_label, sock_prob) cv2.putText(frame, msg_screen, (20, 200), cv2.FONT_HERSHEY_SIMPLEX, 5, (0, 0, 255), 12) local_display.set_frame_data(frame) except Exception as ex: client.publish( topic=iot_topic, payload='Error in object detection lambda: {}'.format(ex))
def greengrass_infinite_infer_run(): """ Entry point of the lambda function""" try: thing_name = os.environ['AWS_IOT_THING_NAME'] except Exception as ex: thing_name = "deeplens_cIqzAO10SVWwwHcsqXD5Jg" iot_topic = '$aws/things/{}/infer'.format(thing_name) if True: #try: # This object detection model is implemented as single shot detector (ssd), since # the number of labels is small we create a dictionary that will help us convert # the machine labels to human readable labels. model_type = 'ssd' output_map = {0: 'golf ball', 1: 'hole'} # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') #if os.environ['AWS_IOT_THING_NAME'] == "": # iot_topic = '$aws/things/deeplens_cIqzAO10SVWwwHcsqXD5Jg/infer' #else: # iot_topic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The height and width of the training set images #input_height = 720 base_width = 850 #input_width = 212 input_width = 150 input_height = input_width scale = base_width / float(input_width) # model_algo = "vgg16_reduced" model_algo = "resnet50" if os.path.isfile('/opt/awscam/artifacts/deploy_ssd_' + model_algo + '_' + str(input_width) + '.xml'): model_path = '/opt/awscam/artifacts/deploy_ssd_' + model_algo + '_' + str( input_width) + '.xml' else: #ret, model_path = mo.optimize('deploy_ssd_' + model_algo + '_' + str(input_width),input_width, input_height, aux_inputs={'--img-channels': 2}) ret, model_path = mo.optimize( 'deploy_ssd_' + model_algo + '_' + str(input_width), input_width, input_height) #model_path = '/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml' # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading object detection model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Object detection model loaded') # Set the threshold for detection detection_threshold = 0.5 # Do inference until the lambda is killed. ballIn = False ballInTime = datetime.datetime(1970, 1, 1) countOverlap = 0 mod = 1 send_no = False while True: print(" ") timeNow = datetime.datetime.now() timeDiff = timeNow - ballInTime if (timeDiff.total_seconds() >= 5): ballIn = False ballInTime = datetime.datetime(1970, 1, 1) countOverlap = 0 # Get a frame from the video stream ret, frame = awscam.getLastFrame() frame_debug = frame if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = frame[670:None, 900:1750] # frame_resize = frame[670:670+450, 900+250:900+250+450] # frame_resize_file = "/tmp/frame_resize.jpeg" #if not os.path.exists(frame_resize_file): # os.mkfifo(frame_resize_file) # cv2.imwrite(frame_resize_file, frame_resize) # frame_resize = cv2.imread(frame_resize_file) # frame_resize = cv2.imread("/home/aws_cam/test2-small.jpeg") frame_resize = cv2.resize(frame_resize, (input_width, input_height)) # frame_resize = cv2.cvtColor(frame_resize, cv2.COLOR_BGR2GRAY) # frame_resize = cv2.cvtColor(frame_resize, cv2.COLOR_GRAY2BGR) # frame_resize = cv2.imread("/home/aws_cam/test2-small.jpeg") #cv2.imwrite("/tmp/frame" + str(input_width) + ".jpeg", frame_resize) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. a = datetime.datetime.now() parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) b = datetime.datetime.now() c = b - a print("Inference Time: " + str(c.total_seconds())) # print(parsed_inference_results) # Compute the scale in order to draw bounding boxes on the full resolution # image. yoffset = 670 xoffset = 900 # Dictionary to be filled with labels and probabilities for MQTT cloud_output = {} balls = [] holes = [] hole = None # Get the detected objects and probabilities for obj in parsed_inference_results[model_type]: if obj['prob'] > detection_threshold: # Add bounding boxes to full resolution frame xmin = int(xoffset + obj['xmin'] * scale) ymin = int(yoffset + obj['ymin'] * scale) xmax = int(xoffset + obj['xmax'] * scale) ymax = int(yoffset + obj['ymax'] * scale) # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.rectangle method. # Method signature: image, point1, point2, color, and tickness. cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 10) # cv2.rectangle(frame_resize, (int(obj['xmin']), int(obj['ymin'])), (int(obj['xmax']), int(obj['ymax'])), (255, 165, 20), 10) # Amount to offset the label/probability text above the bounding box. text_offset = 15 # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, # and tickness cv2.putText( frame_debug, "{}: {:.2f}%".format(output_map[obj['label']], obj['prob'] * 100), (xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6) # Store label and probability to send to cloud cloud_output[output_map[obj['label']]] = obj['prob'] if obj['label'] == 1 and obj['prob'] > 0.7: holes.append(obj) elif obj['label'] == 0: balls.append(obj) for hole_i in holes: if hole is None: hole = hole_i else: if hole_i['prob'] > hole['prob']: hole = hole_i for ball in balls: x_overlap = 0 y_overlap = 0 ballArea = 10000 holeArea = 0 try: x_overlap = min(hole["xmax"], ball["xmax"]) - max( hole["xmin"], ball["xmin"]) y_overlap = min(hole["ymax"], ball["ymax"]) - max( hole["ymin"], ball["ymin"]) ballArea = (ball["xmax"] - ball["xmin"]) * (ball["ymax"] - ball["ymin"]) holeArea = (hole["xmax"] - hole["xmin"]) * (hole["ymax"] - hole["ymin"]) except Exception as ex: print(ex) overlapArea = max(0, max(x_overlap, 0) * max(y_overlap, 0)) overlapPerc = overlapArea / ballArea print("Overlap " + str(overlapPerc)) cloud_output["ballArea"] = ballArea cloud_output["holeArea"] = holeArea cloud_output["overlapPerc"] = overlapPerc if overlapPerc >= 0.5 and ballArea < 370: countOverlap += 1 if (countOverlap >= 1): ballInTime = datetime.datetime.now() ballIn = True countOverlap = 0 if (ballIn): cloud_output["scored"] = "yes" cv2.putText(frame, "Scored!", (300, 300), cv2.FONT_HERSHEY_DUPLEX, 5, (255, 165, 20), 6) cv2.putText(frame_debug, "Scored!", (300, 300), cv2.FONT_HERSHEY_DUPLEX, 5, (255, 165, 20), 6) else: cloud_output["scored"] = "no" # Set the next frame in the local display stream. local_display.set_frame_data(frame) # cv2.imwrite('/tmp/result.jpeg', frame) # cv2.imwrite('/tmp/frame_resize.jpeg', frame_resize) # Send results to the cloud if (mod % 1) == 0: if (ballIn): send_no = False client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) mod = 1 else: mod = mod + 1 if (not send_no and not ballIn): client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) send_no = True print(json.dumps(cloud_output)) d = datetime.datetime.now() runtime = d - timeNow print("Total Runtime: " + str(runtime.total_seconds()))
def infinite_infer_run(): """ Run the DeepLens inference loop frame by frame""" try: # Number of top classes to output num_top_k = 2 model_type = 'classification' model_name = 'image-classification' with open('labels.txt', 'r') as f: output_map = [l for l in f] # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # Optimize the model error, model_path = mo.optimize(model_name, INPUT_WIDTH, INPUT_HEIGHT) # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading model') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Model loaded') while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Fixing eyefish calibration undistorted_img = undistort(frame) # Isolate the drawing (get frames) isolated_frame = isolate_image(undistorted_img) # # Check if cropping a frame is successful # if (isolated_frame == None) # print("Cannot crop the frame") # break # Resize frame to the same size as the training set frame_resize = cv2.resize(isolated_frame, (INPUT_HEIGHT, INPUT_WIDTH)) # Pre processing the input image # Convert the image to grap scale and invert the color gray = cv2.cvtColor(frame_resize, cv2.COLOR_BGR2GRAY) gray = np.array(gray, float) gray /= 255 normImage = 1 - gray # crop horizontal frames normImage[0:20] = 0 normImage[108:] = 0 # crop vertical frames normImage[:, 0:20] = 0 normImage[:, 108:] = 0 normImage = crop_image(normImage, 0.4) top, bottom, left, right = [5] * 4 img_with_border = cv2.copyMakeBorder(normImage, top, bottom, left, right, cv2.BORDER_CONSTANT, value=[0, 0]) border_image = cv2.resize(img_with_border, (INPUT_HEIGHT, INPUT_WIDTH)) kernel = np.ones((3, 3), np.uint8) dilated_image = cv2.dilate(border_image, kernel, iterations=2) stacked_img = 255 * np.stack((dilated_image, ) * 3, axis=-1) finalOutput = stacked_img.astype(np.uint8) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a classification model, # a simple API is provided. parsed_inference_results = model.parseResult( model_type, model.doInference(finalOutput)) # Get top k results with highest probabilities top_k = parsed_inference_results[model_type][0:num_top_k] # Add the label of the top result to the frame used by local display. # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, and thickness output_text = '{} : {:.2f}'.format(output_map[top_k[0]['label']], top_k[0]['prob']) cv2.putText(frame, output_text, (10, 70), cv2.FONT_HERSHEY_SIMPLEX, 3, (255, 165, 20), 8) # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send the top k results to the IoT console via MQTT cloud_output = {} for obj in top_k: cloud_output[output_map[obj['label']]] = obj['prob'] client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: print('Error in lambda {}'.format(ex)) client.publish(topic=iot_topic, payload='Error in lambda: {}'.format(ex))
def greengrass_infinite_infer_run(): """ Entry point of the lambda function""" try: model_type = 'classification' ############################################################ ### ### ### Map inference return codes to outputs ### ### Map inference return codes to text colour ### ### ### ############################################################ output_map = { 0: 'compliant', 1: 'not compliant', 2: 'unknown', 3: 'no subject' } green = (0, 255, 0) red = (0, 0, 255) blue = (255, 0, 0) colour_map = {0: green, 1: red, 2: blue, 3: blue} # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format( os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() ############################################################ ### ### ### Optimize model for Intel using clDNN ### ### ### ############################################################ model_name = 'image-classification' error, model_path = mo.optimize(model_name, 224, 224, aux_inputs={"--epoch": 2}) # Load the model onto the GPU. client.publish(topic=iot_topic, payload='Loading Worksite safety model to GPU') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Worksite safety model loaded') # Since this is a four class classifier only retrieve 4 classes. num_top_k = 4 # The height and width of the training set images input_height = 224 input_width = 224 counter = 0 # Do inference until the lambda is killed. while True: ############################################################ ### ### ### Get a frame from the video stream ### ### ### ############################################################ ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') ############################################################ ### ### ### Resize frame to the same size as model expects ### ### ### ############################################################ frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a classification model, # a simple API is provided. ############################################################ ### ### ### Perform inference against model ### ### ### ############################################################ parsed_inference_results = model.parseResult( model_type, model.doInference(frame_resize)) #client.publish(topic=iot_topic, payload="About to publish parsed_inference_results") ############################################################ ### ### ### Get top results with highest probabilities ### ### ### ############################################################ top_k = parsed_inference_results[model_type][0:num_top_k] print top_k adjusted_results = top_k # Add the label of the top result to the frame used by local display. # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, colour (in BGR), and thickness ############################################################ ### ### ### Set output text and text colour based on top result ### ### ### ############################################################ output = output_map[top_k[0]['label']] confidence = top_k[0]['prob'] colour = colour_map[top_k[0]['label']] ############################################################ ### ### ### If confidence is low, force result to unsure ### ### ### ############################################################ if ((output == "compliant" or output == "not compliant") and (confidence < 0.5)): adjusted_results[0]['label'] = 2 adjusted_results[0]['prob'] = 1 output = output_map[adjusted_results[0]['label']] colour = colour_map[adjusted_results[0]['label']] ############################################################ ### ### ### Superimpose text on the stored frame ### ### ### ############################################################ cv2.putText(frame, output, (20, 150), cv2.FONT_HERSHEY_SIMPLEX, 6, colour, 8) ############################################################ ### ### ### Display marked up frame in project stream ### ### ### ############################################################ local_display.set_frame_data(frame) # Every 10th inference, send the top k results to the IoT console via MQTT counter = counter + 1 if counter >= 10: cloud_output = {} for obj in top_k: cloud_output[output_map[obj['label']]] = obj['prob'] #client.publish(topic=iot_topic, payload="About to publish cloud_output") client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) counter = 0 except Exception as ex: client.publish( topic=iot_topic, payload='Error in worksite safety lambda: {}'.format(ex))
def greengrass_infinite_infer_run(): try: input_width = 224 input_height = 224 model_name = "image-classification" error, model_path = mo.optimize(model_name, input_width, input_height, aux_inputs={'--epoch': 60}) # Load model to GPU (use {"GPU": 0} for CPU) mcfg = {"GPU": 1} model = awscam.Model(model_path, mcfg) client.publish(topic=iotTopic, payload="Model loaded") model_type = "classification" topk = 5 results_thread = FIFO_Thread() results_thread.start() # Send a starting message to IoT console client.publish(topic=iotTopic, payload="Inference is starting") doInfer = True while doInfer: # Get a frame from the video stream ret, frame0 = awscam.getLastFrame() # Raise an exception if failing to get a frame if ret == False: raise Exception("Failed to get frame from the stream") height, width, channels = frame0.shape y = height / 2 x = width / 2 f = 320 frame = frame0[y - f:y + f, x - f:x + f] # Resize frame to fit model input requirement frameResize = cv2.resize(frame, (input_width, input_height)) # Run model inference on the resized frame inferOutput = model.doInference(frameResize) # Output inference result to the fifo file so it can be viewed with mplayer parsed_results = model.parseResult(model_type, inferOutput) #top_k = parsed_results[model_type][0:topk] msg = '{' msg += '"{}": "{}"'.format(inferOutput, parsed_results) msg += "}" obj = parsed_results[model_type][0] if obj["prob"] > 0.9: if obj["label"] == 1: msg = "dog : {:.4f}".format(obj["prob"]) elif obj["label"] == 0: msg = "cat : {:.4f}".format(obj["prob"]) client.publish(topic=iotTopic, payload=msg) cv2.putText(frame, msg, (0, 22), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 165, 20), 4) global jpeg ret, jpeg = cv2.imencode('.jpg', frame) except Exception as e: msg = "myModel Lambda failed: " + str(e) client.publish(topic=iotTopic, payload=msg) # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run(): """ Entry point of the lambda function""" client.publish(topic=iot_topic, payload='Start of run loop...') try: # This object detection model is implemented as single shot detector (ssd), since # the number of labels is small we create a dictionary that will help us convert # the machine labels to human readable labels. model_type = 'ssd' output_map = {} with open('classes.txt') as f: for line in f: (key, val) = line.split() output_map[int(key)] = val client.publish(topic=iot_topic, payload='Classes to be detected: ' + str(output_map)) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The height and width of the training set images input_height = 512 input_width = 512 # Load the model onto the GPU. # optimize the model client.publish(topic=iot_topic, payload='Optimizing model...') ret, model_path = mo.optimize('deploy_ssd_resnet50_512', input_width, input_height) # load the model client.publish(topic=iot_topic, payload='Loading model...') model = awscam.Model(model_path, {'GPU': 1}) client.publish(topic=iot_topic, payload='Custom object detection model loaded') # Set the threshold for detection detection_threshold = 0.40 # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (input_height, input_width)) # Run the images through the inference engine and parse the results using # the parser API. Note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. client.publish(topic=iot_topic, payload='Calling inference on next frame...') parsed_inference_results = model.parseResult(model_type, model.doInference(frame_resize)) # Compute the scale in order to draw bounding boxes on the full resolution # image. yscale = float(frame.shape[0]) / float(input_height) xscale = float(frame.shape[1]) / float(input_width) # Dictionary to be filled with labels and probabilities for MQTT cloud_output = {} # Get the detected objects and probabilities default_color = (255, 165, 20) i = 0 for obj in parsed_inference_results[model_type]: if obj['prob'] > detection_threshold: # Add bounding boxes to full resolution frame xmin = int(xscale * obj['xmin']) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) ymax = int(yscale * obj['ymax']) # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.rectangle method. # Method signature: image, point1, point2, color, and tickness. cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), default_color, 10) # Amount to offset the label/probability text above the bounding box. text_offset = 15 # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, # and thickness cv2.putText(frame, "{}: {:.2f}%".format(output_map[obj['label']], obj['prob'] * 100), (xmin, ymin-text_offset), cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6) # Store label and probability to send to cloud cloud_output[output_map[obj['label']]] = obj['prob'] # Set the next frame in the local display stream. local_display.set_frame_data(frame) # Send results to the cloud client.publish(topic=iot_topic, payload=json.dumps(cloud_output)) except Exception as ex: client.publish(topic=iot_topic, payload='Error in object detection lambda: {}'.format(ex))
def head_detection(): """ This method serves as the main entry point of our lambda. """ # Creating a client to send messages via IoT MQTT to the cloud client = greengrasssdk.client('iot-data') # This is the topic where we will publish our messages too iot_topic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME']) try: # define model prefix and the amount to down sample the image by. input_height = 84 input_width = 84 # Send message to IoT via MQTT client.publish(topic=iot_topic, payload="Optimizing model") ret, model_path = mo.optimize("deeplens-headpose-detection", input_width, input_height, platform='tf') # Send message to IoT via MQTT client.publish(topic=iot_topic, payload="Model optimization complete") if ret is not 0: raise Exception("Model optimization failed, error code: {}".format(ret)) # Send message to IoT via MQTT client.publish(topic=iot_topic, payload="Loading model") # Load the model into cl-dnn model = awscam.Model(model_path, {"GPU": 1}) # Send message to IoT via MQTT client.publish(topic=iot_topic, payload="Model loaded") # We need to sample a frame so that we can determine where the center of # the image is located. We assume that # resolution is constant during the lifetime of the lmabda. ret, sample_frame = awscam.getLastFrame() if not ret: raise Exception("Failed to get frame from the stream") # Construct the custom helper object. This object just lets us handle postprocessing # in a managable way. head_pose_detection = HeadDetection(sample_frame.shape[1]/2, sample_frame.shape[0]/2) # Dictionary that will allow us to convert the inference labels # into a human a readable format. output_map = ({0:'Bottom Right', 1:'Middle Right', 2:'Top Right', 3:'Bottom Middle', 4:'Middle Middle', 5:'Top Middle', 6:'Bottom Left', 7:'Middle Left', 8:'Top Left'}) # This model is a ResNet classifier, so our out put will be classification. model_type = "classification" # Define a rectangular region where we expect the persons head to be. crop_upper_left_y = 440 crop_height = 640 crop_upper_left_x = 1024 crop_width = 640 while True: # Grab the latest frame off the mjpeg stream. ret, frame = awscam.getLastFrame() if not ret: raise Exception("Failed to get frame from the stream") # Mirror Image frame = cv2.flip(frame, 1) # Draw the rectangle around the area that we expect the persons head to be. cv2.rectangle(frame, (crop_upper_left_x, crop_upper_left_y), (crop_upper_left_x + crop_width, crop_upper_left_y + crop_height), (255, 40, 0), 4) # Crop the the frame so that we can do inference on the area of the frame where # expect the persons head to be. frame_crop = frame[crop_upper_left_y:crop_upper_left_y + crop_height, crop_upper_left_x:crop_upper_left_x + crop_width] # Down sample the image. frame_resize = cv2.resize(frame_crop, (input_width, input_height)) # Renormalize the image so that the color magnitudes are between 0 and 1 frame_resize = frame_resize.astype(np.float32)/255.0 # Run the down sampled image through the inference engine. infer_output = model.doInference(frame_resize) # Parse the results so that we get back a more manageble data structure. parsed_results = model.parseResult(model_type, infer_output)[model_type] # Post process the image and send it to the FIFO file head_pose_detection.send_data(frame, parsed_results) # Send the results to MQTT in JSON format. client.publish(topic=iot_topic, payload=head_pose_detection.get_results(parsed_results, output_map)) except Exception as ex: msg = "Lambda has failure, error msg {}: ".format(ex) client.publish(topic=iot_topic, payload=msg)
def catcritter_infinite_infer_run(): """ Entry point of the lambda function""" try: # captured ssd image info for training ssd_image_list = [] capture_class_images = False class_labels = { 0: 'background', 1: 'buddy', 2: 'jade', 3: 'lucy', 4: 'tim' } # This object detection model is implemented as single shot detector (ssd), since # the number of labels is small we create a dictionary that will help us convert # the machine labels to human readable labels. output_map = { 1: 'aeroplane', 2: 'bicycle', 3: 'bird', 4: 'boat', 5: 'bottle', 6: 'bus', 7: 'car', 8: 'cat', 9: 'chair', 10: 'cow', 11: 'dinning table', 12: 'dog', 13: 'horse', 14: 'motorbike', 15: 'person', 16: 'pottedplant', 17: 'sheep', 18: 'sofa', 19: 'train', 20: 'tvmonitor' } # Create an IoT client for sending to messages to the cloud. client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format( os.environ['AWS_IOT_THING_NAME']) # Create a local display instance that will dump the image bytes to a FIFO # file that the image can be rendered locally. local_display = LocalDisplay('480p') local_display.start() # The height and width of the training set images class_input_height = 100 class_input_width = 100 ssd_input_height = 300 ssd_input_width = 300 ssd_model_type = "ssd" class_model_type = "classification" class_model_name = "image-classification" client.publish(topic=iot_topic, payload='optimizing model') error, class_model_path = mo.optimize(class_model_name, class_input_width, class_input_height, aux_inputs={'--epoch': 100}) # The aux_inputs is equal to the number of epochs and in this case, it is 100 # Load model to GPU (use {"GPU": 0} for CPU) mcfg = {"GPU": 1} # The sample projects come with optimized artifacts, hence only the artifact # path is required. ssd_model_path = '/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml' # Load the model onto the GPU client.publish(topic=iot_topic, payload='Loading object detection model') ssd_model = awscam.Model(ssd_model_path, mcfg) class_model = awscam.Model(class_model_path, mcfg) client.publish(topic=iot_topic, payload='Object detection model loaded') # Set the threshold for detection detection_threshold = 0.25 counter = 1 ssd_counter = 1 irand = randrange(0, 1000) num_classes = 4 # prepare training csv if not os.path.isdir("/tmp/cats"): os.mkdir("/tmp/cats") os.mkdir("/tmp/cats/train") os.chmod("/tmp/cats", stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) os.chmod("/tmp/cats/train", stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) if not os.path.isfile("/tmp/cats/train/train.csv"): with open('/tmp/cats/train/train.csv', 'a') as outcsv: writer = csv.writer(outcsv) writer.writerow( ['frame', 'xmin', 'xmax', 'ymin', 'ymax', 'class_id']) outcsv.close() today = datetime.datetime.now().strftime("%Y%m%d") # Do inference until the lambda is killed. while True: # Get a frame from the video stream ret, frame = awscam.getLastFrame() if not ret: raise Exception('Failed to get frame from the stream') # Resize frame to the same size as the training set. frame_resize = cv2.resize(frame, (ssd_input_height, ssd_input_width)) # Run the images through the inference engine and parse the results using # the parser API, note it is possible to get the output of doInference # and do the parsing manually, but since it is a ssd model, # a simple API is provided. parsed_inference_results = ssd_model.parseResult( ssd_model_type, ssd_model.doInference(frame_resize)) #client.publish(topic=iot_topic, payload='ssd infer complete') # Compute the scale in order to draw bounding boxes on the full resolution # image. yscale = float(frame.shape[0] / ssd_input_height) xscale = float(frame.shape[1] / ssd_input_width) # Dictionary to be filled with labels and probabilities for MQTT cloud_output = {} image_saved = False # Get the detected objects and probabilities for obj in parsed_inference_results[ssd_model_type]: if obj['prob'] > detection_threshold: # Add bounding boxes to full resolution frame xmin = int(xscale * obj['xmin']) \ + int((obj['xmin'] - ssd_input_width/2) + ssd_input_width/2) ymin = int(yscale * obj['ymin']) xmax = int(xscale * obj['xmax']) \ + int((obj['xmax'] - ssd_input_width/2) + ssd_input_width/2) ymax = int(yscale * obj['ymax']) if xmin < 0: xmin = 0 if ymin < 0: ymin = 0 # if we found a cat, then save the image to a file and publish to IOT if obj['label'] == 8 or obj['label'] == 12 or obj[ 'label'] == 15: # save the ssd image if not image_saved: frame_filename = "{}_{:03d}_{}_{:03d}".format( today, ssd_counter, 'cats', irand) frame_path = "/tmp/cats/train/" + frame_filename + '.jpg' cv2.imwrite(frame_path, frame_resize) ssd_counter += 1 image_saved = True crop = frame[ymin:ymax, xmin:xmax].copy() crop_resize = cv2.resize( crop, (class_input_height, class_input_width)) # Run model inference on the cropped frame inferOutput = class_model.doInference(crop_resize) #client.publish(topic=iot_topic, payload='classification infer complete') class_inference_results = class_model.parseResult( class_model_type, inferOutput) top_k = class_inference_results[class_model_type][ 0:num_classes] first = top_k[0] if first['prob'] > detection_threshold: #client.publish(topic=iot_topic, payload='found {}, saving file'.format(labels[first['label']])) if capture_class_images: dir = "/tmp/cats/train/" + frame_filename if not os.path.isdir(dir): os.mkdir(dir) os.chmod( dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) the_label = class_labels[first['label']] the_class = first['label'] if (obj['label'] == 15 and the_class < 3): the_label = 'person' the_class = 3 path = "/tmp/cats/train/{}/train_{}_{:03d}_{}_{:03d}_{:03d}_{:03d}_{:03d}_{:03d}.jpg".format( frame_filename, today, counter, the_label, irand, int(round(obj['xmin'])), int(round(obj['xmax'])), int(round(obj['ymin'])), int(round(obj['ymax']))) if capture_class_images: cv2.imwrite(path, crop) os.chmod( path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH) counter += 1 msg = '{' prob_num = 0 for kitty in top_k: if prob_num == num_classes - 1: msg += '"{}": {:.2f}'.format( class_labels[kitty["label"]], kitty["prob"] * 100) else: msg += '"{}": {:.2f},'.format( class_labels[kitty["label"]], kitty["prob"] * 100) prob_num += 1 msg += "}" # Send results to the cloud #client.publish(topic=iot_topic, payload=json.dumps(msg)) else: the_class = 0 the_label = class_labels[the_class] path = "/tmp/cats/train/{}/train_{}_{:03d}_{}_{:03d}_{:03d}_{:03d}_{:03d}_{:03d}.jpg".format( frame_filename, today, counter, the_label, irand, int(round(obj['xmin'])), int(round(obj['xmax'])), int(round(obj['ymin'])), int(round(obj['ymax']))) if capture_class_images: cv2.imwrite(path, crop) os.chmod( path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH) counter += 1 # create ssd entry ssd_image_desc = [ frame_filename + ".jpg", int(round(obj['xmin'])), int(round(obj['xmax'])), int(round(obj['ymin'])), int(round(obj['ymax'])), the_class ] ssd_image_list.append(ssd_image_desc) if obj['label'] < 20: # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.rectangle method. # Method signature: image, point1, point2, color, and tickness. cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 10) # Amount to offset the label/probability text above the bounding box. text_offset = 15 # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html # for more information about the cv2.putText method. # Method signature: image, text, origin, font face, font scale, color, # and tickness cv2.putText( frame, "{}: {:.2f}%".format(output_map[obj['label']], obj['prob'] * 100), (xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6) # Set the next frame in the local display stream. local_display.set_frame_data(frame) if image_saved: with open('/tmp/cats/train/train.csv', 'a') as outcsv: writer = csv.writer(outcsv) writer.writerows(ssd_image_list) ssd_image_list = [] outcsv.close() except Exception as ex: client.publish( topic=iot_topic, payload='Error in cat-plus-critter lambda: {}'.format(ex)) outcsv.close()
def greengrass_infinite_infer_run(): try: input_width = 224 input_height = 224 model_name = "image-classification" model_type = "classification" client = greengrasssdk.client('iot-data') iotTopic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME']) local_display = LocalDisplay('480p') local_display.start() error, model_path = mo.optimize(model_name,input_width,input_height, aux_inputs={'--epoch': 30}) mcfg = {"GPU": 1} print("model_path: " + model_path) model = awscam.Model(model_path, mcfg) client.publish(topic=iotTopic, payload="Model loaded") with open('pinehead_labels.txt', 'r') as f: labels = [l.rstrip() for l in f] num_top_k = 2 # Send a starting message to IoT console client.publish(topic=iotTopic, payload="Inference is starting") doInfer = True while doInfer: # Get a frame from the video stream ret, frame = awscam.getLastFrame() # Raise an exception if failing to get a frame if ret == False: raise Exception("Failed to get frame from the stream") # Resize frame to fit model input requirement s = frame.shape cropped = frame[0:s[0], int((s[1]-s[0])/2):s[0]+int((s[1]-s[0])/2)] frameResize = cv2.resize(cropped, (input_width, input_height)) # Run model inference on the resized frame inferOutput = model.doInference(frameResize) parsed_inference_results = model.parseResult(model_type, model.doInference(frameResize)) top_k = parsed_inference_results[model_type][0:num_top_k-1] msg = "{" msg += '"{}"'.format(labels[top_k[0]["label"]]) msg += "}" client.publish(topic=iotTopic, payload = msg) font = cv2.FONT_HERSHEY_SIMPLEX cv2.putText(frame, labels[top_k[0]["label"]], (10, 140), font, 5, (174, 235, 52), 10) local_display.set_frame_data(frame) except Exception as e: msg = "myModel Lambda failed: " + str(e) client.publish(topic=iotTopic, payload=msg) # Asynchronously schedule this function to be run again in 15 seconds Timer(15, greengrass_infinite_infer_run).start()