def infinite_infer_run():
""" Run the DeepLens inference loop frame by frame"""
try:
# Number of top classes to output
num_top_k = 2
model_type = 'classification'
model_name = 'image-classification'
with open('labels.txt', 'r') as f:
output_map = [l for l in f]
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# Optimize the model
error, model_path = mo.optimize(model_name, INPUT_WIDTH, INPUT_HEIGHT)
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload='Loading model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='Model loaded')
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (INPUT_HEIGHT, INPUT_WIDTH))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a classification model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
# Get top k results with highest probabilities
top_k = parsed_inference_results[model_type][0:num_top_k]
# Add the label of the top result to the frame used by local display.
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color, and thickness
output_text = '{} : {:.2f}'.format(output_map[top_k[0]['label']],
top_k[0]['prob'])
cv2.putText(frame, output_text, (10, 70), cv2.FONT_HERSHEY_SIMPLEX,
3, (255, 165, 20), 8)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send the top k results to the IoT console via MQTT
cloud_output = {}
for obj in top_k:
cloud_output[output_map[obj['label']]] = obj['prob']
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
print('Error in lambda {}'.format(ex))
client.publish(topic=iot_topic,
payload='Error in lambda: {}'.format(ex))
def greengrass_infinite_infer_run():
try:
global clients
input_width = 224
input_height = 224
model_name = "image-classification"
error, model_path = mo.optimize(model_name, input_width, input_height)
if error:
print error
raise Exception(error)
print("Optimized model")
model = awscam.Model(model_path, {"GPU": 1})
print("Loaded model")
model_type = "classification"
labels = ['a','b','c','d','e','f','g','h','i','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','not']
topk = 1
while True:
start = datetime.now()
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
height, width, channels = frame.shape
frame_cropped = frame[0:height, (width-height)/2:width-(width-height)/2]
frame_resize = cv2.resize(frame_cropped, (input_width, input_height))
infer_output = model.doInference(frame_resize)
parsed_results = model.parseResult(model_type, infer_output)
top_k = parsed_results[model_type][0:topk]
ret, jpeg = cv2.imencode(".jpg", frame_resize)
try:
end = datetime.now()
top_k[0]["label"] = labels[top_k[0]["label"]];
top_k[0]["jpeg"] = b64encode(jpeg)
top_k[0]["delay"] = (end - start).microseconds / 1000
msg = json.dumps(top_k[0]) + u''
for client in clients:
client.sendMessage(msg)
except:
print("error", sys.exc_info()[0])
except Exception as e:
print("Lambda failed: " + str(e))
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run():
try:
modelType = "classification"
model_name = "image-classification"
input_width = 224
input_height = 224
max_threshold = 0.75
error, model_path = mo.optimize(model_name,input_width,input_height)
outMap = { 0: "a", 1: "b", 2: "c", 3: "d", 4: "e", 5: "f", 6: "g", 7 : "h", 8 : "i", 9 : "k", 10 : "l", 11 : "m", 12 : "n", 13 : "o", 14 : "p", 15 : "q", 16 : "r", 17 : "s", 18 : "t", 19 : "u", 20 : "v", 21 : "w", 22 : "x", 23 : "y", 24 : "z" }
client.publish(topic=iotTopic, payload="Object detection starts now")
mcfg = {"GPU": 1}
model = awscam.Model(model_path, mcfg)
client.publish(topic=iotTopic, payload="Model loaded")
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
doInfer = True
while doInfer:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
# Raise an exception if failing to get a frame
if ret == False:
raise Exception("Failed to get frame from the stream")
margin = (frame.shape[1] - frame.shape[0]) / 2
cropped = frame[0:frame.shape[0], margin:frame.shape[1] - margin]
frameResize = cv2.resize(cropped, (input_width, input_height))
inferOutput = model.doInference(frameResize)
parsed_results = model.parseResult(modelType, inferOutput)["classification"]
client.publish(topic=iotTopic, payload = str(parsed_results))
for obj in parsed_results:
if obj["prob"] > max_threshold and detected != outMap[obj["label"]]:
detected = outMap[obj["label"]]
client.publish(topic=iotTopic, payload = outMap[obj["label"]])
playsound("../letters/" + outMap[obj["label"]] + ".mp3");
except Exception as e:
msg = "Test failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
client.publish(topic=iot_topic, payload='Start of run loop...')
try:
# This object detection model is implemented as single shot detector (ssd), since
# the number of labels is small we create a dictionary that will help us convert
# the machine labels to human readable labels.
model_type = 'ssd'
output_map = {}
with open('classes.txt') as f:
for line in f:
(key, val) = line.split()
output_map[int(key)] = val
client.publish(topic=iot_topic,
payload='Classes to be detected: ' + str(output_map))
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The height and width of the training set images
input_height = 512
input_width = 512
# Load the model onto the GPU.
# optimize the model
client.publish(topic=iot_topic, payload='Optimizing model...')
ret, model_path = mo.optimize('deploy_ssd_resnet50_512', input_width,
input_height)
# load the model
client.publish(topic=iot_topic, payload='Loading model...')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic,
payload='Custom object detection model loaded')
# Set the threshold for detection
detection_threshold = 0.40
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API. Note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
client.publish(topic=iot_topic,
payload='Calling inference on next frame...')
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0]) / float(input_height)
xscale = float(frame.shape[1]) / float(input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Get the detected objects and probabilities
default_color = (255, 165, 20)
i = 0
for obj in parsed_inference_results[model_type]:
if obj['prob'] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj['xmin'])
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax'])
ymax = int(yscale * obj['ymax'])
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
default_color, 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color,
# and thickness
cv2.putText(
frame, "{}: {:.2f}%".format(output_map[obj['label']],
obj['prob'] * 100),
(xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX,
2.5, (255, 165, 20), 6)
# Store label and probability to send to cloud
cloud_output[output_map[obj['label']]] = obj['prob']
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send results to the cloud
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(
topic=iot_topic,
payload='Error in object detection lambda: {}'.format(ex))
def infinite_infer_run():
""" Entry point of the lambda function"""
try:
# This bird detection model is implemented as multi classifier. The number of labels
# is quite large so we upload them to a list to map the machine labels to human readable
# labels.
model_type = 'classification'
with open('labels.txt', 'r') as labels_file:
output_map = [class_label.rstrip() for class_label in labels_file]
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(
os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The height and width of the training set images
input_height = 224
input_width = 224
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
ret, model_path = mo.optimize('bird_classification_resnet-18',
input_width, input_height, 'mx')
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload='Loading bird detection model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='Bird detection loaded')
# The number of top results to stream to IoT.
num_top_k = 5
# Define the detection region size.
region_size = 800
# Define the inference display region size. This size was decided based on the longest label.
label_region_width = 940
label_region_height = 600
# Heading for the inference display.
prediction_label = 'Top 5 bird predictions'
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Crop the detection region for inference.
frame_crop = frame[int(frame.shape[0]/2-region_size/2):int(frame.shape[0]/2+region_size/2), \
int(frame.shape[1]/2-region_size/2):int(frame.shape[1]/2+region_size/2), :]
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame_crop, (input_height, input_width))
# Model was trained in RGB format but getLastFrame returns image
# in BGR format so need to switch.
frame_resize = cv2.cvtColor(frame_resize, cv2.COLOR_BGR2RGB)
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a classification model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
# Get top k results with highest probabilities
top_k = parsed_inference_results[model_type][0:num_top_k]
# Create a copy of the original frame.
overlay = frame.copy()
# Create the rectangle that shows the inference results.
cv2.rectangle(overlay, (0, 0), \
(int(label_region_width), int(label_region_height)), (211,211,211), -1)
# Blend with the original frame.
opacity = 0.7
cv2.addWeighted(overlay, opacity, frame, 1 - opacity, 0, frame)
# Add the header for the inference results.
cv2.putText(frame, prediction_label, (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 4)
# Add the label along with the probability of the top result to the frame used by local display.
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color, and tickness
for i in range(num_top_k):
cv2.putText(frame, output_map[top_k[i]['label']] + ' ' + str(round(top_k[i]['prob'], 3) * 100) + '%', \
(0, 100*i+150), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 0, 0), 3)
# Display the detection region.
cv2.rectangle(frame, (int(frame.shape[1]/2-region_size/2), int(frame.shape[0]/2-region_size/2)), \
(int(frame.shape[1]/2+region_size/2), int(frame.shape[0]/2+region_size/2)), (255,0,0), 5)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send the top k results to the IoT console via MQTT
cloud_output = {}
for obj in top_k:
cloud_output[output_map[obj['label']]] = obj['prob']
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(topic=iot_topic,
payload='Error in bird detection lambda: {}'.format(ex))
Write_To_FIFO = True
FIRST_RUN = True
PLAY_INTRO = True
input_width = 300
input_height = 300
prob_thresh = 0.55
outMap = {0: 'text_block'}
model_name = "read-to-me"
occursThreshold = 10
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME'])
jpeg = None
logger = logging.getLogger()
logger.setLevel(logging.INFO)
error, model_path = mo.optimize(model_name, input_width, input_height)
model = awscam.Model(model_path, {"GPU": 1}, awscam.Runtime.DLDT)
def log_message(message):
logger.info(message)
# client.publish(topic=iot_topic, payload=message)
def first_run():
log_message('first run')
global FIRST_RUN
try:
if PLAY_INTRO:
log_message('attempting to play audio files for first run')
speak.playAudioFile(os.path.join('staticfiles', 'intro.mp3'))
sleep(0.5)
speak.playAudioFile(os.path.join('staticfiles', 'dir1.mp3'))
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
try:
model_type = 'classification'
model_name = 'image-classification'
output_map = {0: 'weed', 1: 'grass'}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(
os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# model_path = '/opt/awscam/artifacts/image-classification.xml'
# The height and width of the training set images
input_height = 224
input_width = 224
error, model_path = mo.optimize(model_name,
input_height,
input_width,
aux_inputs={'--epoch': 10})
# Load the model onto the GPU.
client.publish(topic=iot_topic,
payload='Loading image classification model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic,
payload='Image classification model loaded')
# Set the threshold for classification
classification_threshold = 0.25
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0] / input_height)
xscale = float(frame.shape[1] / input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Get the classified images and probabilities
for obj in parsed_inference_results[model_type]:
if obj['prob'] > classification_threshold:
# Store label and probability to send to cloud
cloud_output[output_map[obj['label']]] = obj['prob']
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send results to the cloud
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(
topic=iot_topic,
payload='Error in image classification lambda: {}'.format(ex))
def infinite_infer_run():
""" Entry point of the lambda function"""
try:
model_type = 'classification'
output_map = {
0: 'airplane',
1: 'automobile',
2: 'bird',
3: 'cat',
4: 'deer',
5: 'dog',
6: 'frog',
7: 'horse',
8: 'ship',
9: 'truck'
}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(
os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
error, model_path = mo.optimize('model', 32, 32)
#model_path = '/opt/awscam/artifacts/mxnet_resnet18-catsvsdogs_FP32_FUSED.xml'
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload='Loading cifar model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='cifar model loaded')
num_top_k = 10
# The height and width of the training set images
input_height = 32
input_width = 32
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a classification model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
# Get top k results with highest probabilities
top_k = parsed_inference_results[model_type][0:num_top_k]
# Add the label of the top result to the frame used by local display.
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color, and tickness
cv2.putText(frame, output_map[top_k[0]['label']], (10, 70),
cv2.FONT_HERSHEY_SIMPLEX, 3, (255, 165, 20), 8)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send the top k results to the IoT console via MQTT
cloud_output = {}
for obj in top_k:
cloud_output[output_map[obj['label']]] = obj['prob']
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(topic=iot_topic,
payload='Error in cat-dog lambda: {}'.format(ex))
def greengrass_infinite_infer_run():
try:
# Optimizing model (you have to update MxNet with: pip3 install mxnet --upgrade)
input_width = 224
input_height = 224
model_name = "deeplens-asl"
error, model_path = mo.optimize(model_name, input_width, input_height)
if error:
raise Exception(error)
# Loading model
model = awscam.Model(model_path, {"GPU": 1})
client.publish(topic=iot_topic, payload="Model loaded")
# Model variables
model_type = "classification"
labels = ['a','b','c','d','e','f','g','h','i','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','spok','like','f**k']
topk = 1
# Streaming results
results_thread = FIFO_Thread()
results_thread.start()
client.publish(topic=iot_topic, payload="Inference is starting")
# Preparing sentence display
actual_letter = ''
actual_frame_count = 0
message = ''
minimum_count = 20
space_count = 30
reset_count = 70
total_frame_count = 0
# Inference loop
while True:
# Retrieving last frame
total_frame_count = total_frame_count + 1
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
# Resizing frame
height, width, channels = frame.shape
frame_cropped = frame[0:height, (width-height)/2:width-(width-height)/2]
frame_resize = cv2.resize(frame_cropped, (input_width, input_height))
infer_output = model.doInference(frame_resize)
parsed_results = model.parseResult(model_type, infer_output)
top_k = parsed_results[model_type][0:topk]
output_frame = frame_cropped
# Filtering 40% confidence
if (top_k[0]["prob"] > 0.4):
# Writing inference results
letter = labels[top_k[0]["label"]]
if (actual_letter == letter):
actual_frame_count = actual_frame_count + 1
if (actual_frame_count == minimum_count):
message = message + letter
else:
actual_frame_count = 1
actual_letter = labels[top_k[0]["label"]]
cv2.putText(output_frame, '{}: {:.2f}'.format(labels[top_k[0]["label"]], top_k[0]["prob"]), (0,20), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 165, 20), 4)
# Publishing results to IoT topic
#msg = "{"
#prob_num = 0
#for obj in top_k:
# if prob_num == topk-1:
# msg += '"{}": {:.2f}'.format(labels[obj["label"]], obj["prob"])
# else:
# msg += '"{}": {:.2f},'.format(labels[obj["label"]], obj["prob"])
# prob_num += 1
#msg += "}"
#client.publish(topic=iot_topic, payload=msg)
else:
# Handling whitespaces
if (actual_letter != ''):
actual_frame_count = 0
actual_letter = ''
else :
actual_frame_count = actual_frame_count + 1
if (actual_frame_count == space_count and not message.endswith(" ")):
message = message + " "
if (actual_frame_count == reset_count):
message = ""
# Handling caret
total_message = message
if (total_frame_count % 4 < 2) :
total_message = message + "_"
# Writing sentence
cv2.putText(output_frame, total_message, (0,height-50), cv2.FONT_HERSHEY_SIMPLEX, 4, (45, 145, 236), 4)
# Streaming frame
global jpeg
ret, jpeg = cv2.imencode('.jpg', output_frame)
except Exception as e:
msg = "Lambda failed: " + str(e)
client.publish(topic=iot_topic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run():
try:
input_width = 224
input_height = 224
model_name = "image-classification"
error, model_path = mo.optimize(model_name,input_width,input_height, aux_inputs={'--epoch': 10})
# The aux_inputs is equal to the number of epochs and in this case, it is 10
# Load model to GPU (use {"GPU": 0} for CPU)
mcfg = {"GPU": 1}
model = awscam.Model(model_path, mcfg)
client.publish(topic=iotTopic, payload="Model loaded")
model_type = "classification"
with open('caltech256_labels.txt', 'r') as f:
labels = [l.rstrip() for l in f]
topk = 5
results_thread = FIFO_Thread()
results_thread.start()
# Send a starting message to IoT console
client.publish(topic=iotTopic, payload="Inference is starting")
doInfer = True
while doInfer:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
# Raise an exception if failing to get a frame
if ret == False:
raise Exception("Failed to get frame from the stream")
# Resize frame to fit model input requirement
frameResize = cv2.resize(frame, (input_width, input_height))
# Run model inference on the resized frame
inferOutput = model.doInference(frameResize)
# Output inference result to the fifo file so it can be viewed with mplayer
parsed_results = model.parseResult(model_type, inferOutput)
top_k = parsed_results[model_type][0:topk]
msg = '{'
prob_num = 0
for obj in top_k:
if prob_num == topk-1:
msg += '"{}": {:.2f}'.format(labels[obj["label"]], obj["prob"]*100)
else:
msg += '"{}": {:.2f},'.format(labels[obj["label"]], obj["prob"]*100)
prob_num += 1
msg += "}"
client.publish(topic=iotTopic, payload = msg)
cv2.putText(frame, labels[top_k[0]["label"]], (0,22), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 165, 20), 4)
global jpeg
ret,jpeg = cv2.imencode('.jpg', frame)
except Exception as e:
msg = "caltech256 Lambda failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def infinite_infer_run():
""" Entry point of the lambda function"""
try:
# This object detection model is implemented as single shot detector (ssd), since
# the number of labels is small we create a dictionary that will help us convert
# the machine labels to human readable labels.
model_type = 'ssd'
output_map = {0: '1', 1: '2', 2: '3', 3: '4'}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(
os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('square')
local_display.start()
# The height and width of the training set images
input_height = 300
input_width = 300
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
model_path = '/opt/awscam/artifacts/deploy_ssd_resnet50_300.xml'
if not os.path.exists(model_path):
# Optimization needed
client.publish(
topic=iot_topic,
payload='Model does not exist in "{}", optimizing...'.format(
model_path))
res, model_path = mo.optimize('deploy_ssd_resnet50_300',
input_height, input_width)
if res == 0:
client.publish(
topic=iot_topic,
payload=
'Model optimized successful. The optimized model is "{}"'.
format(model_path))
# Load the model onto the GPU.
client.publish(topic=iot_topic,
payload='Loading object detection model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic,
payload='Object detection model loaded')
# Set the threshold for detection
detection_threshold = 0.30
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
frame = crop_frame_square(frame)
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0]) / float(input_height)
xscale = float(frame.shape[1]) / float(input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Get the detected objects and probabilities
for obj in parsed_inference_results[model_type]:
if obj['prob'] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj['xmin'])
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax'])
ymax = int(yscale * obj['ymax'])
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
(255, 165, 20), 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color,
# and tickness
cv2.putText(
frame, "{}: {:.2f}%".format(output_map[obj['label']],
obj['prob'] * 100),
(xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX,
2.5, (255, 165, 20), 6)
# Store label and probability to send to cloud
cloud_output[output_map[obj['label']]] = obj['prob']
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send results to the cloud
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(
topic=iot_topic,
payload='Error in object detection lambda: {}. Stack trace: {}'.
format(ex,
traceback.format_exc().splitlines()))
def greengrass_infinite_infer_run():
try:
model_type = "ssd"
input_width = 224
input_height = 224
prob_thresh = 0.25
results_thread = FIFO_Thread()
results_thread.start()
haar_face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_alt.xml')
# Send a starting message to IoT console
client.publish(topic=iotTopic, payload="Face detection starts now")
# Load model to GPU (use {"GPU": 0} for CPU)
mcfg = {"GPU": 1}
error, modelPath = mo.optimize('smile-net', input_width, input_height)
model = awscam.Model(modelPath, mcfg)
client.publish(topic=iotTopic, payload="Model loaded")
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
doInfer = True
while doInfer:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
# Raise an exception if failing to get a frame
if ret == False:
raise Exception("Failed to get frame from the stream")
img_copy = np.copy(frame)
# convert the test image to gray image as opencv face detector expects gray images
gray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY)
# let's detect multiscale (some images may be closer to camera than others) images
faces = haar_face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5);
# go over list of faces and draw them as rectangles on original colored img
for face in faces:
(x, y, w, h) = face
face_img = img_copy[y:y + h, x:x + w]
resize_face = cv2.resize(face_img, (224, 224))
infer_output = model.doInference(resize_face)
parsed_results = model.parseResult(model_type, infer_output)
prob = parsed_results[model_type][0]['prob']
if prob > prob_thresh:
# smiling, show blueish
cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 165, 20), 4)
else:
# Neutral, show orangish
cv2.rectangle(frame, (x, y), (x + w, y + h), (20, 165, 255), 4)
msg = "{"
msg += '"{}":{:.2f},'.format('smile', prob)
msg += "}"
client.publish(topic=iotTopic, payload=msg)
global jpeg
ret, jpeg = cv2.imencode('.jpg', frame)
except Exception as e:
msg = "Test failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
client.publish(topic=iot_topic, payload='Start of run loop...')
try:
# This object detection model
model_type = "classification"
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The height and width of the training set images
input_height = 512
input_width = 512
# optimize the model
client.publish(topic=iot_topic, payload='Optimizing model...')
model_name = "image-classification"
error, model_path = mo.optimize(model_name,
input_width,
input_height,
aux_inputs={'--epoch': 15})
# Load model to GPU (use {"GPU": 0} for CPU)
mcfg = {"GPU": 1}
model = awscam.Model(model_path, mcfg)
client.publish(topic=iot_topic,
payload='Custom object detection model loaded')
# Load labels from text file
with open('sock_labels.txt', 'r') as f:
labels = [l.rstrip() for l in f]
topk = 2
# Send a starting message to IoT console
client.publish(topic=iot_topic, payload="Inference is starting")
doInfer = True
while doInfer:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
# Raise an exception if failing to get a frame
if ret == False:
raise Exception("Failed to get frame from the stream")
# Resize frame to fit model input requirement
frameResize = cv2.resize(frame, (input_width, input_height))
# Run model inference on the resized frame
inferOutput = model.doInference(frameResize)
# Output inference result to the fifo file so it can be viewed with mplayer
parsed_results = model.parseResult(model_type, inferOutput)
top_k = parsed_results[model_type][0:topk]
sock_label = labels[top_k[0]["label"]]
sock_prob = top_k[0]["prob"] * 100
# Write to MQTT
json_payload = {"image": sock_label, "probability": sock_prob}
client.publish(topic=iot_topic, payload=json.dumps(json_payload))
client2.publish(topic=mqttconfig.mqtt_topic,
payload=json.dumps(json_payload),
qos=0,
retain=False)
# Write to image buffer; screen display
msg_screen = '{} {:.0f}%'.format(sock_label, sock_prob)
cv2.putText(frame, msg_screen, (20, 200), cv2.FONT_HERSHEY_SIMPLEX,
5, (0, 0, 255), 12)
local_display.set_frame_data(frame)
except Exception as ex:
client.publish(
topic=iot_topic,
payload='Error in object detection lambda: {}'.format(ex))
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
try:
thing_name = os.environ['AWS_IOT_THING_NAME']
except Exception as ex:
thing_name = "deeplens_cIqzAO10SVWwwHcsqXD5Jg"
iot_topic = '$aws/things/{}/infer'.format(thing_name)
if True: #try:
# This object detection model is implemented as single shot detector (ssd), since
# the number of labels is small we create a dictionary that will help us convert
# the machine labels to human readable labels.
model_type = 'ssd'
output_map = {0: 'golf ball', 1: 'hole'}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
#if os.environ['AWS_IOT_THING_NAME'] == "":
# iot_topic = '$aws/things/deeplens_cIqzAO10SVWwwHcsqXD5Jg/infer'
#else:
# iot_topic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The height and width of the training set images
#input_height = 720
base_width = 850
#input_width = 212
input_width = 150
input_height = input_width
scale = base_width / float(input_width)
# model_algo = "vgg16_reduced"
model_algo = "resnet50"
if os.path.isfile('/opt/awscam/artifacts/deploy_ssd_' + model_algo +
'_' + str(input_width) + '.xml'):
model_path = '/opt/awscam/artifacts/deploy_ssd_' + model_algo + '_' + str(
input_width) + '.xml'
else:
#ret, model_path = mo.optimize('deploy_ssd_' + model_algo + '_' + str(input_width),input_width, input_height, aux_inputs={'--img-channels': 2})
ret, model_path = mo.optimize(
'deploy_ssd_' + model_algo + '_' + str(input_width),
input_width, input_height)
#model_path = '/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml'
# Load the model onto the GPU.
client.publish(topic=iot_topic,
payload='Loading object detection model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic,
payload='Object detection model loaded')
# Set the threshold for detection
detection_threshold = 0.5
# Do inference until the lambda is killed.
ballIn = False
ballInTime = datetime.datetime(1970, 1, 1)
countOverlap = 0
mod = 1
send_no = False
while True:
print(" ")
timeNow = datetime.datetime.now()
timeDiff = timeNow - ballInTime
if (timeDiff.total_seconds() >= 5):
ballIn = False
ballInTime = datetime.datetime(1970, 1, 1)
countOverlap = 0
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
frame_debug = frame
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = frame[670:None, 900:1750]
# frame_resize = frame[670:670+450, 900+250:900+250+450]
# frame_resize_file = "/tmp/frame_resize.jpeg"
#if not os.path.exists(frame_resize_file):
# os.mkfifo(frame_resize_file)
# cv2.imwrite(frame_resize_file, frame_resize)
# frame_resize = cv2.imread(frame_resize_file)
# frame_resize = cv2.imread("/home/aws_cam/test2-small.jpeg")
frame_resize = cv2.resize(frame_resize,
(input_width, input_height))
# frame_resize = cv2.cvtColor(frame_resize, cv2.COLOR_BGR2GRAY)
# frame_resize = cv2.cvtColor(frame_resize, cv2.COLOR_GRAY2BGR)
# frame_resize = cv2.imread("/home/aws_cam/test2-small.jpeg")
#cv2.imwrite("/tmp/frame" + str(input_width) + ".jpeg", frame_resize)
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
a = datetime.datetime.now()
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
b = datetime.datetime.now()
c = b - a
print("Inference Time: " + str(c.total_seconds()))
# print(parsed_inference_results)
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yoffset = 670
xoffset = 900
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
balls = []
holes = []
hole = None
# Get the detected objects and probabilities
for obj in parsed_inference_results[model_type]:
if obj['prob'] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xoffset + obj['xmin'] * scale)
ymin = int(yoffset + obj['ymin'] * scale)
xmax = int(xoffset + obj['xmax'] * scale)
ymax = int(yoffset + obj['ymax'] * scale)
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
(255, 165, 20), 10)
# cv2.rectangle(frame_resize, (int(obj['xmin']), int(obj['ymin'])), (int(obj['xmax']), int(obj['ymax'])), (255, 165, 20), 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color,
# and tickness
cv2.putText(
frame_debug,
"{}: {:.2f}%".format(output_map[obj['label']],
obj['prob'] * 100),
(xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX,
2.5, (255, 165, 20), 6)
# Store label and probability to send to cloud
cloud_output[output_map[obj['label']]] = obj['prob']
if obj['label'] == 1 and obj['prob'] > 0.7:
holes.append(obj)
elif obj['label'] == 0:
balls.append(obj)
for hole_i in holes:
if hole is None:
hole = hole_i
else:
if hole_i['prob'] > hole['prob']:
hole = hole_i
for ball in balls:
x_overlap = 0
y_overlap = 0
ballArea = 10000
holeArea = 0
try:
x_overlap = min(hole["xmax"], ball["xmax"]) - max(
hole["xmin"], ball["xmin"])
y_overlap = min(hole["ymax"], ball["ymax"]) - max(
hole["ymin"], ball["ymin"])
ballArea = (ball["xmax"] - ball["xmin"]) * (ball["ymax"] -
ball["ymin"])
holeArea = (hole["xmax"] - hole["xmin"]) * (hole["ymax"] -
hole["ymin"])
except Exception as ex:
print(ex)
overlapArea = max(0, max(x_overlap, 0) * max(y_overlap, 0))
overlapPerc = overlapArea / ballArea
print("Overlap " + str(overlapPerc))
cloud_output["ballArea"] = ballArea
cloud_output["holeArea"] = holeArea
cloud_output["overlapPerc"] = overlapPerc
if overlapPerc >= 0.5 and ballArea < 370:
countOverlap += 1
if (countOverlap >= 1):
ballInTime = datetime.datetime.now()
ballIn = True
countOverlap = 0
if (ballIn):
cloud_output["scored"] = "yes"
cv2.putText(frame, "Scored!", (300, 300),
cv2.FONT_HERSHEY_DUPLEX, 5, (255, 165, 20), 6)
cv2.putText(frame_debug, "Scored!", (300, 300),
cv2.FONT_HERSHEY_DUPLEX, 5, (255, 165, 20), 6)
else:
cloud_output["scored"] = "no"
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# cv2.imwrite('/tmp/result.jpeg', frame)
# cv2.imwrite('/tmp/frame_resize.jpeg', frame_resize)
# Send results to the cloud
if (mod % 1) == 0:
if (ballIn):
send_no = False
client.publish(topic=iot_topic,
payload=json.dumps(cloud_output))
mod = 1
else:
mod = mod + 1
if (not send_no and not ballIn):
client.publish(topic=iot_topic,
payload=json.dumps(cloud_output))
send_no = True
print(json.dumps(cloud_output))
d = datetime.datetime.now()
runtime = d - timeNow
print("Total Runtime: " + str(runtime.total_seconds()))
def infinite_infer_run():
""" Run the DeepLens inference loop frame by frame"""
try:
# Number of top classes to output
num_top_k = 2
model_type = 'classification'
model_name = 'image-classification'
with open('labels.txt', 'r') as f:
output_map = [l for l in f]
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# Optimize the model
error, model_path = mo.optimize(model_name, INPUT_WIDTH, INPUT_HEIGHT)
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload='Loading model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='Model loaded')
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Fixing eyefish calibration
undistorted_img = undistort(frame)
# Isolate the drawing (get frames)
isolated_frame = isolate_image(undistorted_img)
# # Check if cropping a frame is successful
# if (isolated_frame == None)
# print("Cannot crop the frame")
# break
# Resize frame to the same size as the training set
frame_resize = cv2.resize(isolated_frame,
(INPUT_HEIGHT, INPUT_WIDTH))
# Pre processing the input image
# Convert the image to grap scale and invert the color
gray = cv2.cvtColor(frame_resize, cv2.COLOR_BGR2GRAY)
gray = np.array(gray, float)
gray /= 255
normImage = 1 - gray
# crop horizontal frames
normImage[0:20] = 0
normImage[108:] = 0
# crop vertical frames
normImage[:, 0:20] = 0
normImage[:, 108:] = 0
normImage = crop_image(normImage, 0.4)
top, bottom, left, right = [5] * 4
img_with_border = cv2.copyMakeBorder(normImage,
top,
bottom,
left,
right,
cv2.BORDER_CONSTANT,
value=[0, 0])
border_image = cv2.resize(img_with_border,
(INPUT_HEIGHT, INPUT_WIDTH))
kernel = np.ones((3, 3), np.uint8)
dilated_image = cv2.dilate(border_image, kernel, iterations=2)
stacked_img = 255 * np.stack((dilated_image, ) * 3, axis=-1)
finalOutput = stacked_img.astype(np.uint8)
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a classification model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(finalOutput))
# Get top k results with highest probabilities
top_k = parsed_inference_results[model_type][0:num_top_k]
# Add the label of the top result to the frame used by local display.
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color, and thickness
output_text = '{} : {:.2f}'.format(output_map[top_k[0]['label']],
top_k[0]['prob'])
cv2.putText(frame, output_text, (10, 70), cv2.FONT_HERSHEY_SIMPLEX,
3, (255, 165, 20), 8)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send the top k results to the IoT console via MQTT
cloud_output = {}
for obj in top_k:
cloud_output[output_map[obj['label']]] = obj['prob']
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
print('Error in lambda {}'.format(ex))
client.publish(topic=iot_topic,
payload='Error in lambda: {}'.format(ex))
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
try:
model_type = 'classification'
############################################################
### ###
### Map inference return codes to outputs ###
### Map inference return codes to text colour ###
### ###
############################################################
output_map = {
0: 'compliant',
1: 'not compliant',
2: 'unknown',
3: 'no subject'
}
green = (0, 255, 0)
red = (0, 0, 255)
blue = (255, 0, 0)
colour_map = {0: green, 1: red, 2: blue, 3: blue}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(
os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
############################################################
### ###
### Optimize model for Intel using clDNN ###
### ###
############################################################
model_name = 'image-classification'
error, model_path = mo.optimize(model_name,
224,
224,
aux_inputs={"--epoch": 2})
# Load the model onto the GPU.
client.publish(topic=iot_topic,
payload='Loading Worksite safety model to GPU')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='Worksite safety model loaded')
# Since this is a four class classifier only retrieve 4 classes.
num_top_k = 4
# The height and width of the training set images
input_height = 224
input_width = 224
counter = 0
# Do inference until the lambda is killed.
while True:
############################################################
### ###
### Get a frame from the video stream ###
### ###
############################################################
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
############################################################
### ###
### Resize frame to the same size as model expects ###
### ###
############################################################
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a classification model,
# a simple API is provided.
############################################################
### ###
### Perform inference against model ###
### ###
############################################################
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
#client.publish(topic=iot_topic, payload="About to publish parsed_inference_results")
############################################################
### ###
### Get top results with highest probabilities ###
### ###
############################################################
top_k = parsed_inference_results[model_type][0:num_top_k]
print top_k
adjusted_results = top_k
# Add the label of the top result to the frame used by local display.
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, colour (in BGR), and thickness
############################################################
### ###
### Set output text and text colour based on top result ###
### ###
############################################################
output = output_map[top_k[0]['label']]
confidence = top_k[0]['prob']
colour = colour_map[top_k[0]['label']]
############################################################
### ###
### If confidence is low, force result to unsure ###
### ###
############################################################
if ((output == "compliant" or output == "not compliant")
and (confidence < 0.5)):
adjusted_results[0]['label'] = 2
adjusted_results[0]['prob'] = 1
output = output_map[adjusted_results[0]['label']]
colour = colour_map[adjusted_results[0]['label']]
############################################################
### ###
### Superimpose text on the stored frame ###
### ###
############################################################
cv2.putText(frame, output, (20, 150), cv2.FONT_HERSHEY_SIMPLEX, 6,
colour, 8)
############################################################
### ###
### Display marked up frame in project stream ###
### ###
############################################################
local_display.set_frame_data(frame)
# Every 10th inference, send the top k results to the IoT console via MQTT
counter = counter + 1
if counter >= 10:
cloud_output = {}
for obj in top_k:
cloud_output[output_map[obj['label']]] = obj['prob']
#client.publish(topic=iot_topic, payload="About to publish cloud_output")
client.publish(topic=iot_topic,
payload=json.dumps(cloud_output))
counter = 0
except Exception as ex:
client.publish(
topic=iot_topic,
payload='Error in worksite safety lambda: {}'.format(ex))
def greengrass_infinite_infer_run():
try:
input_width = 224
input_height = 224
model_name = "image-classification"
error, model_path = mo.optimize(model_name,
input_width,
input_height,
aux_inputs={'--epoch': 60})
# Load model to GPU (use {"GPU": 0} for CPU)
mcfg = {"GPU": 1}
model = awscam.Model(model_path, mcfg)
client.publish(topic=iotTopic, payload="Model loaded")
model_type = "classification"
topk = 5
results_thread = FIFO_Thread()
results_thread.start()
# Send a starting message to IoT console
client.publish(topic=iotTopic, payload="Inference is starting")
doInfer = True
while doInfer:
# Get a frame from the video stream
ret, frame0 = awscam.getLastFrame()
# Raise an exception if failing to get a frame
if ret == False:
raise Exception("Failed to get frame from the stream")
height, width, channels = frame0.shape
y = height / 2
x = width / 2
f = 320
frame = frame0[y - f:y + f, x - f:x + f]
# Resize frame to fit model input requirement
frameResize = cv2.resize(frame, (input_width, input_height))
# Run model inference on the resized frame
inferOutput = model.doInference(frameResize)
# Output inference result to the fifo file so it can be viewed with mplayer
parsed_results = model.parseResult(model_type, inferOutput)
#top_k = parsed_results[model_type][0:topk]
msg = '{'
msg += '"{}": "{}"'.format(inferOutput, parsed_results)
msg += "}"
obj = parsed_results[model_type][0]
if obj["prob"] > 0.9:
if obj["label"] == 1:
msg = "dog : {:.4f}".format(obj["prob"])
elif obj["label"] == 0:
msg = "cat : {:.4f}".format(obj["prob"])
client.publish(topic=iotTopic, payload=msg)
cv2.putText(frame, msg, (0, 22), cv2.FONT_HERSHEY_SIMPLEX, 1,
(255, 165, 20), 4)
global jpeg
ret, jpeg = cv2.imencode('.jpg', frame)
except Exception as e:
msg = "myModel Lambda failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
client.publish(topic=iot_topic, payload='Start of run loop...')
try:
# This object detection model is implemented as single shot detector (ssd), since
# the number of labels is small we create a dictionary that will help us convert
# the machine labels to human readable labels.
model_type = 'ssd'
output_map = {}
with open('classes.txt') as f:
for line in f:
(key, val) = line.split()
output_map[int(key)] = val
client.publish(topic=iot_topic, payload='Classes to be detected: ' + str(output_map))
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The height and width of the training set images
input_height = 512
input_width = 512
# Load the model onto the GPU.
# optimize the model
client.publish(topic=iot_topic, payload='Optimizing model...')
ret, model_path = mo.optimize('deploy_ssd_resnet50_512', input_width, input_height)
# load the model
client.publish(topic=iot_topic, payload='Loading model...')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='Custom object detection model loaded')
# Set the threshold for detection
detection_threshold = 0.40
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API. Note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
client.publish(topic=iot_topic, payload='Calling inference on next frame...')
parsed_inference_results = model.parseResult(model_type,
model.doInference(frame_resize))
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0]) / float(input_height)
xscale = float(frame.shape[1]) / float(input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Get the detected objects and probabilities
default_color = (255, 165, 20)
i = 0
for obj in parsed_inference_results[model_type]:
if obj['prob'] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj['xmin'])
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax'])
ymax = int(yscale * obj['ymax'])
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), default_color, 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color,
# and thickness
cv2.putText(frame, "{}: {:.2f}%".format(output_map[obj['label']],
obj['prob'] * 100),
(xmin, ymin-text_offset),
cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6)
# Store label and probability to send to cloud
cloud_output[output_map[obj['label']]] = obj['prob']
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send results to the cloud
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(topic=iot_topic, payload='Error in object detection lambda: {}'.format(ex))
def head_detection():
""" This method serves as the main entry point of our lambda.
"""
# Creating a client to send messages via IoT MQTT to the cloud
client = greengrasssdk.client('iot-data')
# This is the topic where we will publish our messages too
iot_topic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME'])
try:
# define model prefix and the amount to down sample the image by.
input_height = 84
input_width = 84
# Send message to IoT via MQTT
client.publish(topic=iot_topic, payload="Optimizing model")
ret, model_path = mo.optimize("deeplens-headpose-detection", input_width, input_height, platform='tf')
# Send message to IoT via MQTT
client.publish(topic=iot_topic, payload="Model optimization complete")
if ret is not 0:
raise Exception("Model optimization failed, error code: {}".format(ret))
# Send message to IoT via MQTT
client.publish(topic=iot_topic, payload="Loading model")
# Load the model into cl-dnn
model = awscam.Model(model_path, {"GPU": 1})
# Send message to IoT via MQTT
client.publish(topic=iot_topic, payload="Model loaded")
# We need to sample a frame so that we can determine where the center of
# the image is located. We assume that
# resolution is constant during the lifetime of the lmabda.
ret, sample_frame = awscam.getLastFrame()
if not ret:
raise Exception("Failed to get frame from the stream")
# Construct the custom helper object. This object just lets us handle postprocessing
# in a managable way.
head_pose_detection = HeadDetection(sample_frame.shape[1]/2, sample_frame.shape[0]/2)
# Dictionary that will allow us to convert the inference labels
# into a human a readable format.
output_map = ({0:'Bottom Right', 1:'Middle Right', 2:'Top Right', 3:'Bottom Middle',
4:'Middle Middle', 5:'Top Middle', 6:'Bottom Left', 7:'Middle Left',
8:'Top Left'})
# This model is a ResNet classifier, so our out put will be classification.
model_type = "classification"
# Define a rectangular region where we expect the persons head to be.
crop_upper_left_y = 440
crop_height = 640
crop_upper_left_x = 1024
crop_width = 640
while True:
# Grab the latest frame off the mjpeg stream.
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception("Failed to get frame from the stream")
# Mirror Image
frame = cv2.flip(frame, 1)
# Draw the rectangle around the area that we expect the persons head to be.
cv2.rectangle(frame, (crop_upper_left_x, crop_upper_left_y),
(crop_upper_left_x + crop_width, crop_upper_left_y + crop_height),
(255, 40, 0), 4)
# Crop the the frame so that we can do inference on the area of the frame where
# expect the persons head to be.
frame_crop = frame[crop_upper_left_y:crop_upper_left_y + crop_height,
crop_upper_left_x:crop_upper_left_x + crop_width]
# Down sample the image.
frame_resize = cv2.resize(frame_crop, (input_width, input_height))
# Renormalize the image so that the color magnitudes are between 0 and 1
frame_resize = frame_resize.astype(np.float32)/255.0
# Run the down sampled image through the inference engine.
infer_output = model.doInference(frame_resize)
# Parse the results so that we get back a more manageble data structure.
parsed_results = model.parseResult(model_type, infer_output)[model_type]
# Post process the image and send it to the FIFO file
head_pose_detection.send_data(frame, parsed_results)
# Send the results to MQTT in JSON format.
client.publish(topic=iot_topic,
payload=head_pose_detection.get_results(parsed_results, output_map))
except Exception as ex:
msg = "Lambda has failure, error msg {}: ".format(ex)
client.publish(topic=iot_topic, payload=msg)
def catcritter_infinite_infer_run():
""" Entry point of the lambda function"""
try:
# captured ssd image info for training
ssd_image_list = []
capture_class_images = False
class_labels = {
0: 'background',
1: 'buddy',
2: 'jade',
3: 'lucy',
4: 'tim'
}
# This object detection model is implemented as single shot detector (ssd), since
# the number of labels is small we create a dictionary that will help us convert
# the machine labels to human readable labels.
output_map = {
1: 'aeroplane',
2: 'bicycle',
3: 'bird',
4: 'boat',
5: 'bottle',
6: 'bus',
7: 'car',
8: 'cat',
9: 'chair',
10: 'cow',
11: 'dinning table',
12: 'dog',
13: 'horse',
14: 'motorbike',
15: 'person',
16: 'pottedplant',
17: 'sheep',
18: 'sofa',
19: 'train',
20: 'tvmonitor'
}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(
os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The height and width of the training set images
class_input_height = 100
class_input_width = 100
ssd_input_height = 300
ssd_input_width = 300
ssd_model_type = "ssd"
class_model_type = "classification"
class_model_name = "image-classification"
client.publish(topic=iot_topic, payload='optimizing model')
error, class_model_path = mo.optimize(class_model_name,
class_input_width,
class_input_height,
aux_inputs={'--epoch': 100})
# The aux_inputs is equal to the number of epochs and in this case, it is 100
# Load model to GPU (use {"GPU": 0} for CPU)
mcfg = {"GPU": 1}
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
ssd_model_path = '/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml'
# Load the model onto the GPU
client.publish(topic=iot_topic,
payload='Loading object detection model')
ssd_model = awscam.Model(ssd_model_path, mcfg)
class_model = awscam.Model(class_model_path, mcfg)
client.publish(topic=iot_topic,
payload='Object detection model loaded')
# Set the threshold for detection
detection_threshold = 0.25
counter = 1
ssd_counter = 1
irand = randrange(0, 1000)
num_classes = 4
# prepare training csv
if not os.path.isdir("/tmp/cats"):
os.mkdir("/tmp/cats")
os.mkdir("/tmp/cats/train")
os.chmod("/tmp/cats", stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
os.chmod("/tmp/cats/train",
stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
if not os.path.isfile("/tmp/cats/train/train.csv"):
with open('/tmp/cats/train/train.csv', 'a') as outcsv:
writer = csv.writer(outcsv)
writer.writerow(
['frame', 'xmin', 'xmax', 'ymin', 'ymax', 'class_id'])
outcsv.close()
today = datetime.datetime.now().strftime("%Y%m%d")
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame,
(ssd_input_height, ssd_input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = ssd_model.parseResult(
ssd_model_type, ssd_model.doInference(frame_resize))
#client.publish(topic=iot_topic, payload='ssd infer complete')
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0] / ssd_input_height)
xscale = float(frame.shape[1] / ssd_input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
image_saved = False
# Get the detected objects and probabilities
for obj in parsed_inference_results[ssd_model_type]:
if obj['prob'] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj['xmin']) \
+ int((obj['xmin'] - ssd_input_width/2) + ssd_input_width/2)
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax']) \
+ int((obj['xmax'] - ssd_input_width/2) + ssd_input_width/2)
ymax = int(yscale * obj['ymax'])
if xmin < 0:
xmin = 0
if ymin < 0:
ymin = 0
# if we found a cat, then save the image to a file and publish to IOT
if obj['label'] == 8 or obj['label'] == 12 or obj[
'label'] == 15:
# save the ssd image
if not image_saved:
frame_filename = "{}_{:03d}_{}_{:03d}".format(
today, ssd_counter, 'cats', irand)
frame_path = "/tmp/cats/train/" + frame_filename + '.jpg'
cv2.imwrite(frame_path, frame_resize)
ssd_counter += 1
image_saved = True
crop = frame[ymin:ymax, xmin:xmax].copy()
crop_resize = cv2.resize(
crop, (class_input_height, class_input_width))
# Run model inference on the cropped frame
inferOutput = class_model.doInference(crop_resize)
#client.publish(topic=iot_topic, payload='classification infer complete')
class_inference_results = class_model.parseResult(
class_model_type, inferOutput)
top_k = class_inference_results[class_model_type][
0:num_classes]
first = top_k[0]
if first['prob'] > detection_threshold:
#client.publish(topic=iot_topic, payload='found {}, saving file'.format(labels[first['label']]))
if capture_class_images:
dir = "/tmp/cats/train/" + frame_filename
if not os.path.isdir(dir):
os.mkdir(dir)
os.chmod(
dir, stat.S_IRWXU | stat.S_IRWXG
| stat.S_IRWXO)
the_label = class_labels[first['label']]
the_class = first['label']
if (obj['label'] == 15 and the_class < 3):
the_label = 'person'
the_class = 3
path = "/tmp/cats/train/{}/train_{}_{:03d}_{}_{:03d}_{:03d}_{:03d}_{:03d}_{:03d}.jpg".format(
frame_filename, today, counter, the_label,
irand, int(round(obj['xmin'])),
int(round(obj['xmax'])),
int(round(obj['ymin'])),
int(round(obj['ymax'])))
if capture_class_images:
cv2.imwrite(path, crop)
os.chmod(
path, stat.S_IRUSR | stat.S_IWUSR
| stat.S_IRGRP | stat.S_IWGRP
| stat.S_IROTH | stat.S_IWOTH)
counter += 1
msg = '{'
prob_num = 0
for kitty in top_k:
if prob_num == num_classes - 1:
msg += '"{}": {:.2f}'.format(
class_labels[kitty["label"]],
kitty["prob"] * 100)
else:
msg += '"{}": {:.2f},'.format(
class_labels[kitty["label"]],
kitty["prob"] * 100)
prob_num += 1
msg += "}"
# Send results to the cloud
#client.publish(topic=iot_topic, payload=json.dumps(msg))
else:
the_class = 0
the_label = class_labels[the_class]
path = "/tmp/cats/train/{}/train_{}_{:03d}_{}_{:03d}_{:03d}_{:03d}_{:03d}_{:03d}.jpg".format(
frame_filename, today, counter, the_label,
irand, int(round(obj['xmin'])),
int(round(obj['xmax'])),
int(round(obj['ymin'])),
int(round(obj['ymax'])))
if capture_class_images:
cv2.imwrite(path, crop)
os.chmod(
path, stat.S_IRUSR | stat.S_IWUSR
| stat.S_IRGRP | stat.S_IWGRP
| stat.S_IROTH | stat.S_IWOTH)
counter += 1
# create ssd entry
ssd_image_desc = [
frame_filename + ".jpg",
int(round(obj['xmin'])),
int(round(obj['xmax'])),
int(round(obj['ymin'])),
int(round(obj['ymax'])), the_class
]
ssd_image_list.append(ssd_image_desc)
if obj['label'] < 20:
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
(255, 165, 20), 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color,
# and tickness
cv2.putText(
frame,
"{}: {:.2f}%".format(output_map[obj['label']],
obj['prob'] * 100),
(xmin, ymin - text_offset),
cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
if image_saved:
with open('/tmp/cats/train/train.csv', 'a') as outcsv:
writer = csv.writer(outcsv)
writer.writerows(ssd_image_list)
ssd_image_list = []
outcsv.close()
except Exception as ex:
client.publish(
topic=iot_topic,
payload='Error in cat-plus-critter lambda: {}'.format(ex))
outcsv.close()
def greengrass_infinite_infer_run():
try:
input_width = 224
input_height = 224
model_name = "image-classification"
model_type = "classification"
client = greengrasssdk.client('iot-data')
iotTopic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME'])
local_display = LocalDisplay('480p')
local_display.start()
error, model_path = mo.optimize(model_name,input_width,input_height, aux_inputs={'--epoch': 30})
mcfg = {"GPU": 1}
print("model_path: " + model_path)
model = awscam.Model(model_path, mcfg)
client.publish(topic=iotTopic, payload="Model loaded")
with open('pinehead_labels.txt', 'r') as f:
labels = [l.rstrip() for l in f]
num_top_k = 2
# Send a starting message to IoT console
client.publish(topic=iotTopic, payload="Inference is starting")
doInfer = True
while doInfer:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
# Raise an exception if failing to get a frame
if ret == False:
raise Exception("Failed to get frame from the stream")
# Resize frame to fit model input requirement
s = frame.shape
cropped = frame[0:s[0], int((s[1]-s[0])/2):s[0]+int((s[1]-s[0])/2)]
frameResize = cv2.resize(cropped, (input_width, input_height))
# Run model inference on the resized frame
inferOutput = model.doInference(frameResize)
parsed_inference_results = model.parseResult(model_type, model.doInference(frameResize))
top_k = parsed_inference_results[model_type][0:num_top_k-1]
msg = "{"
msg += '"{}"'.format(labels[top_k[0]["label"]])
msg += "}"
client.publish(topic=iotTopic, payload = msg)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(frame, labels[top_k[0]["label"]], (10, 140), font, 5, (174, 235, 52), 10)
local_display.set_frame_data(frame)
except Exception as e:
msg = "myModel Lambda failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
