def capture_img(model_type, output_map, client, iot_topic, local_display,
model_path, model, detection_threshold, input_height,
input_width):
# client.publish(topic=iot_topic, payload='inside function')
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0] / input_height)
xscale = float(frame.shape[1] / input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Get the detected faces and probabilities
for obj in parsed_inference_results[model_type]:
if obj['prob'] > detection_threshold:
cloud_output[output_map[obj['label']]] = obj['prob']
# client.publish(topic=iot_topic, payload='Ajout a la liste')
liste_of_frame.append(frame)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
def infinite_infer_run():
""" Run the DeepLens inference loop frame by frame"""
try:
# Number of top classes to output
num_top_k = 2
model_type = 'classification'
model_name = 'image-classification'
with open('labels.txt', 'r') as f:
output_map = [l for l in f]
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# Optimize the model
error, model_path = mo.optimize(model_name, INPUT_WIDTH, INPUT_HEIGHT)
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload='Loading model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='Model loaded')
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (INPUT_HEIGHT, INPUT_WIDTH))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a classification model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
# Get top k results with highest probabilities
top_k = parsed_inference_results[model_type][0:num_top_k]
# Add the label of the top result to the frame used by local display.
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color, and thickness
output_text = '{} : {:.2f}'.format(output_map[top_k[0]['label']],
top_k[0]['prob'])
cv2.putText(frame, output_text, (10, 70), cv2.FONT_HERSHEY_SIMPLEX,
3, (255, 165, 20), 8)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send the top k results to the IoT console via MQTT
cloud_output = {}
for obj in top_k:
cloud_output[output_map[obj['label']]] = obj['prob']
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
print('Error in lambda {}'.format(ex))
client.publish(topic=iot_topic,
payload='Error in lambda: {}'.format(ex))
def greengrass_infinite_infer_run():
try:
client.publish(topic=iotTopic, payload="Trying main loop..")
ret, frameA = awscam.getLastFrame()
Stream = True
while Stream:
#time.sleep(1)
ret, frameB = awscam.getLastFrame()
difference = getDifference(frameA, frameB)
client.publish(topic=iotTopic,
payload="Difference: " + str(difference))
frameA = frameB
except Exception as e:
msg = "Test failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
Timer(15, greengrass_infinite_infer_run).start()
def infinite_infer_run():
session = onnxruntime.InferenceSession(PATH_TO_CKPT)
""" Run the DeepLens inference loop frame by frame"""
# Load the model here
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
img = cv2.resize(frame,
dsize=(640, 640),
interpolation=cv2.INTER_CUBIC)
res = img[:, :, ::-1].transpose(2, 0, 1)
res = np.expand_dims(res, axis=0).astype(np.float32)
# Using onnx model to get the bounding boxes of objects
outcome = session.run(None, {"images": res})
msg = '{'
for idx, val in enumerate(outcome):
msg += str(val)
msg += '}'
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, infinite_infer_run).start()
def infinite_infer_run():
""" Entry point of the lambda function"""
try:
# This cat-dog model is implemented as binary classifier, since the number
# of labels is small, create a dictionary that converts the machine
# labels to human readable labels.
model_type = 'classification'
output_map = {0: 'dog', 1: 'cat'}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
model_path = '/opt/awscam/artifacts/mxnet_resnet18-catsvsdogs_FP32_FUSED.xml'
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload='Loading action cat-dog model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='Cat-Dog model loaded')
# Since this is a binary classifier only retrieve 2 classes.
num_top_k = 2
# The height and width of the training set images
input_height = 224
input_width = 224
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a classification model,
# a simple API is provided.
parsed_inference_results = model.parseResult(model_type,
model.doInference(frame_resize))
# Get top k results with highest probabilities
top_k = parsed_inference_results[model_type][0:num_top_k]
# Add the label of the top result to the frame used by local display.
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color, and tickness
cv2.putText(frame, output_map[top_k[0]['label']], (10, 70),
cv2.FONT_HERSHEY_SIMPLEX, 3, (255, 165, 20), 8)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send the top k results to the IoT console via MQTT
cloud_output = {}
for obj in top_k:
cloud_output[output_map[obj['label']]] = obj['prob']
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(topic=iot_topic, payload='Error in cat-dog lambda: {}'.format(ex))
def analyze_frame():
print('analyze_frame called')
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
yscale = float(frame.shape[0] / input_height)
xscale = float(frame.shape[1] / input_width)
ret, jpeg = cv2.imencode('.jpg', frame)
if ret == False:
raise Exception("Failed to get frame from the stream")
# Resize frame to fit model input requirement
frameResize = cv2.resize(frame, (input_width, input_height))
# Run model inference on the resized frame
inferOutput = model.doInference(frameResize)
# Output inference result to the fifo file so it can be viewed with mplayer
parsed_results = model.parseResult(modelType, inferOutput)['ssd']
face_count = 0
male_count = 0
female_count = 0
person_array = []
for obj in parsed_results:
if obj['prob'] < prob_thresh:
continue
xmin = int(xscale * obj['xmin']) + int(
(obj['xmin'] - input_width / 2) + input_width / 2)
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax']) + int(
(obj['xmax'] - input_width / 2) + input_width / 2)
ymax = int(yscale * obj['ymax'])
roi = frame[ymin:ymax, xmin:xmax]
found, om_label_index, om_prob = recognize_face(
faceModel, roi, face_prob_thresh)
if found == True:
if om_label_index == 1:
male_count += 1
person_array.append(1)
else:
female_count += 1
person_array.append(0)
face_count += 1
global last_image
if face_count >= 1 and female_count == 0:
if last_image != 1:
pil_hom_image.show()
last_image = 1
elif face_count >= 1 and male_count == 0:
if last_image != 2:
pil_fem_image.show()
last_image = 2
elif male_count > 1 and female_count > 1:
face_count >= 1 and female_count == 0
def greengrass_infinite_infer_run():
try:
modelType = "classification"
model_name = "image-classification"
input_width = 224
input_height = 224
max_threshold = 0.75
error, model_path = mo.optimize(model_name,input_width,input_height)
outMap = { 0: "a", 1: "b", 2: "c", 3: "d", 4: "e", 5: "f", 6: "g", 7 : "h", 8 : "i", 9 : "k", 10 : "l", 11 : "m", 12 : "n", 13 : "o", 14 : "p", 15 : "q", 16 : "r", 17 : "s", 18 : "t", 19 : "u", 20 : "v", 21 : "w", 22 : "x", 23 : "y", 24 : "z" }
client.publish(topic=iotTopic, payload="Object detection starts now")
mcfg = {"GPU": 1}
model = awscam.Model(model_path, mcfg)
client.publish(topic=iotTopic, payload="Model loaded")
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
doInfer = True
while doInfer:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
# Raise an exception if failing to get a frame
if ret == False:
raise Exception("Failed to get frame from the stream")
margin = (frame.shape[1] - frame.shape[0]) / 2
cropped = frame[0:frame.shape[0], margin:frame.shape[1] - margin]
frameResize = cv2.resize(cropped, (input_width, input_height))
inferOutput = model.doInference(frameResize)
parsed_results = model.parseResult(modelType, inferOutput)["classification"]
client.publish(topic=iotTopic, payload = str(parsed_results))
for obj in parsed_results:
if obj["prob"] > max_threshold and detected != outMap[obj["label"]]:
detected = outMap[obj["label"]]
client.publish(topic=iotTopic, payload = outMap[obj["label"]])
playsound("../letters/" + outMap[obj["label"]] + ".mp3");
except Exception as e:
msg = "Test failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def infinite_infer_run():
""" Run the DeepLens inference loop frame by frame"""
try:
model_directory = "/opt/awscam/artifacts/"
model_name = "model" # onnx-model
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# When the ONNX model is imported via DeepLens console, the model is copied
# to the AWS DeepLens device, which is located in the "/opt/awscam/artifacts/".
model_file_path = os.path.join(model_directory, model_name)
sess = rt.InferenceSession(model_file_path)
class_labels = ['circle', 'square', 'star', 'triangle']
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Preprocess the frame to crop it into a square and
# resize it to make it the same size as the model's input size.
input_img = preprocess(frame)
# Inference.
inferences = makeInferences(sess, input_img)
inference = np.argmax(inferences)
# TODO: Add the label of predicted digit to the frame used by local display.
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color, and thickness
frame = cv2.putText(frame, class_labels[inference], (250, 300),
cv2.FONT_HERSHEY_SIMPLEX, 10, (0, 255, 0), 15)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Outputting the result logs as "MQTT messages" to AWS IoT.
cloud_output = {}
cloud_output["scores"] = inferences.tolist()
print(inference, cloud_output)
except Exception as ex:
# Outputting error logs as "MQTT messages" to AWS IoT.
print('Error in lambda {}'.format(ex))
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print("error details:" + str(exc_type) + str(fname) +
str(exc_tb.tb_lineno))
def loop():
try:
nextFrameIndexTime = datetime.datetime.now() + datetime.timedelta(
minutes=1)
log("Inside main loop.")
ret, previousFrame = awscam.getLastFrame()
doLoop = True
while doLoop:
ret, currentFrame = awscam.getLastFrame()
err = mse(previousFrame, currentFrame)
if (err > motionThreshold) or (datetime.datetime.now() >=
nextFrameIndexTime):
push_to_s3(currentFrame, err)
nextFrameIndexTime = datetime.datetime.now(
) + datetime.timedelta(minutes=1)
previousFrame = currentFrame
except Exception as e:
log("Error: " + str(e))
Timer(15, loop).start()
def greengrass_infinite_infer_run():
try:
global clients
input_width = 224
input_height = 224
model_name = "image-classification"
error, model_path = mo.optimize(model_name, input_width, input_height)
if error:
print error
raise Exception(error)
print("Optimized model")
model = awscam.Model(model_path, {"GPU": 1})
print("Loaded model")
model_type = "classification"
labels = ['a','b','c','d','e','f','g','h','i','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','not']
topk = 1
while True:
start = datetime.now()
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
height, width, channels = frame.shape
frame_cropped = frame[0:height, (width-height)/2:width-(width-height)/2]
frame_resize = cv2.resize(frame_cropped, (input_width, input_height))
infer_output = model.doInference(frame_resize)
parsed_results = model.parseResult(model_type, infer_output)
top_k = parsed_results[model_type][0:topk]
ret, jpeg = cv2.imencode(".jpg", frame_resize)
try:
end = datetime.now()
top_k[0]["label"] = labels[top_k[0]["label"]];
top_k[0]["jpeg"] = b64encode(jpeg)
top_k[0]["delay"] = (end - start).microseconds / 1000
msg = json.dumps(top_k[0]) + u''
for client in clients:
client.sendMessage(msg)
except:
print("error", sys.exc_info()[0])
except Exception as e:
print("Lambda failed: " + str(e))
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def infinite_infer_run():
""" Run the DeepLens inference loop frame by frame"""
try:
model_directory = "/opt/awscam/artifacts/"
model_name = "my_model.onnx" # onnx-model
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# When the ONNX model is imported via DeepLens console, the model is copied
# to the AWS DeepLens device, which is located in the "/opt/awscam/artifacts/".
model_file_path = os.path.join(model_directory, model_name)
sess = rt.InferenceSession(model_file_path)
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Preprocess the frame to crop it into a square and
# resize it to make it the same size as the model's input size.
input_img = preprocess(frame)
# Inference.
inferences = makeInferences(sess, input_img)
inference = np.argmax(inferences) + 1 # + 1 because of zero-indexing of classes
# Add the label of predicted digit to the frame used by local display.
cv2.putText(frame, f"{inference}", (300,600), cv2.FONT_HERSHEY_SIMPLEX, 8, (255,0,0), 7)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Outputting the result logs as "MQTT messages" to AWS IoT.
cloud_output = {}
cloud_output["scores"] = inferences.tolist()
print(inference, cloud_output)
except Exception as ex:
# Outputting error logs as "MQTT messages" to AWS IoT.
print('Error in lambda {}'.format(ex))
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print("error details:" + str(exc_type) + str(fname) + str(exc_tb.tb_lineno))
def run(self):
# Load the optimized object detection model
self.model = awscam.Model(model_path, {'GPU': 1})
while not self.stop_request.isSet():
res, frame = awscam.getLastFrame()
if not res:
continue
frame_resize = cv2.resize(frame, (input_height, input_width))
# Process the frame data with the object detection model and parse the result with
# the AWS DeepLens' builtin API.
result = self.model.parseResult(
model_type, self.model.doInference(frame_resize))
self.yscale = float(frame.shape[0]) / float(input_height)
self.xscale = float(frame.shape[1]) / float(input_width)
self.process_result(result, frame)
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
try:
print("About to greengrass_infinite_infer_run()")
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(
os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# path is required.
# The height and width of the training set images
input_height = 300
input_width = 300
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
#frame_resize = cv2.resize(frame, (input_height, input_width))
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
frame = cv2.rectangle(frame, (0, 0), (40, 20), (255, 165, 20), 10)
# Amount to offset the label/probability text above the bounding box.
#text_offset = 15
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color,
# and tickness
#cv2.putText(frame, "Project Stream"),
# (0, text_offset),
# cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send results to the cloud
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(
topic=iot_topic,
payload='Error in object detection lambda: {}'.format(ex))
def infinite_infer_run():
""" Run the DeepLens inference loop frame by frame"""
try:
s3BucketName = "dmanwill-project-dataset"
deepLensTempDirectory = "/tmp"
s3Client = boto3.client(
's3',
aws_access_key_id="", # TODO
aws_secret_access_key="" # TODO
)
local_display = LocalDisplay('480p')
local_display.start()
# Collects 200 images (can change to whatever number of images desired)
for i in range(200):
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
try:
# Preprocessing the image
preprocessedImage = preprocess(frame)
local_display.set_frame_data(frame)
frameImageFileName = f"image{i}.jpg"
frameImageDeepLensLocation = os.path.join(
deepLensTempDirectory, frameImageFileName)
preprocessedImage.save(frameImageDeepLensLocation)
s3Client.upload_file(Filename=frameImageDeepLensLocation,
Bucket=s3BucketName,
Key=frameImageFileName)
os.remove(frameImageDeepLensLocation)
time.sleep(0.5)
print(f"Saving image {i}")
except Exception as e:
print(e)
except Exception as ex:
print('Error in lambda {}'.format(ex))
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print("error details:" + str(exc_type) + str(fname) +
str(exc_tb.tb_lineno))
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
try:
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client("iot-data")
iot_topic = "$aws/things/{}/infer".format(
os.environ["AWS_IOT_THING_NAME"])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay("480p")
local_display.start()
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
model_path = "/opt/awscam/artifacts/mxnet_style_FP32_FUSE.xml"
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload="Loading style transfer model")
model = awscam.Model(model_path, {"GPU": 1})
client.publish(topic=iot_topic, payload="Style transfer model loaded")
# The style model is classified as a segmentation model, in the sense that the output is
# an image.
model_type = "segmentation"
# The height and width of the training set images
input_height = 224
input_width = 224
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception("Failed to get frame from the stream")
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results
infer_output = model.doInference(frame_resize)
parsed_results = infer_output["Convolution_last_conv"]
parsed_results = np.reshape(parsed_results[::-1],
(input_height, input_width, 3),
order="F")
parsed_results = np.rot90(parsed_results)
# Set the next frame in the local display stream.
local_display.set_frame_data(parsed_results * 2)
except Exception as ex:
print("Error", ex)
client.publish(topic=iot_topic,
payload="Error in style transfer lambda: {}".format(ex))
def readLastFrame(self):
'''
source AWSCAM :
source WEBCAM :
source localFile :
'''
# READ FROM AWSCAM OR WEBCAM
if self.source == "AWSCAM":
import awscam
ret, frame = awscam.getLastFrame()
return (ret, frame)
elif self.source == "WEBCAM":
ret, frame = self.cap.read()
return (ret, frame)
else:
ret, frame = self.cap.read()
return (ret, frame)
# READ FROM LOCAL DISK WITH METADATA
'''
save_dir = "/home/aws_cam/Desktop/video"
metadata = os.path.join(save_dir, "metadata.txt")
num = float(time.time())
data = dict()
with open(metadata, "r", os.O_NONBLOCK) as f:
lines = f.readlines()
for line in lines:
line = (line.strip()).split(',')
data[line[0]] = line[1]
start_time = time.time()
ans = data[num] if num in data.keys() else data[min(data.keys(), key=lambda k: abs(float(k)-num))]
ret = True
frame = cv2.imread(os.path.join(save_dir, "frame_" + str(ans) + ".jpg"), 1)
print(ans, time.time() - start_time)
return (ret, frame)
'''
raise NotImplementedError("Local file read last frame not defined.")
def greengrass_infinite_infer_run():
try:
results_thread = FIFO_Thread()
results_thread.start()
# Send a starting message to the AWS IoT console.
client.publish(topic=iotTopic, payload="About to do nothing")
ret, frame = awscam.getLastFrame()
# Convert to grayscale
gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
global jpeg
ret,jpeg = cv2.imencode('.jpg', gray_frame)
except Exception as e:
msg = "Test failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds.
Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run():
try:
modelPath = "/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml"
modelType = "ssd"
input_width = 300
input_height = 300
results_thread = FIFO_Thread()
results_thread.start()
print '** Object detection starts now'
try:
voice.say_phrase('loading')
# threading.Thread(target=voice.say_phrase, args=('loading', )).start()
except Exception as e:
print '** Unable to say loading'
mcfg = {"GPU": 1}
model = awscam.Model(modelPath, mcfg)
print '** Model loaded'
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
print '** Got last frame'
yscale = float(frame.shape[0] / input_height)
xscale = float(frame.shape[1] / input_width)
firstGo = True
spotted_objects_last_time = {}
print '** Inference now starting'
doInfer = True
while doInfer:
spotted_objects_this_time = {}
if firstGo:
print '** First time - about to play welcome message'
try:
questions.intro()
# threading.Thread(target=questions.intro).start()
except Exception as e:
print "!! Unable to play intro: " + str(e)
print '** First time - finished playing welcome message'
firstGo = False
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
# Raise an exception if failing to get a frame
if ret == False:
print "!! Failed to get frame from the stream"
raise Exception("Failed to get frame from the stream")
# Resize frame to fit model input requirement
frameResize = cv2.resize(frame, (input_width, input_height))
# Run model inference on the resized frame
inferOutput = model.doInference(frameResize)
# Output inference result to the fifo file so it can be viewed with mplayer
parsed_results = model.parseResult(modelType, inferOutput)['ssd']
# print "Before parsed results..."
for obj in parsed_results:
object_name = out_map[obj['label']]
if object_name in thresholds:
if obj['prob'] > thresholds[object_name]:
object_name = out_map[obj['label']]
spotted_objects_this_time[object_name] = obj['prob']
xmin = int(xscale * obj['xmin']) + int(
(obj['xmin'] - input_width / 2) + input_width / 2)
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax']) + int(
(obj['xmax'] - input_width / 2) + input_width / 2)
ymax = int(yscale * obj['ymax'])
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
(255, 165, 20), 4)
label_show = "XX:{}: {:.2f}%".format(
object_name, obj['prob'] * 100)
cv2.putText(frame, label_show, (xmin, ymin - 15),
cv2.FONT_HERSHEY_SIMPLEX, 1.2,
(255, 165, 20), 4)
global jpeg
ret, jpeg = cv2.imencode('.jpg', frame)
# print "...parsing results finished"
try:
considerChangeInObjects(spotted_objects_last_time,
spotted_objects_this_time)
except Exception as e:
print '!! Problem considering change in objects: ' + str(e)
spotted_objects_last_time = spotted_objects_this_time
except IOError as e:
print "FATAL I/O error({0}): {1}".format(e.errno, e.strerror)
except ValueError as e:
print "FATAL ValueError: " + str(e)
except 0:
print "Got a 0"
traceback.print_exc()
except Exception as e:
print "The non-0 exception was", e
print "Oh well - now running again:"
greengrass_infinite_infer_run()
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infer_image_run():
""" Entry point of the lambda function"""
try:
# This object detection model is implemented as single shot detector (ssd), since
# the number of labels is small we create a dictionary that will help us convert
# the machine labels to human readable labels.
model_type = 'ssd'
output_map = {1: 'aeroplane', 2: 'bicycle', 3: 'bird', 4: 'boat', 5: 'bottle', 6: 'bus',
7 : 'car', 8 : 'cat', 9 : 'chair', 10 : 'cow', 11 : 'dinning table',
12 : 'dog', 13 : 'horse', 14 : 'motorbike', 15 : 'person',
16 : 'pottedplant', 17 : 'sheep', 18 : 'sofa', 19 : 'train',
20 : 'tvmonitor'}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
model_path = '/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml'
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload='Loading object detection model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='Object detection model loaded')
# Set the threshold for detection
detection_threshold = 0.25
# The height and width of the training set images
input_height = 300
input_width = 300
# Do inference until the lambda is killed.
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = model.parseResult(model_type, model.doInference(frame_resize))
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0]) / float(input_height)
xscale = float(frame.shape[1]) / float(input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Get the detected objects and probabilities
for obj in parsed_inference_results[model_type]:
if obj['prob'] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj['xmin'])
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax'])
ymax = int(yscale * obj['ymax'])
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
cv2.putText(frame, "{}: {:.2f}%".format(output_map[obj['label']], obj['prob'] * 100),
(xmin, ymin-text_offset),
cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6)
# Store label and probability to send to cloud
cloud_output[output_map[obj['label']]] = obj['prob']
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send results to the cloud
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(topic=iot_topic, payload='Error in object detection lambda: {}'.format(ex))
def catcritter_infinite_infer_run():
""" Entry point of the lambda function"""
try:
# captured ssd image info for training
ssd_image_list = []
capture_class_images = False
class_labels = {
0: 'background',
1: 'buddy',
2: 'jade',
3: 'lucy',
4: 'tim'
}
# This object detection model is implemented as single shot detector (ssd), since
# the number of labels is small we create a dictionary that will help us convert
# the machine labels to human readable labels.
output_map = {
1: 'aeroplane',
2: 'bicycle',
3: 'bird',
4: 'boat',
5: 'bottle',
6: 'bus',
7: 'car',
8: 'cat',
9: 'chair',
10: 'cow',
11: 'dinning table',
12: 'dog',
13: 'horse',
14: 'motorbike',
15: 'person',
16: 'pottedplant',
17: 'sheep',
18: 'sofa',
19: 'train',
20: 'tvmonitor'
}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(
os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The height and width of the training set images
class_input_height = 100
class_input_width = 100
ssd_input_height = 300
ssd_input_width = 300
ssd_model_type = "ssd"
class_model_type = "classification"
class_model_name = "image-classification"
client.publish(topic=iot_topic, payload='optimizing model')
error, class_model_path = mo.optimize(class_model_name,
class_input_width,
class_input_height,
aux_inputs={'--epoch': 100})
# The aux_inputs is equal to the number of epochs and in this case, it is 100
# Load model to GPU (use {"GPU": 0} for CPU)
mcfg = {"GPU": 1}
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
ssd_model_path = '/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml'
# Load the model onto the GPU
client.publish(topic=iot_topic,
payload='Loading object detection model')
ssd_model = awscam.Model(ssd_model_path, mcfg)
class_model = awscam.Model(class_model_path, mcfg)
client.publish(topic=iot_topic,
payload='Object detection model loaded')
# Set the threshold for detection
detection_threshold = 0.25
counter = 1
ssd_counter = 1
irand = randrange(0, 1000)
num_classes = 4
# prepare training csv
if not os.path.isdir("/tmp/cats"):
os.mkdir("/tmp/cats")
os.mkdir("/tmp/cats/train")
os.chmod("/tmp/cats", stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
os.chmod("/tmp/cats/train",
stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
if not os.path.isfile("/tmp/cats/train/train.csv"):
with open('/tmp/cats/train/train.csv', 'a') as outcsv:
writer = csv.writer(outcsv)
writer.writerow(
['frame', 'xmin', 'xmax', 'ymin', 'ymax', 'class_id'])
outcsv.close()
today = datetime.datetime.now().strftime("%Y%m%d")
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame,
(ssd_input_height, ssd_input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = ssd_model.parseResult(
ssd_model_type, ssd_model.doInference(frame_resize))
#client.publish(topic=iot_topic, payload='ssd infer complete')
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0] / ssd_input_height)
xscale = float(frame.shape[1] / ssd_input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
image_saved = False
# Get the detected objects and probabilities
for obj in parsed_inference_results[ssd_model_type]:
if obj['prob'] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj['xmin']) \
+ int((obj['xmin'] - ssd_input_width/2) + ssd_input_width/2)
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax']) \
+ int((obj['xmax'] - ssd_input_width/2) + ssd_input_width/2)
ymax = int(yscale * obj['ymax'])
if xmin < 0:
xmin = 0
if ymin < 0:
ymin = 0
# if we found a cat, then save the image to a file and publish to IOT
if obj['label'] == 8 or obj['label'] == 12 or obj[
'label'] == 15:
# save the ssd image
if not image_saved:
frame_filename = "{}_{:03d}_{}_{:03d}".format(
today, ssd_counter, 'cats', irand)
frame_path = "/tmp/cats/train/" + frame_filename + '.jpg'
cv2.imwrite(frame_path, frame_resize)
ssd_counter += 1
image_saved = True
crop = frame[ymin:ymax, xmin:xmax].copy()
crop_resize = cv2.resize(
crop, (class_input_height, class_input_width))
# Run model inference on the cropped frame
inferOutput = class_model.doInference(crop_resize)
#client.publish(topic=iot_topic, payload='classification infer complete')
class_inference_results = class_model.parseResult(
class_model_type, inferOutput)
top_k = class_inference_results[class_model_type][
0:num_classes]
first = top_k[0]
if first['prob'] > detection_threshold:
#client.publish(topic=iot_topic, payload='found {}, saving file'.format(labels[first['label']]))
if capture_class_images:
dir = "/tmp/cats/train/" + frame_filename
if not os.path.isdir(dir):
os.mkdir(dir)
os.chmod(
dir, stat.S_IRWXU | stat.S_IRWXG
| stat.S_IRWXO)
the_label = class_labels[first['label']]
the_class = first['label']
if (obj['label'] == 15 and the_class < 3):
the_label = 'person'
the_class = 3
path = "/tmp/cats/train/{}/train_{}_{:03d}_{}_{:03d}_{:03d}_{:03d}_{:03d}_{:03d}.jpg".format(
frame_filename, today, counter, the_label,
irand, int(round(obj['xmin'])),
int(round(obj['xmax'])),
int(round(obj['ymin'])),
int(round(obj['ymax'])))
if capture_class_images:
cv2.imwrite(path, crop)
os.chmod(
path, stat.S_IRUSR | stat.S_IWUSR
| stat.S_IRGRP | stat.S_IWGRP
| stat.S_IROTH | stat.S_IWOTH)
counter += 1
msg = '{'
prob_num = 0
for kitty in top_k:
if prob_num == num_classes - 1:
msg += '"{}": {:.2f}'.format(
class_labels[kitty["label"]],
kitty["prob"] * 100)
else:
msg += '"{}": {:.2f},'.format(
class_labels[kitty["label"]],
kitty["prob"] * 100)
prob_num += 1
msg += "}"
# Send results to the cloud
#client.publish(topic=iot_topic, payload=json.dumps(msg))
else:
the_class = 0
the_label = class_labels[the_class]
path = "/tmp/cats/train/{}/train_{}_{:03d}_{}_{:03d}_{:03d}_{:03d}_{:03d}_{:03d}.jpg".format(
frame_filename, today, counter, the_label,
irand, int(round(obj['xmin'])),
int(round(obj['xmax'])),
int(round(obj['ymin'])),
int(round(obj['ymax'])))
if capture_class_images:
cv2.imwrite(path, crop)
os.chmod(
path, stat.S_IRUSR | stat.S_IWUSR
| stat.S_IRGRP | stat.S_IWGRP
| stat.S_IROTH | stat.S_IWOTH)
counter += 1
# create ssd entry
ssd_image_desc = [
frame_filename + ".jpg",
int(round(obj['xmin'])),
int(round(obj['xmax'])),
int(round(obj['ymin'])),
int(round(obj['ymax'])), the_class
]
ssd_image_list.append(ssd_image_desc)
if obj['label'] < 20:
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
(255, 165, 20), 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color,
# and tickness
cv2.putText(
frame,
"{}: {:.2f}%".format(output_map[obj['label']],
obj['prob'] * 100),
(xmin, ymin - text_offset),
cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6)
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
if image_saved:
with open('/tmp/cats/train/train.csv', 'a') as outcsv:
writer = csv.writer(outcsv)
writer.writerows(ssd_image_list)
ssd_image_list = []
outcsv.close()
except Exception as ex:
client.publish(
topic=iot_topic,
payload='Error in cat-plus-critter lambda: {}'.format(ex))
outcsv.close()
def greengrass_infinite_infer_run():
try:
modelPath = "/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml"
modelType = "ssd"
input_width = 300
input_height = 300
max_threshold = 0.25
outMap = ({
1: 'aeroplane',
2: 'bicycle',
3: 'bird',
4: 'boat',
5: 'bottle',
6: 'bus',
7: 'car',
8: 'cat',
9: 'chair',
10: 'cow',
11: 'dining table',
12: 'dog',
13: 'horse',
14: 'motorbike',
15: 'person',
16: 'pottedplant',
17: 'sheep',
18: 'sofa',
19: 'train',
20: 'tvmonitor'
})
results_thread = FIFO_Thread()
results_thread.start()
# Send a starting message to the AWS IoT console.
client.publish(topic=iotTopic, payload="Object detection starts now")
# Load the model to the GPU (use {"GPU": 0} for CPU).
mcfg = {"GPU": 1}
model = awscam.Model(modelPath, mcfg)
client.publish(topic=iotTopic, payload="Model loaded")
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
yscale = float(frame.shape[0] / input_height)
xscale = float(frame.shape[1] / input_width)
doInfer = True
while doInfer:
# Get a frame from the video stream.
ret, frame = awscam.getLastFrame()
# If you fail to get a frame, raise an exception.
if ret == False:
raise Exception("Failed to get frame from the stream")
# Resize the frame to meet the model input requirement.
frameResize = cv2.resize(frame, (input_width, input_height))
# Run model inference on the resized frame.
inferOutput = model.doInference(frameResize)
# Output the result of inference to the fifo file so it can be viewed with mplayer.
parsed_results = model.parseResult(modelType, inferOutput)['ssd']
label = '{'
for obj in parsed_results:
if obj['prob'] > max_threshold:
xmin = int(xscale * obj['xmin']) + int(
(obj['xmin'] - input_width / 2) + input_width / 2)
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax']) + int(
(obj['xmax'] - input_width / 2) + input_width / 2)
ymax = int(yscale * obj['ymax'])
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
(255, 165, 20), 4)
label += '"{}": {:.2f},'.format(outMap[obj['label']],
obj['prob'])
label_show = "{}: {:.2f}%".format(
outMap[obj['label']], obj['prob'] * 100)
cv2.putText(frame, label_show, (xmin, ymin - 15),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 165, 20),
4)
label += '"null": 0.0'
label += '}'
client.publish(topic=iotTopic, payload=label)
global jpeg
ret, jpeg = cv2.imencode('.jpg', frame)
except Exception as e:
msg = "Test failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds.
Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
try:
# This face detection model is implemented as single shot detector (ssd).
model_type = 'ssd'
output_map = {1: 'face'}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
thing_name = os.environ['AWS_IOT_THING_NAME']
iot_topic = '$aws/things/{}/infer'.format(thing_name)
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# Create a s3 backgound uploader
session = Session()
s3 = session.create_client('s3',
region_name=os.getenv(
'REGION_NAME', 'ap-southeast-2'))
bucket = os.getenv('FRAMES_BUCKET',
'virtual-concierge-frames-ap-southeast-2')
uploader = ImageUploader(s3, bucket, client, iot_topic)
uploader.start()
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
model_dir = '/opt/awscam/artifacts/'
model_path = model_dir + 'mxnet_deploy_ssd_FP16_FUSED.xml'
# Load the model onto the GPU.
msg = 'Loading face detection model for {}'.format(thing_name)
client.publish(topic=iot_topic, payload=msg)
model_start = time.time()
model = awscam.Model(model_path, {'GPU': 1})
msg = 'Face detection model loaded in {}s'.format(time.time() -
model_start)
client.publish(topic=iot_topic, payload=msg)
# Attempt to load scorer library
try:
model_start = time.time()
scorer = Scorer(model_dir)
msg = 'Image classification model loaded {} in {}s'.format(
scorer.vecs.shape[0],
time.time() - model_start)
client.publish(topic=iot_topic, payload=msg)
except Exception as e:
print('Failed to load scorer', e)
# Set the threshold for detection
detection_threshold = float(os.getenv('DETECT_THRESHOLD', '0.7'))
# This is the similarity threshold
sim_threshold = float(os.getenv('DETECT_THRESHOLD', '0.99'))
# The height and width of the training set images
input_height = 300
input_width = 300
# Do inference until the lambda is killed.
while True:
# get thing shadow state, to see if we should register
cloud_output = {}
# Get a frame from the video stream
cloud_output["frame_start"] = time.time()
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Future integrate the shadow callback
if False:
cloud_output["shadow_start"] = time.time()
shadow = client.get_thing_shadow(thingName=thing_name)
jsonState = json.loads(shadow["payload"])
register = jsonState['state']['desired'].get('register')
cloud_output["shadow_register"] = register
cloud_output["shadow_latency"] = time.time(
) - cloud_output["shadow_start"]
# Resize frame to the same size as the training set.
cloud_output["detect_start"] = time.time()
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
cloud_output["detect_latency"] = time.time(
) - cloud_output["detect_start"]
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0] / input_height)
xscale = float(frame.shape[1] / input_width)
# Dictionary to be filled with labels and probabilities for MQTT
# Get the detected faces and probabilities
for i, obj in enumerate(parsed_inference_results[model_type]):
if obj['prob'] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj['xmin']) \
+ int((obj['xmin'] - input_width/2) + input_width/2)
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax']) \
+ int((obj['xmax'] - input_width/2) + input_width/2)
ymax = int(yscale * obj['ymax'])
# Set the default title and color
title = '{:.2f}%'.format(obj['prob'] * 100)
color = (255, 0, 0) # blue
upload = False
if scorer:
try:
# Attempt to find similar face
cloud_output['classify_start'] = time.time()
bbox = [xmin, ymin, xmax, ymax]
vec = scorer.vectorize(frame, bbox)
sim, z_score, prob, name = scorer.similar(vec)
if prob >= sim_threshold:
title = name
if round(prob, 3) < 1.0:
title += ' ({:.2f}%)'.format(prob)
color = (0, 255, 0) # green
upload = True
cloud_output['classify'] = {
'name': name,
'sim': float(sim),
'zscore': float(z_score),
'prob': float(prob)
}
cloud_output['classify_latency'] = time.time(
) - cloud_output['classify_start']
except Exception as e:
msg = "Face similarity error: " + str(e)
client.publish(topic=iot_topic, payload=msg)
if upload:
try:
metadata = {
'ThingName':
thing_name,
'FullName':
title,
'Confidence':
str(obj['prob']),
'Similarity':
str(cloud_output['classify']['sim']),
'Probability':
str(cloud_output['classify']['prob']),
'FaceHeight':
str(xmax - xmin),
'FaceWidth':
str(ymax - ymin),
}
crop_img = uploader.crop(frame, xmin, ymin, xmax,
ymax)
item = uploader.upload(crop_img,
i,
metadata=metadata)
if item:
cloud_output['upload_key'] = item['key']
else:
cloud_output['upload_skip'] = True
except Exception as e:
msg = "Upload error: " + str(e)
client.publish(topic=iot_topic, payload=msg)
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cloud_output["draw_start"] = time.time()
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), color, 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 12
cv2.putText(frame, title, (xmin, ymin - text_offset),
cv2.FONT_HERSHEY_SIMPLEX, 2.5, color, 6)
# Store label and probability to send to cloud
cloud_output[output_map[obj['label']]] = obj['prob']
cloud_output["draw_latency"] = time.time(
) - cloud_output["draw_start"]
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
cloud_output["frame_end"] = time.time()
cloud_output["frame_latency"] = cloud_output[
"frame_end"] - cloud_output["frame_start"]
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
print('Error in face detection lambda: {}'.format(ex))
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
try:
lapse = 0
interval = 10
# This object detection model is implemented as single shot detector (ssd), since
# the number of labels is small we create a dictionary that will help us convert
# the machine labels to human readable labels.
model_type = 'ssd'
output_map = {1: 'thumbs-up'}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client('iot-data')
iot_topic = '$aws/things/{}/infer'.format(
os.environ['AWS_IOT_THING_NAME'])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
model_name = 'model_algo_1'
#aux_inputs = {--epoch: 30}
#error, model_path = mo.optimize(model_name, 512, 512, 'MXNet', aux_inputs)
#if not error:
#raise Exception('Failed to optimize model')
model_path = '/opt/awscam/artifacts/model_algo_1.xml'
# Load the model onto the GPU.
client.publish(topic=iot_topic,
payload='Loading object detection model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic,
payload='Object detection model loaded')
## Debiging AWS_IOT_THING_NAME
client.publish(topic=iot_topic,
payload=os.environ['AWS_IOT_THING_NAME'])
# Set the threshold for detection
detection_threshold = 0.90
# The height and width of the training set images
input_height = 512
input_width = 512
# A dictionary to identify device id's
devices = {}
devices['deeplens_HoVip9KQTXiC3UFub47lJA'] = "Seattle01"
devices['deeplens_bmTWwitIRUi_mASjZASUHA'] = "Chicago01"
devices['deeplens_Hgs6kj_yQASF2x-3fOxCHA'] = "Chicago02"
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
##Store a raw resized image before inference to be used for retraining later
raw_training = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize))
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0] / input_height)
xscale = float(frame.shape[1] / input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Index to keep track of multiple detections in a frame
index = 0
# Get the detected objects and probabilities
# A dictionary containing data for a single detection in a frame
frame_time = time.strftime("%Y%m%d-%H%M%S")
detection = {}
detection["device_id"] = devices[os.environ['AWS_IOT_THING_NAME']]
detection["timestamp"] = frame_time
# A list that will contain the information of the objects detected in the frame
objects_det = []
# A boolean recording if a detection was made or not
detection_made = False
##Set a list of annotations to be outputted in json file for training
annotations = []
for obj in parsed_inference_results[model_type]:
# A dictionary to contain the info from an object detected
object = {}
# A dictionary containing annotation info for retraining
annotation = {}
if obj['prob'] > detection_threshold:
# Set the annotation data for retraining
annotation['class_id'] = 0
annotation['left'] = int(obj['xmin'])
annotation['top'] = int(obj['ymin'])
annotation['width'] = abs(
int(obj['xmax']) - int(obj['xmin']))
annotation['height'] = abs(
int(obj['ymax']) - int(obj['ymin']))
## append to the list of annotations
annotations.append(annotation)
#Set detection_made to true
detection_made = True
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj['xmin']) \
+ int((obj['xmin'] - input_width/2))
ymin = int(yscale * obj['ymin']) \
+ int((obj['ymin'] - input_height/2))
xmax = int(xscale * obj['xmax']) \
+ int((obj['xmax'] - input_width/2))
ymax = int(yscale * obj['ymax']) \
+ int((obj['ymax'] - input_height/2))
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
(255, 165, 20), 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color,
# and tickness
cv2.putText(
frame, "{}: {:.2f}%".format(output_map[obj['label']],
obj['prob'] * 100),
(xmin, ymin - text_offset), cv2.FONT_HERSHEY_SIMPLEX,
2.5, (255, 165, 20), 6)
##This is putting bounding boxes for images resized to 512
cv2.rectangle(frame_resize,
(int(obj['xmin']), int(obj['ymin'])),
(int(obj['xmax']), int(obj['ymax'])),
(255, 165, 20), 10)
cv2.putText(
frame_resize,
"{}: {:.2f}%".format(output_map[obj['label']],
obj['prob'] * 100),
(int(obj['xmin']), int(obj['ymin']) - text_offset),
cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6)
# Store label and probability to send to cloud
#cloud_output[output_map[obj['label']]] = obj['prob']
##Add to the dictionary of cloudoutput the index of the detection
#cloud_output['index'] = index
# set detection data for the object
object["index"] = index
object["object"] = output_map[obj['label']]
object["confidence"] = obj['prob']
# append the object to the list of detections for the frame
objects_det.append(object)
index += 1
# Add the detections to the dictionary for the frame
detection["objects"] = objects_det
# add a link to the image to detections
img_file_name = 'images/image_' + detection[
"device_id"] + '_' + frame_time + '.jpg'
link = 'https://s3.amazonaws.com/thumbs-up-output-3/' + img_file_name
detection["link_to_img"] = link
#a filename for the raw image to be used for trianing later
raw_file_name = 'retrain_' + detection[
"device_id"] + '_' + frame_time
# Upload to S3 to allow viewing the image in the browser, only if a detection was made
if detection_made and lapse > interval:
#Create the json for retraining
training_json = create_training_json(raw_file_name,
annotations)
#Upload the retraining data
write_training_data(raw_file_name, raw_training, training_json)
#Upload the inference data
image_url = write_image_to_s3(frame, detection, frame_time,
img_file_name, devices,
frame_resize)
#Publish success messages
client.publish(topic=iot_topic,
payload='{{"img":"{}"}}'.format(image_url))
client.publish(topic=iot_topic, payload=json.dumps(detection))
lapse = 0
else:
client.publish(topic=iot_topic, payload="NO DETECTIONS MADE")
lapse += 1
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
except Exception as ex:
client.publish(
topic=iot_topic,
payload='Error in object detection lambda: {}'.format(ex))
def greengrass_infinite_infer_run():
try:
modelPath = "/opt/awscam/artifacts/mxnet_deploy_ssd_FP16_FUSED.xml"
modelType = "ssd"
input_width = 300
input_height = 300
prob_thresh = 0.1
results_thread = FIFO_Thread()
results_thread.start()
# Send a starting message to IoT console
client.publish(topic=iotTopic, payload="Face detection starts now")
# Load model to GPU (use {"GPU": 0} for CPU)
mcfg = {"GPU": 1}
model = awscam.Model(modelPath, mcfg)
client.publish(topic=iotTopic, payload="Model loaded")
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
yscale = float(frame.shape[0] / input_height)
xscale = float(frame.shape[1] / input_width)
font = cv2.FONT_HERSHEY_SIMPLEX
rgb_color = (255, 165, 20)
#Timers for cooldown and countdown
cooldown = datetime.datetime.now()
countdown = datetime.datetime.now()
doInfer = True
onCountdown = False
while doInfer:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
# Raise an exception if failing to get a frame
if ret == False:
raise Exception("Failed to get frame from the stream")
# Resize frame to fit model input requirement
frameResize = cv2.resize(frame, (input_width, input_height))
# Run model inference on the resized frame
inferOutput = model.doInference(frameResize)
# Output inference result to the fifo file so it can be viewed with mplayer
parsed_results = model.parseResult(modelType, inferOutput)['ssd']
label = '{'
msg = 'false'
time_now = datetime.datetime.now()
for obj in parsed_results:
if (obj['prob'] < prob_thresh):
break
xmin = int(xscale * obj['xmin']) + int(
(obj['xmin'] - input_width / 2) + input_width / 2)
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax']) + int(
(obj['xmax'] - input_width / 2) + input_width / 2)
ymax = int(yscale * obj['ymax'])
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), rgb_color, 4)
label += '"{}": {:.2f},'.format("prob", obj['prob'])
label_show = '{}: {:.2f}'.format(str(obj['label']),
obj['prob'])
cv2.putText(frame, label_show, (xmin, ymin - 15), font, 0.5,
rgb_color, 4)
msg = "true"
if (time_now >= cooldown) and obj['prob'] >= 0.60:
# Uploading to Amazon S3 if cooldown and countdown allow it
if onCountdown and time_now >= countdown:
message = "uploading to s3..."
client.publish(topic=iotTopic, payload=message)
key = 'images/frame-' + time.strftime(
"%Y%m%d-%H%M%S") + '.jpg'
session = Session()
s3 = session.create_client('s3')
_, jpg_data = cv2.imencode('.jpg', frame)
result = s3.put_object(Body=jpg_data.tostring(),
Bucket=bucket_name,
Key=key)
message = "uploaded to s3: " + key
client.publish(topic=iotTopic, payload=message)
cooldown = time_now + datetime.timedelta(seconds=10)
onCountdown = False
# Starting countdown
elif not onCountdown:
onCountdown = True
countdown = time_now + datetime.timedelta(seconds=4)
if not onCountdown:
cv2.putText(
frame, "Wait for picture: " +
str(max(0, int(
(cooldown - time_now).total_seconds()))) + " seconds",
(950, 100), font, 2, rgb_color, 4)
if int((cooldown - time_now).total_seconds()) >= 5:
cv2.putText(frame, "Image Uploaded! ", (1150, 200), font,
2, rgb_color, 4)
cv2.putText(frame, "Please check the leaderboard",
(900, 300), font, 2, rgb_color, 4)
else:
if int((countdown - time_now).total_seconds()) >= -5:
cv2.putText(frame, "Say Cheese!", (1000, 1000), font, 3,
rgb_color, 4)
cv2.putText(
frame,
str(max(0, int(
(countdown - time_now).total_seconds()))) + "...",
(1200, 1100), font, 3, rgb_color, 4)
else:
onCountdown = False
label += '"face": "' + msg + '"'
label += '}'
client.publish(topic=iotTopic, payload=label)
global jpeg
ret, jpeg = cv2.imencode('.jpg', frame)
except Exception as e:
msg = "Test failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
try:
# This face detection model is implemented as single shot detector (ssd).
model_type = 'ssd'
output_map = {1: 'face'}
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
model_path = '/opt/awscam/artifacts/mxnet_deploy_ssd_FP16_FUSED.xml'
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload='Loading face detection model')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='Face detection model loaded')
# Set the threshold for detection
detection_threshold = 0.25
# The height and width of the training set images
input_height = 300
input_width = 300
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = model.parseResult(model_type,
model.doInference(frame_resize))
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0]/input_height)
xscale = float(frame.shape[1]/input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Get the detected faces and probabilities
for i, obj in enumerate(parsed_inference_results[model_type]):
if obj['prob'] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj['xmin']) \
+ int((obj['xmin'] - input_width/2) + input_width/2)
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax']) \
+ int((obj['xmax'] - input_width/2) + input_width/2)
ymax = int(yscale * obj['ymax'])
if ymin >25:
ymin = ymin - 25
if ymax <275:
ymax = ymax + 25
if xmin > 25:
xmin = xmin - 25
if xmax <275:
xmax = xmax + 25
crop_img = frame[ymin:ymax, xmin:xmax]
push_to_s3(crop_img, i)
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
cv2.putText(frame, '{:.2f}%'.format(obj['prob'] * 100),
(xmin, ymin-text_offset),
cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6)
# Store label and probability to send to cloud
cloud_output[output_map[obj['label']]] = obj['prob']
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send results to the cloud
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(topic=iot_topic, payload='Error in face detection lambda: {}'.format(ex))
def infinite_infer_run():
""" Entry point of the lambda function"""
try:
# This object detection model is implemented as single shot detector (ssd), since
# the number of labels is small we create a dictionary that will help us convert
# the machine labels to human readable labels.
model_type = "ssd"
output_map = {
1: "aeroplane",
2: "bicycle",
3: "bird",
4: "boat",
5: "bottle",
6: "bus",
7: "car",
8: "cat",
9: "chair",
10: "cow",
11: "dinning table",
12: "dog",
13: "horse",
14: "motorbike",
15: "person",
16: "pottedplant",
17: "sheep",
18: "sofa",
19: "train",
20: "tvmonitor",
}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client("iot-data")
iot_topic = "$aws/things/{}/infer".format(os.environ["AWS_IOT_THING_NAME"])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay("480p")
local_display.start()
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
model_path = (
"/opt/awscam/artifacts/mxnet_deploy_ssd_resnet50_300_FP16_FUSED.xml"
)
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload="Loading object detection model")
model = awscam.Model(model_path, {"GPU": 1})
client.publish(topic=iot_topic, payload="Object detection model loaded")
# Set the threshold for detection
detection_threshold = 0.25
# The height and width of the training set images
input_height = 300
input_width = 300
"""extra part of code for rekognition"""
# model trained in us east 2
# projectVersionArn = "arn:aws:rekognition:us-east-2:510335724440:project/PPE_detection_May_2020/version/PPE_detection_May_2020.2020-06-01T23.33.22/1591025603184"
# model trained in us east 1, version 1
# projectVersionArn = "arn:aws:rekognition:us-east-1:510335724440:project/ppe-detection-deeplens/version/ppe-detection-deeplens.2020-06-12T14.25.57/1591943158364"
# model trained in us east 1, version 2
projectVersionArn = "arn:aws:rekognition:us-east-1:510335724440:project/ppe-detection-deeplens/version/ppe-detection-deeplens.2020-06-17T14.28.47/1592375328862"
rekognition = boto3.client("rekognition")
customLabels = []
s3 = boto3.client("s3")
iterator = 0
"""extra part of code for rekognition"""
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception("Failed to get frame from the stream")
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize)
)
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0]) / float(input_height)
xscale = float(frame.shape[1]) / float(input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Get the detected objects and probabilities
# for obj in parsed_inference_results[model_type]:
# if obj["prob"] > detection_threshold:
# # Add bounding boxes to full resolution frame
# xmin = int(xscale * obj["xmin"])
# ymin = int(yscale * obj["ymin"])
# xmax = int(xscale * obj["xmax"])
# ymax = int(yscale * obj["ymax"])
# # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# # for more information about the cv2.rectangle method.
# # Method signature: image, point1, point2, color, and tickness.
# # comment out the drawing part to avoid the results of two models all on one frame
# cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 10)
# # Amount to offset the label/probability text above the bounding box.
# text_offset = 15
# # See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# # for more information about the cv2.putText method.
# # Method signature: image, text, origin, font face, font scale, color,
# # and tickness
# cv2.putText(
# frame,
# "{}: {:.2f}%".format(
# output_map[obj["label"]], obj["prob"] * 100
# ),
# (xmin, ymin - text_offset),
# cv2.FONT_HERSHEY_SIMPLEX,
# 2.5,
# (255, 165, 20),
# 6,
# )
# # Store label and probability to send to cloud
# cloud_output[output_map[obj["label"]]] = obj["prob"]
# # Set the next frame in the local display stream.
# local_display.set_frame_data(frame)
# # Send results to the cloud
# client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
"""extra part of code for rekognition"""
hasFrame, imageBytes = cv2.imencode(".jpg", frame)
client.publish(topic=iot_topic, payload="import done")
if hasFrame:
response = rekognition.detect_custom_labels(
Image={"Bytes": imageBytes.tobytes(),},
ProjectVersionArn=projectVersionArn,
)
client.publish(topic=iot_topic, payload="analyse done")
# image = Img.fromarray(frame)
# imgWidth, imgHeight = image.size
# draw = ImageDraw.Draw(image)
imgHeight, imgWidth, c = frame.shape
image = frame
ppe = 0
person = 0
for elabel in response["CustomLabels"]:
# elabel["Timestamp"] = (frameId/frameRate)*1000
customLabels.append(elabel)
print("Label " + str(elabel["Name"]))
print("Confidence " + str(elabel["Confidence"]))
if str(elabel["Name"]) == "PPE":
ppe = ppe + 1
else if str(elabel["Name"]) == "person"
person = person + 1
if "Geometry" in elabel:
box = elabel["Geometry"]["BoundingBox"]
left = imgWidth * box["Left"]
top = imgHeight * box["Top"]
width = imgWidth * box["Width"]
height = imgHeight * box["Height"]
if str(elabel["Name"]) == "person":
cv2.putText(
image,
elabel["Name"],
(int(left), int(top)),
cv2.FONT_HERSHEY_COMPLEX,
1,
(0, 255, 0),
1,
)
else:
cv2.putText(
image,
elabel["Name"],
(int(left), int(top)),
cv2.FONT_HERSHEY_COMPLEX,
1,
(255, 0, 0),
1,
)
print("Left: " + "{0:.0f}".format(left))
print("Top: " + "{0:.0f}".format(top))
print("Label Width: " + "{0:.0f}".format(width))
print("Label Height: " + "{0:.0f}".format(height))
# points = (
# (left, top),
# (left + width, top),
# (left + width, top + height),
# (left, top + height),
# (left, top),
# )
# if str(elabel["Name"]) == "person":
# draw.line(points, fill="#00d400", width=3)
# else:
# draw.line(points, fill="#800000", width=3)
if str(elabel["Name"]) == "person":
cv2.rectangle(
image,
(int(left), int(top)),
(int(left + width), int(top + height)),
(0, 255, 0),
2,
)
else:
cv2.rectangle(
image,
(int(left), int(top)),
(int(left + width), int(top + height)),
(255, 0, 0),
2,
)
# save the image locally and then upload them into s3
client.publish(topic=iot_topic, payload="drawing done")
# cv2.imwrite("frame" + format(iterator) + ".jpg", image)
# image.save("frame" + format(iterator) + ".jpg")
# dont save it to the disk anymore
#client.publish(topic=iot_topic, payload="image saving done")
iterator = iterator + 1
# upload as bytes
# imageBytes = image.tobytes()
# with io.BytesIO(imageBytes) as f:
# s3.upload_fileobj(
# f,
# "custom-labels-console-us-east-1-5e4c514f5b",
# "frameID: " + format(iterator) + ".jpg",
# )
# write the metadata
# metadata = {"NumberOfPersons": str(person), "NumberOfPPEs": str(ppe)}
# upload as string
img_str = cv2.imencode('.jpg', image)[1].tostring()
s3.put_object(
Bucket="custom-labels-console-us-east-1-5e4c514f5b",
Key="frameID: " + format(iterator) + ".jpg",
Body=img_str,
ACL="public-read",
Metadata={"NumberOfPersons": str(person), "NumberOfPPEs": str(ppe)}
)
client.publish(topic=iot_topic, payload="send to s3 done")
# to retrieve meatadata in s3
# $ aws s3api head-object --bucket custom-labels-console-us-east-1-5e4c514f5b --key testImage.jpg
# print(customLabels)
"""extra part of code for rekognition"""
# cap.release() not sure if we need to keep this
except Exception as ex:
client.publish(
topic=iot_topic, payload="Error in object detection lambda: {}".format(ex)
)
def greengrass_infinite_infer_run():
try:
# TODO: Parameterize model name
model_name = 'deploy_ssd_resnet50_300'
model_path = "/opt/awscam/artifacts/mxnet_{}_FP16_FUSED.xml".format(
model_name)
model_type = "ssd"
# Trained image shape
input_width = 300
input_height = 300
with open('docs/synset.txt', 'r') as _f:
labels = dict(enumerate(_f, 1))
client.publish(topic=iotTopic, payload="Labels loaded")
# Start thread to write frames to disk
results_thread = FIFO_Thread()
results_thread.start()
# MxNet Model
model = awscam.Model(model_path, {"GPU": 1})
client.publish(topic=iotTopic, payload="Model loaded")
while True:
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
inferred_obj = InferEvent(frame, model_name, model_type)
frameResize = cv2.resize(frame, (input_width, input_height))
# Run inference
output = model.doInference(frameResize)
parsed_results = model.parseResult(model_type, output)['ssd']
# Parse and document vectors
for obj in parsed_results:
if obj['prob'] > 0.5:
vector = Vector((
obj['xmin'],
obj['xmax'],
obj['ymin'],
obj['ymax'],
), inferred_obj.xscale, inferred_obj.yscale)
inferred_obj.add_vector(vector, (
labels[obj['label']],
obj['prob'],
))
cv2.rectangle(frame, (vector.xmin, vector.ymin),
(vector.xmax, vector.ymax), (255, 165, 20),
4)
cv2.putText(
frame, '{}: {:.2f}'.format(labels[obj['label']],
obj['prob']),
(vector.xmin, vector.ymin - 15),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 165, 20), 4)
# Trigger event pipeline
client.publish(topic=iotTopic, payload=str(inferred_obj))
# Update output
global jpeg
ret, jpeg = cv2.imencode('.jpg', frame)
except Exception as e:
client.publish(topic=iotTopic, payload=str(e))
def greengrass_infinite_infer_run():
""" Entry point of the lambda function"""
client.publish(topic=iot_topic, payload='Start of run loop...')
try:
# This object detection model is implemented as single shot detector (ssd), since
# the number of labels is small we create a dictionary that will help us convert
# the machine labels to human readable labels.
model_type = 'ssd'
output_map = {}
with open('classes.txt') as f:
for line in f:
(key, val) = line.split()
output_map[int(key)] = val
client.publish(topic=iot_topic, payload='Classes to be detected: ' + str(output_map))
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay('480p')
local_display.start()
# The height and width of the training set images
input_height = 512
input_width = 512
# Load the model onto the GPU.
# optimize the model
client.publish(topic=iot_topic, payload='Optimizing model...')
ret, model_path = mo.optimize('deploy_ssd_resnet50_512', input_width, input_height)
# load the model
client.publish(topic=iot_topic, payload='Loading model...')
model = awscam.Model(model_path, {'GPU': 1})
client.publish(topic=iot_topic, payload='Custom object detection model loaded')
# Set the threshold for detection
detection_threshold = 0.40
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception('Failed to get frame from the stream')
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API. Note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
client.publish(topic=iot_topic, payload='Calling inference on next frame...')
parsed_inference_results = model.parseResult(model_type,
model.doInference(frame_resize))
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0]) / float(input_height)
xscale = float(frame.shape[1]) / float(input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Get the detected objects and probabilities
default_color = (255, 165, 20)
i = 0
for obj in parsed_inference_results[model_type]:
if obj['prob'] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj['xmin'])
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax'])
ymax = int(yscale * obj['ymax'])
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), default_color, 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color,
# and thickness
cv2.putText(frame, "{}: {:.2f}%".format(output_map[obj['label']],
obj['prob'] * 100),
(xmin, ymin-text_offset),
cv2.FONT_HERSHEY_SIMPLEX, 2.5, (255, 165, 20), 6)
# Store label and probability to send to cloud
cloud_output[output_map[obj['label']]] = obj['prob']
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send results to the cloud
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(topic=iot_topic, payload='Error in object detection lambda: {}'.format(ex))
iotTopic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME'])
def cropFace(img, x, y, w, h):
#Crop face
cimg = img[y:y + h, x:x + w]
#Convert to jpeg
ret, jpeg = cv2.imencode('.jpg', cimg)
face = base64.b64encode(jpeg.tobytes())
return face
ret, frame = awscam.getLastFrame()
ret, jpeg = cv2.imencode('.jpg', frame)
Write_To_FIFO = True
class FIFO_Thread(Thread):
def __init__(self):
''' Constructor. '''
Thread.__init__(self)
def run(self):
fifo_path = "/tmp/results.mjpeg"
if not os.path.exists(fifo_path):
os.mkfifo(fifo_path)
f = open(fifo_path, 'w')
client.publish(topic=iotTopic, payload="Opened Pipe")
def greengrass_infinite_infer_run():
try:
modelPath = "/opt/awscam/artifacts/mxnet_deploy_ssd_FP16_FUSED.xml"
modelType = "ssd"
input_width = 300
input_height = 300
prob_thresh = 0.25
results_thread = FIFO_Thread()
results_thread.start()
# Send a starting message to IoT console
client.publish(topic=iotTopic, payload="Face detection starts now")
# Load model to GPU (use {"GPU": 0} for CPU)
mcfg = {"GPU": 1}
model = awscam.Model(modelPath, mcfg)
client.publish(topic=iotTopic, payload="Model loaded")
ret, frame = awscam.getLastFrame()
if ret == False:
raise Exception("Failed to get frame from the stream")
yscale = float(frame.shape[0] / input_height)
xscale = float(frame.shape[1] / input_width)
doInfer = True
while doInfer:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
# Raise an exception if failing to get a frame
if ret == False:
raise Exception("Failed to get frame from the stream")
# Resize frame to fit model input requirement
frameResize = cv2.resize(frame, (input_width, input_height))
# Run model inference on the resized frame
inferOutput = model.doInference(frameResize)
# Output inference result to the fifo file so it can be viewed with mplayer
parsed_results = model.parseResult(modelType, inferOutput)['ssd']
label = '{'
for obj in parsed_results:
if obj['prob'] < prob_thresh:
break
xmin = int(xscale * obj['xmin']) + int(
(obj['xmin'] - input_width / 2) + input_width / 2)
ymin = int(yscale * obj['ymin'])
xmax = int(xscale * obj['xmax']) + int(
(obj['xmax'] - input_width / 2) + input_width / 2)
ymax = int(yscale * obj['ymax'])
#Crop face
################
client.publish(topic=iotTopic, payload="cropping face")
try:
cimage = cropFace(frame, xmin, ymin, xmax - xmin,
ymax - ymin)
lblconfidence = '"confidence" : ' + str(obj['prob'])
client.publish(topic=iotTopic,
payload='{ "face" : "' + cimage + '", ' +
lblconfidence + '}')
except Exception as e:
msg = "Crop image failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
client.publish(topic=iotTopic, payload="Crop face complete")
################
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
(255, 165, 20), 4)
label += '"{}": {:.2f},'.format(str(obj['label']), obj['prob'])
label_show = '{}: {:.2f}'.format(str(obj['label']),
obj['prob'])
cv2.putText(frame, label_show, (xmin, ymin - 15),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 165, 20), 4)
label += '"null": 0.0'
label += '}'
#client.publish(topic=iotTopic, payload = label)
global jpeg
ret, jpeg = cv2.imencode('.jpg', frame)
except Exception as e:
msg = "Test failed: " + str(e)
client.publish(topic=iotTopic, payload=msg)
# Asynchronously schedule this function to be run again in 15 seconds
Timer(15, greengrass_infinite_infer_run).start()
def infinite_infer_run():
""" Entry point of the lambda function"""
try:
# This face detection model is implemented as single shot detector (ssd).
model_type = "ssd"
output_map = {1: "face"}
# Create an IoT client for sending to messages to the cloud.
client = greengrasssdk.client("iot-data")
iot_topic = "$aws/things/{}/infer".format(os.environ["AWS_IOT_THING_NAME"])
# Create a local display instance that will dump the image bytes to a FIFO
# file that the image can be rendered locally.
local_display = LocalDisplay("480p")
local_display.start()
# The sample projects come with optimized artifacts, hence only the artifact
# path is required.
model_path = "/opt/awscam/artifacts/mxnet_deploy_ssd_FP16_FUSED.xml"
# Load the model onto the GPU.
client.publish(topic=iot_topic, payload="Loading face detection model")
model = awscam.Model(model_path, {"GPU": 1})
client.publish(topic=iot_topic, payload="Face detection model loaded")
# Set the threshold for detection
detection_threshold = 0.25
# The height and width of the training set images
input_height = 300
input_width = 300
# Do inference until the lambda is killed.
while True:
# Get a frame from the video stream
ret, frame = awscam.getLastFrame()
if not ret:
raise Exception("Failed to get frame from the stream")
# Resize frame to the same size as the training set.
frame_resize = cv2.resize(frame, (input_height, input_width))
# Run the images through the inference engine and parse the results using
# the parser API, note it is possible to get the output of doInference
# and do the parsing manually, but since it is a ssd model,
# a simple API is provided.
parsed_inference_results = model.parseResult(
model_type, model.doInference(frame_resize)
)
# Compute the scale in order to draw bounding boxes on the full resolution
# image.
yscale = float(frame.shape[0]) / float(input_height)
xscale = float(frame.shape[1]) / float(input_width)
# Dictionary to be filled with labels and probabilities for MQTT
cloud_output = {}
# Get the detected faces and probabilities
for obj in parsed_inference_results[model_type]:
if obj["prob"] > detection_threshold:
# Add bounding boxes to full resolution frame
xmin = int(xscale * obj["xmin"])
ymin = int(yscale * obj["ymin"])
xmax = int(xscale * obj["xmax"])
ymax = int(yscale * obj["ymax"])
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.rectangle method.
# Method signature: image, point1, point2, color, and tickness.
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 165, 20), 10)
# Amount to offset the label/probability text above the bounding box.
text_offset = 15
# See https://docs.opencv.org/3.4.1/d6/d6e/group__imgproc__draw.html
# for more information about the cv2.putText method.
# Method signature: image, text, origin, font face, font scale, color,
# and tickness
cv2.putText(
frame,
"{:.2f}%".format(obj["prob"] * 100),
(xmin, ymin - text_offset),
cv2.FONT_HERSHEY_SIMPLEX,
2.5,
(255, 165, 20),
6,
)
# Store label and probability to send to cloud
cloud_output[output_map[obj["label"]]] = obj["prob"]
# Set the next frame in the local display stream.
local_display.set_frame_data(frame)
# Send results to the cloud
client.publish(topic=iot_topic, payload=json.dumps(cloud_output))
except Exception as ex:
client.publish(
topic=iot_topic, payload="Error in face detection lambda: {}".format(ex)
)
