class TensorCamera:
"""Wrapping in a class to capture the tensor and camera state in order to avoid
some verbose functions or structures"""
def __init__(self, width=640, height=480):
frame_rate_calc = 1
freq = cv2.getTickFrequency()
font = cv2.FONT_HERSHEY_SIMPLEX
self.IM_WIDTH = width
self.IMG_HEIGHT = height
self.sensor = Sensors()
# load in the graph and muddle with the default graph
# this setup allows the same session to be used throughout
# tf_session is *not* thread-safe
self.graph = tf.Graph()
with self.graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(PATH_TO_FROZEN_GRAPH, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
self.tf_session = tf.Session(graph=self.graph)
self.camera = PiCamera(resolution=(width, height), framerate=30)
# Capture and set all to zero to get a blank array of the right size
self._frame = PiRGBArray(self.camera, size=(width, height))
self._frame.truncate(0)
def capture_and_tell(self):
"""Uses the tensorflow graph to capture an image, do magic, ask the sensor
board for the distance from an object, performs some business logic, and then
send an update to the sensor board with brake status
returns the system status for brake, distance, and image. This is to
be used by a server component"""
# clear the buffer frame used for capture
self._frame.truncate(0)
self.camera.capture(self._frame, format="rgb", use_video_port=True)
frame = np.copy(self._frame.array)
frame.setflags(write=1)
graph_ops = self.graph.get_operations()
with self.graph.as_default():
output_dict = run_inference_for_single_image(
frame, graph_ops, self.tf_session)
# Draw labeled bounding boxes for any detected objects whose score is greater than 0.3
vis_util.visualize_boxes_and_labels_on_image_array(
frame,
output_dict['detection_boxes'],
output_dict['detection_classes'],
output_dict['detection_scores'],
category_index,
instance_masks=output_dict.get('detection_masks'),
use_normalized_coordinates=True,
line_thickness=6,
min_score_thresh=0.3)
# Debug
# from matplotlib import pyplot
# pyplot.imshow(frame)
# pyplot.savefig("test.png")
# putting this in the main loop and requiring it for every frame
# could faster, but this is low hanging fruit. Does not respond
# well to network hiccups
distance = self.sensor.find_distance()
# Update the brake based on the detection and distance logic
# send_brake does not send a signal to the light if not needed (i.e. no state change)
# NOTE: Here we've used a static threshold for both object distance and the detection score for when to signal
# the brake because those have seemed to work well in our testing. Obviously we don't have a real vehicle
# to test this on, but if we did, then we could easily adjust this behavior accordingly. For example,
# we could make the score threshold based on the object distance, or store the last distance measurement
# or two (using variables on our TensorCamera class) to determine whether and how fast an object was
# approaching.
if distance < 100 and any(output_dict['detection_scores']) > 0.3:
brake = self.sensor.send_brake(True)
else:
brake = self.sensor.send_brake(False)
self._frame.truncate(0)
# Invert the frame to RGB and then encode it into
# a png that can be transmitted as raw bytes to be displayed on the head unit
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
(_, buf) = cv2.imencode('.png', rgb_frame)
# return the dashboard payload
return {
'image': buf.tobytes(),
'distance': str(distance) + " cm",
'brake': brake
}
