def log_bounding_boxes(carla_image, depth_frame, segmented_frame,
traffic_lights, tl_color, speed_signs, stop_signs,
weather, town):
game_time = int(carla_image.timestamp * 1000)
print("Processing game time {} in {} with weather {}".format(
game_time, town, weather))
frame = bgra_to_bgr(to_bgra_array(carla_image))
# Copy the frame to ensure its on the heap.
frame = copy.deepcopy(frame)
transform = to_pylot_transform(carla_image.transform)
_, world = get_world()
town_name = world.get_map().name
speed_limit_det_objs = []
if speed_signs:
speed_limit_det_objs = pylot.simulation.utils.get_speed_limit_det_objs(
speed_signs, transform, transform, depth_frame, FLAGS.frame_width,
FLAGS.frame_height, FLAGS.camera_fov, segmented_frame)
traffic_stop_det_objs = []
if stop_signs:
traffic_stop_det_objs = pylot.simulation.utils.get_traffic_stop_det_objs(
stop_signs, transform, depth_frame, FLAGS.frame_width,
FLAGS.frame_height, FLAGS.camera_fov)
traffic_light_det_objs = []
if traffic_lights:
traffic_light_det_objs = get_traffic_light_objs(
traffic_lights, transform, depth_frame, segmented_frame,
FLAGS.frame_width, FLAGS.frame_height, tl_color, town_name)
det_objs = (speed_limit_det_objs + traffic_stop_det_objs +
traffic_light_det_objs)
if FLAGS.visualize_bboxes:
visualize_ground_bboxes('bboxes', game_time, frame, det_objs)
# Log the frame.
rgb_frame = bgr_to_rgb(frame)
file_name = '{}signs-{}_{}_{}.png'.format(FLAGS.data_path, game_time,
weather, town)
rgb_img = Image.fromarray(np.uint8(rgb_frame))
rgb_img.save(file_name)
if FLAGS.log_bbox_images:
annotate_image_with_bboxes(game_time, frame, det_objs)
rgb_frame = bgr_to_rgb(frame)
file_name = '{}annotated-signs-{}_{}_{}.png'.format(
FLAGS.data_path, game_time, weather, town)
rgb_img = Image.fromarray(np.uint8(rgb_frame))
rgb_img.save(file_name)
# Log the bounding boxes.
bboxes = [det_obj.get_bbox_label() for det_obj in det_objs]
file_name = '{}bboxes-{}_{}_{}.json'.format(FLAGS.data_path, game_time,
weather, town)
with open(file_name, 'w') as outfile:
json.dump(bboxes, outfile)
def on_msg_camera_stream(self, msg):
""" Invoked when the operator receives a message on the data stream."""
self._logger.info('{} received frame {}'.format(
self.name, msg.timestamp))
start_time = time.time()
# The models expect BGR images.
assert msg.encoding == 'BGR', 'Expects BGR frames'
image_np = msg.frame
# Expand dimensions since the model expects images to have
# shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
(boxes, scores, classes, num_detections) = self._tf_session.run(
[
self._detection_boxes, self._detection_scores,
self._detection_classes, self._num_detections
],
feed_dict={self._image_tensor: image_np_expanded})
num_detections = int(num_detections[0])
res_classes = classes[0][:num_detections]
res_boxes = boxes[0][:num_detections]
res_scores = scores[0][:num_detections]
# TODO(ionel): BIG HACK TO FILTER OUT UNKNOWN CLASSES!
boxes = []
scores = []
labels = []
for i in range(0, num_detections):
if res_classes[i] in self._coco_labels:
labels.append(self._coco_labels[res_classes[i]])
boxes.append(res_boxes[i])
scores.append(res_scores[i])
detected_objects = self.__convert_to_detected_objs(
boxes, scores, labels, msg.height, msg.width)
self._logger.info('Detected objects: {}'.format(detected_objects))
if (self._flags.visualize_detector_output
or self._flags.log_detector_output):
annotate_image_with_bboxes(msg.timestamp, image_np,
detected_objects, self._bbox_colors)
if self._flags.visualize_detector_output:
visualize_image(self.name, image_np)
if self._flags.log_detector_output:
save_image(bgr_to_rgb(image_np), msg.timestamp,
self._flags.data_path,
'detector-{}'.format(self.name))
# Get runtime in ms.
runtime = (time.time() - start_time) * 1000
self._csv_logger.info('{},{},"{}",{}'.format(time_epoch_ms(),
self.name, msg.timestamp,
runtime))
output_msg = DetectorMessage(detected_objects, runtime, msg.timestamp)
self.get_output_stream(self._output_stream_name).send(output_msg)
def on_notification(self, msg):
# Pop the oldest message from each buffer.
with self._lock:
if not self.synchronize_msg_buffers(
msg.timestamp,
[self._depth_imgs, self._bgr_imgs, self._segmented_imgs,
self._can_bus_msgs, self._pedestrians, self._vehicles,
self._traffic_lights, self._speed_limit_signs,
self._stop_signs]):
return
depth_msg = self._depth_imgs.popleft()
bgr_msg = self._bgr_imgs.popleft()
segmented_msg = self._segmented_imgs.popleft()
can_bus_msg = self._can_bus_msgs.popleft()
pedestrians_msg = self._pedestrians.popleft()
vehicles_msg = self._vehicles.popleft()
traffic_light_msg = self._traffic_lights.popleft()
speed_limit_signs_msg = self._speed_limit_signs.popleft()
stop_signs_msg = self._stop_signs.popleft()
self._logger.info('Timestamps {} {} {} {} {} {}'.format(
depth_msg.timestamp, bgr_msg.timestamp, segmented_msg.timestamp,
can_bus_msg.timestamp, pedestrians_msg.timestamp,
vehicles_msg.timestamp, traffic_light_msg.timestamp))
# The popper messages should have the same timestamp.
assert (depth_msg.timestamp == bgr_msg.timestamp ==
segmented_msg.timestamp == can_bus_msg.timestamp ==
pedestrians_msg.timestamp == vehicles_msg.timestamp ==
traffic_light_msg.timestamp)
self._frame_cnt += 1
if (hasattr(self._flags, 'log_every_nth_frame') and
self._frame_cnt % self._flags.log_every_nth_frame != 0):
# There's no point to run the perfect detector if collecting
# data, and only logging every nth frame.
output_msg = DetectorMessage([], 0, msg.timestamp)
self.get_output_stream(self._output_stream_name).send(output_msg)
self.get_output_stream(self._output_stream_name)\
.send(WatermarkMessage(msg.timestamp))
return
depth_array = depth_msg.frame
segmented_image = segmented_msg.frame
vehicle_transform = can_bus_msg.data.transform
det_ped = self.__get_pedestrians(pedestrians_msg.pedestrians,
vehicle_transform, depth_array,
segmented_image)
det_vec = self.__get_vehicles(vehicles_msg.vehicles, vehicle_transform,
depth_array, segmented_image)
det_traffic_lights = pylot.simulation.utils.get_traffic_light_det_objs(
traffic_light_msg.traffic_lights,
vehicle_transform * depth_msg.transform,
depth_msg.frame,
depth_msg.width,
depth_msg.height,
self._town_name,
depth_msg.fov)
det_speed_limits = pylot.simulation.utils.get_speed_limit_det_objs(
speed_limit_signs_msg.speed_signs,
vehicle_transform,
vehicle_transform * depth_msg.transform,
depth_msg.frame, depth_msg.width, depth_msg.height,
depth_msg.fov, segmented_msg.frame)
det_stop_signs = pylot.simulation.utils.get_traffic_stop_det_objs(
stop_signs_msg.stop_signs,
vehicle_transform * depth_msg.transform,
depth_msg.frame, depth_msg.width, depth_msg.height, depth_msg.fov)
det_objs = (det_ped + det_vec + det_traffic_lights +
det_speed_limits + det_stop_signs)
# Send the detected obstacles.
output_msg = DetectorMessage(det_objs, 0, msg.timestamp)
self.get_output_stream(self._output_stream_name).send(output_msg)
# Send watermark on the output stream because operators do not
# automatically forward watermarks when they've registed an
# on completion callback.
self.get_output_stream(self._output_stream_name)\
.send(WatermarkMessage(msg.timestamp))
if (self._flags.visualize_ground_obstacles or
self._flags.log_detector_output):
annotate_image_with_bboxes(
bgr_msg.timestamp, bgr_msg.frame, det_objs)
if self._flags.visualize_ground_obstacles:
visualize_image(self.name, bgr_msg.frame)
if self._flags.log_detector_output:
save_image(pylot.utils.bgr_to_rgb(bgr_msg.frame),
bgr_msg.timestamp,
self._flags.data_path,
'perfect-detector')