def log_bounding_boxes(carla_image, depth_msg, segmented_frame, traffic_lights,
tl_color, speed_signs, stop_signs, weather, town):
game_time = int(carla_image.timestamp * 1000)
print("Processing game time {} in {} with weather {}".format(
game_time, town, weather))
transform = pylot.utils.Transform.from_carla_transform(
carla_image.transform)
camera_setup = RGBCameraSetup("rgb_camera", FLAGS.frame_width,
FLAGS.camera_height, transform,
FLAGS.camera_fov)
frame = CameraFrame.from_carla_frame(carla_image, camera_setup)
_, world = get_world()
town_name = world.get_map().name
speed_limit_det_obstacles = []
if speed_signs:
speed_limit_det_obstacles = pylot.simulation.utils.get_detected_speed_limits(
speed_signs, depth_msg.frame, segmented_frame)
traffic_stop_det_obstacles = []
if stop_signs:
traffic_stop_det_obstacles = pylot.simulation.utils.get_detected_traffic_stops(
stop_signs, depth_msg.frame)
traffic_light_det_obstacles = []
if traffic_lights:
traffic_light_det_obstacles = get_traffic_light_obstacles(
traffic_lights, depth_msg.frame, segmented_frame, tl_color,
town_name)
det_obstacles = (speed_limit_det_obstacles + traffic_stop_det_obstacles +
traffic_light_det_obstacles)
# Log the frame.
file_name = '{}signs-{}_{}_{}.png'.format(FLAGS.data_path, game_time,
weather, town)
rgb_img = Image.fromarray(frame.as_rgb_numpy_array())
rgb_img.save(file_name)
if FLAGS.log_bbox_images:
frame.annotate_with_bounding_boxes(game_time, det_obstacles)
file_name = '{}annotated-signs-{}_{}_{}.png'.format(
FLAGS.data_path, game_time, weather, town)
rgb_img = Image.fromarray(frame.as_rgb_numpy_array())
rgb_img.save(file_name)
# Log the bounding boxes.
bboxes = [obstacle.get_bbox_label() for obstacle in det_obstacles]
file_name = '{}bboxes-{}_{}_{}.json'.format(FLAGS.data_path, game_time,
weather, town)
with open(file_name, 'w') as outfile:
json.dump(bboxes, outfile)
if FLAGS.visualize_bboxes:
frame.annotate_with_bounding_boxes(game_time, det_obstacles)
frame.visualize('bboxes')
def on_camera_msg(carla_image):
game_time = int(carla_image.timestamp * 1000)
print("Received camera msg {}".format(game_time))
camera_transform = pylot.utils.Transform.from_carla_transform(
carla_image.transform)
camera_setup = CameraSetup("rgb_camera",
"sensor.camera.rgb",
800,
600,
camera_transform,
fov=90.0)
global last_frame
last_frame = CameraFrame.from_carla_frame(carla_image, camera_setup)
def process_images(self, carla_image):
""" Invoked when an image is received from the simulator.
Args:
carla_image: a carla.Image.
"""
game_time = int(carla_image.timestamp * 1000)
timestamp = erdos.Timestamp(coordinates=[game_time])
watermark_msg = erdos.WatermarkMessage(timestamp)
with erdos.profile(self.config.name + '.process_images',
self,
event_data={'timestamp': str(timestamp)}):
# Ensure that the code executes serially
with self._lock:
msg = None
if self._camera_setup.camera_type == 'sensor.camera.rgb':
msg = FrameMessage(
timestamp,
CameraFrame.from_carla_frame(carla_image,
self._camera_setup))
elif self._camera_setup.camera_type == 'sensor.camera.depth':
# Include the transform relative to the vehicle.
# Carla carla_image.transform returns the world transform,
# but we do not use it directly.
msg = DepthFrameMessage(
timestamp,
DepthFrame.from_carla_frame(
carla_image,
self._camera_setup,
save_original_frame=self._flags.
visualize_depth_camera))
elif (self._camera_setup.camera_type ==
'sensor.camera.semantic_segmentation'):
msg = SegmentedFrameMessage(
timestamp,
SegmentedFrame.from_carla_image(
carla_image, self._camera_setup))
if self._release_data:
self._camera_stream.send(msg)
self._camera_stream.send(watermark_msg)
else:
# Pickle the data, and release it upon release msg receipt.
pickled_msg = pickle.dumps(
msg, protocol=pickle.HIGHEST_PROTOCOL)
with self._pickle_lock:
self._pickled_messages[msg.timestamp] = pickled_msg
self._notify_reading_stream.send(watermark_msg)
def process_images(self, carla_image):
""" Invoked when an image is received from the simulator.
Args:
carla_image: a carla.Image.
"""
# Ensure that the code executes serially
with self._lock:
game_time = int(carla_image.timestamp * 1000)
timestamp = erdos.Timestamp(coordinates=[game_time])
watermark_msg = erdos.WatermarkMessage(timestamp)
msg = None
if self._camera_setup.camera_type == 'sensor.camera.rgb':
msg = FrameMessage(
timestamp,
CameraFrame.from_carla_frame(carla_image,
self._camera_setup))
elif self._camera_setup.camera_type == 'sensor.camera.depth':
# Include the transform relative to the vehicle.
# Carla carla_image.transform returns the world transform, but
# we do not use it directly.
msg = DepthFrameMessage(
timestamp,
DepthFrame.from_carla_frame(carla_image,
self._camera_setup))
elif self._camera_setup.camera_type == \
'sensor.camera.semantic_segmentation':
msg = SegmentedFrameMessage(
timestamp,
SegmentedFrame.from_carla_image(carla_image,
self._camera_setup))
# Send the message containing the frame.
self._camera_stream.send(msg)
# Note: The operator is set not to automatically propagate
# watermark messages received on input streams. Thus, we can
# issue watermarks only after the Carla callback is invoked.
self._camera_stream.send(watermark_msg)