def on_watermark(self, timestamp: Timestamp,
traffic_lights_stream: WriteStream):
"""Invoked when all input streams have received a watermark.
Args:
timestamp (:py:class:`erdos.timestamp.Timestamp`): The timestamp of
the watermark.
"""
self._logger.debug('@{}: received watermark'.format(timestamp))
if timestamp.is_top:
return
traffic_light_msg = self._traffic_lights.popleft()
bgr_msg = self._bgr_msgs.popleft()
depth_msg = self._depth_frame_msgs.popleft()
segmented_msg = self._segmented_msgs.popleft()
pose_msg = self._pose_msgs.popleft()
vehicle_transform = pose_msg.data.transform
self._frame_cnt += 1
if (hasattr(self._flags, 'log_every_nth_message')
and self._frame_cnt % self._flags.log_every_nth_message != 0):
# There's no point to run the perfect detector if collecting
# data, and only logging every nth frame.
traffic_lights_stream.send(TrafficLightsMessage(timestamp, []))
return
# The camera setup sent with the image is relative to the car, we need
# to transform it relative to the world to detect traffic lights.
depth_msg.frame.camera_setup.set_transform(
vehicle_transform * depth_msg.frame.camera_setup.transform)
det_traffic_lights = get_traffic_lights_obstacles(
traffic_light_msg.obstacles, depth_msg.frame, segmented_msg.frame,
self._town_name)
# Filter out traffic lights that are far away.
det_traffic_lights = [
tl for tl in det_traffic_lights
if tl.transform.location.distance(vehicle_transform.location) <=
self._flags.static_obstacle_distance_threshold
]
# Send the detected traffic lights.
traffic_lights_stream.send(
TrafficLightsMessage(timestamp, det_traffic_lights))
if self._flags.log_detector_output:
bgr_msg.frame.annotate_with_bounding_boxes(bgr_msg.timestamp,
det_traffic_lights,
vehicle_transform)
bgr_msg.frame.save(bgr_msg.timestamp.coordinates[0],
self._flags.data_path, 'perfect-detector')
def __send_ground_actors_data(self, timestamp: Timestamp):
# Get all the actors in the simulation.
actor_list = self._world.get_actors()
(vehicles, people, traffic_lights, speed_limits, traffic_stops
) = pylot.simulation.utils.extract_data_in_pylot_format(actor_list)
# Send ground people and vehicles.
self.ground_obstacles_stream.send(
ObstaclesMessage(timestamp, vehicles + people))
self.ground_obstacles_stream.send(erdos.WatermarkMessage(timestamp))
# Send ground traffic lights.
self.ground_traffic_lights_stream.send(
TrafficLightsMessage(timestamp, traffic_lights))
self.ground_traffic_lights_stream.send(
erdos.WatermarkMessage(timestamp))
# Send ground speed signs.
self.ground_speed_limit_signs_stream.send(
SpeedSignsMessage(timestamp, speed_limits))
self.ground_speed_limit_signs_stream.send(
erdos.WatermarkMessage(timestamp))
# Send stop signs.
self.ground_stop_signs_stream.send(
StopSignsMessage(timestamp, traffic_stops))
self.ground_stop_signs_stream.send(erdos.WatermarkMessage(timestamp))
def on_frame(self, msg, traffic_lights_stream):
"""Invoked whenever a frame message is received on the stream.
Args:
msg: A :py:class:`~pylot.perception.messages.FrameMessage`.
obstacles_stream (:py:class:`erdos.WriteStream`): Stream on which
the operator sends
:py:class:`~pylot.perception.messages.TrafficLightsMessage`
messages for traffic lights.
"""
self._logger.debug('@{}: {} received message'.format(
msg.timestamp, self.config.name))
assert msg.frame.encoding == 'BGR', 'Expects BGR frames'
boxes, scores, labels = self.__run_model(
msg.frame.as_rgb_numpy_array())
traffic_lights = self.__convert_to_detected_tl(
boxes, scores, labels, msg.frame.camera_setup.height,
msg.frame.camera_setup.width)
self._logger.debug('@{}: {} detected traffic lights {}'.format(
msg.timestamp, self.config.name, traffic_lights))
traffic_lights_stream.send(
TrafficLightsMessage(msg.timestamp, traffic_lights))
traffic_lights_stream.send(erdos.WatermarkMessage(msg.timestamp))
if self._flags.log_traffic_light_detector_output:
msg.frame.annotate_with_bounding_boxes(msg.timestamp,
traffic_lights)
msg.frame.save(msg.timestamp.coordinates[0], self._flags.data_path,
'tl-detector-{}'.format(self.config.name))
def main(argv):
(obstacles_stream, traffic_lights_stream, obstacles_tracking_stream,
open_drive_stream, global_trajectory_stream) = create_data_flow()
# Run the data-flow.
erdos.run_async()
top_timestamp = erdos.Timestamp(coordinates=[sys.maxsize])
open_drive_stream.send(erdos.WatermarkMessage(top_timestamp))
waypoints = [[waypoint] for waypoint in read_waypoints()]
global_trajectory_stream.send(
erdos.Message(erdos.Timestamp(coordinates=[0]), waypoints))
global_trajectory_stream.send(erdos.WatermarkMessage(top_timestamp))
time_to_sleep = 1.0 / FLAGS.sensor_frequency
count = 0
while True:
timestamp = erdos.Timestamp(coordinates=[count])
if not FLAGS.obstacle_detection:
obstacles_stream.send(ObstaclesMessage(timestamp, []))
obstacles_stream.send(erdos.WatermarkMessage(timestamp))
if not FLAGS.traffic_light_detection:
traffic_lights_stream.send(TrafficLightsMessage(timestamp, []))
traffic_lights_stream.send(erdos.WatermarkMessage(timestamp))
if not FLAGS.obstacle_tracking:
obstacles_tracking_stream.send(
ObstacleTrajectoriesMessage(timestamp, []))
obstacles_tracking_stream.send(erdos.WatermarkMessage(timestamp))
count += 1
# NOTE: We should offset sleep time by the time it takes to send the
# messages.
time.sleep(time_to_sleep)
def on_watermark(self, timestamp, traffic_lights_stream):
"""Invoked when all input streams have received a watermark.
Args:
timestamp (:py:class:`erdos.timestamp.Timestamp`): The timestamp of
the watermark.
"""
self._logger.debug('@{}: received watermark'.format(timestamp))
traffic_light_msg = self._traffic_lights.popleft()
bgr_msg = self._bgr_msgs.popleft()
depth_msg = self._depth_frame_msgs.popleft()
segmented_msg = self._segmented_msgs.popleft()
pose_msg = self._pose_msgs.popleft()
vehicle_transform = pose_msg.data.transform
self._frame_cnt += 1
if (hasattr(self._flags, 'log_every_nth_message')
and self._frame_cnt % self._flags.log_every_nth_message != 0):
# There's no point to run the perfect detector if collecting
# data, and only logging every nth frame.
traffic_lights_stream.send(TrafficLightsMessage(timestamp, []))
return
# The camera setup sent with the image is relative to the car, we need
# to transform it relative to the world to detect traffic lights.
depth_msg.frame.camera_setup.set_transform(
vehicle_transform * depth_msg.frame.camera_setup.transform)
det_traffic_lights = get_traffic_lights_obstacles(
traffic_light_msg.obstacles, depth_msg.frame, segmented_msg.frame,
self._town_name)
if (self._flags.visualize_detected_traffic_lights
or self._flags.log_detector_output):
bgr_msg.frame.annotate_with_bounding_boxes(bgr_msg.timestamp,
det_traffic_lights,
vehicle_transform)
if self._flags.visualize_detected_traffic_lights:
bgr_msg.frame.visualize(
self.config.name,
pygame_display=pylot.utils.PYGAME_DISPLAY)
if self._flags.log_detector_output:
bgr_msg.frame.save(bgr_msg.timestamp.coordinates[0],
self._flags.data_path, 'perfect-detector')
# Send the detected traffic lights.
traffic_lights_stream.send(
TrafficLightsMessage(timestamp, det_traffic_lights))
def send_perfect_detections(self, perfect_obstacles_stream,
perfect_traffic_lights_stream, timestamp,
tl_camera_location):
"""Send perfect detections for agents and traffic lights.
This method first connects to the simulator to extract all the
agents and traffic light in a scenario. Next, it transforms them into
the types Pylot expects, and sends them on the streams for perfect
detections.
Note: This is only used when executing using a perfect perception
component.
"""
if not (FLAGS.simulator_obstacle_detection
or FLAGS.simulator_traffic_light_detection
or FLAGS.evaluate_obstacle_detection
or FLAGS.evaluate_obstacle_tracking):
return
from pylot.simulation.utils import extract_data_in_pylot_format
actor_list = self._world.get_actors()
(vehicles, people, traffic_lights, _,
_) = extract_data_in_pylot_format(actor_list)
if (FLAGS.simulator_obstacle_detection
or FLAGS.evaluate_obstacle_detection
or FLAGS.evaluate_obstacle_tracking):
perfect_obstacles_stream.send(
ObstaclesMessage(timestamp, vehicles + people))
perfect_obstacles_stream.send(erdos.WatermarkMessage(timestamp))
if FLAGS.simulator_traffic_light_detection:
vec_transform = pylot.utils.Transform.from_simulator_transform(
self._ego_vehicle.get_transform())
tl_camera_transform = pylot.utils.Transform(
tl_camera_location, pylot.utils.Rotation())
visible_tls = []
if self._town_name is None:
self._town_name = self._world.get_map().name
for tl in traffic_lights:
if tl.is_traffic_light_visible(
vec_transform * tl_camera_transform,
self._town_name,
distance_threshold=FLAGS.
static_obstacle_distance_threshold):
if self._town_name not in ['Town01', 'Town02']:
delta_y = -5
if self._town_name == 'Town04':
delta_y = -2
# Move the traffic light location to the road.
tl.transform = tl.transform * pylot.utils.Transform(
pylot.utils.Location(delta_y, 0, 5),
pylot.utils.Rotation())
visible_tls.append(tl)
perfect_traffic_lights_stream.send(
TrafficLightsMessage(timestamp, visible_tls))
perfect_traffic_lights_stream.send(
erdos.WatermarkMessage(timestamp))
def on_frame(self, msg, traffic_lights_stream):
"""Invoked whenever a frame message is received on the stream.
Args:
msg: A :py:class:`~pylot.perception.messages.FrameMessage`.
obstacles_stream (:py:class:`erdos.WriteStream`): Stream on which
the operator sends
:py:class:`~pylot.perception.messages.TrafficLightsMessage`
messages for traffic lights.
"""
self._logger.debug('@{}: {} received message'.format(
msg.timestamp, self.config.name))
assert msg.frame.encoding == 'BGR', 'Expects BGR frames'
# Expand dimensions since the model expects images to have
# shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(msg.frame.as_rgb_numpy_array(),
axis=0)
(boxes, scores, classes, num) = self._tf_session.run(
[
self._detection_boxes, self._detection_scores,
self._detection_classes, self._num_detections
],
feed_dict={self._image_tensor: image_np_expanded})
num_detections = int(num[0])
labels = [self._labels[label] for label in classes[0][:num_detections]]
boxes = boxes[0][:num_detections]
scores = scores[0][:num_detections]
traffic_lights = self.__convert_to_detected_tl(
boxes, scores, labels, msg.frame.camera_setup.height,
msg.frame.camera_setup.width)
self._logger.debug('@{}: {} detected traffic lights {}'.format(
msg.timestamp, self.config.name, traffic_lights))
if (self._flags.visualize_detected_traffic_lights
or self._flags.log_traffic_light_detector_output):
msg.frame.annotate_with_bounding_boxes(msg.timestamp,
traffic_lights)
if self._flags.visualize_detected_traffic_lights:
msg.frame.visualize(self.config.name,
pygame_display=pylot.utils.PYGAME_DISPLAY)
if self._flags.log_traffic_light_detector_output:
msg.frame.save(msg.timestamp.coordinates[0],
self._flags.data_path,
'tl-detector-{}'.format(self.config.name))
traffic_lights_stream.send(
TrafficLightsMessage(msg.timestamp, traffic_lights))
def __publish_ground_actors_data(self, timestamp, watermark_msg):
# Get all the actors in the simulation.
actor_list = self._world.get_actors()
(vehicles, people, traffic_lights, speed_limits,
traffic_stops) = extract_data_in_pylot_format(actor_list)
obstacles_msg = ObstaclesMessage(timestamp, vehicles + people)
self._ground_obstacles_stream.send(obstacles_msg)
self._ground_obstacles_stream.send(watermark_msg)
traffic_lights_msg = TrafficLightsMessage(timestamp, traffic_lights)
self._ground_traffic_lights_stream.send(traffic_lights_msg)
self._ground_traffic_lights_stream.send(watermark_msg)
speed_limit_signs_msg = SpeedSignsMessage(timestamp, speed_limits)
self._ground_speed_limit_signs_stream.send(speed_limit_signs_msg)
self._ground_speed_limit_signs_stream.send(watermark_msg)
stop_signs_msg = StopSignsMessage(timestamp, traffic_stops)
self._ground_stop_signs_stream.send(stop_signs_msg)
self._ground_stop_signs_stream.send(watermark_msg)
def send_ground_data(self, timestamp):
if not (FLAGS.carla_obstacle_detection
or FLAGS.carla_traffic_light_detection):
return
actor_list = self._world.get_actors()
(vehicles, people, traffic_lights, _,
_) = pylot.simulation.utils.extract_data_in_pylot_format(actor_list)
if FLAGS.carla_obstacle_detection:
self._ground_obstacles_stream.send(
ObstaclesMessage(timestamp, vehicles + people))
self._ground_obstacles_stream.send(
erdos.WatermarkMessage(timestamp))
if FLAGS.carla_traffic_light_detection:
vec_transform = pylot.utils.Transform.from_carla_transform(
self._ego_vehicle.get_transform())
tl_camera_transform = pylot.utils.Transform(
CENTER_CAMERA_LOCATION, pylot.utils.Rotation())
visible_tls = []
if self._town_name is None:
self._town_name = self._world.get_map().name
for tl in traffic_lights:
if tl.is_traffic_light_visible(
vec_transform * tl_camera_transform,
self._town_name,
distance_threshold=FLAGS.
static_obstacle_distance_threshold):
if self._town_name not in ['Town01', 'Town02']:
delta_y = -5
if self._town_name == 'Town04':
delta_y = -2
# Move the traffic light location to the road.
tl.transform = tl.transform * pylot.utils.Transform(
pylot.utils.Location(delta_y, 0, 5),
pylot.utils.Rotation())
visible_tls.append(tl)
self._ground_traffic_lights_stream.send(
TrafficLightsMessage(timestamp, visible_tls))
self._ground_traffic_lights_stream.send(
erdos.WatermarkMessage(timestamp))
def main(argv):
(obstacles_stream, traffic_lights_stream, obstacles_tracking_stream,
open_drive_stream, global_trajectory_stream, control_display_stream,
streams_to_send_top_on) = create_data_flow()
# Run the data-flow.
node_handle = erdos.run_async()
signal.signal(signal.SIGINT, shutdown)
# Send waypoints.
waypoints = Waypoints.read_from_csv_file(FLAGS.waypoints_csv_file,
FLAGS.target_speed)
global_trajectory_stream.send(
WaypointsMessage(
erdos.Timestamp(coordinates=[0]), waypoints,
pylot.planning.utils.BehaviorPlannerState.FOLLOW_WAYPOINTS))
# Send top watermark on all streams that require it.
top_msg = erdos.WatermarkMessage(erdos.Timestamp(is_top=True))
open_drive_stream.send(top_msg)
global_trajectory_stream.send(top_msg)
for stream in streams_to_send_top_on:
stream.send(top_msg)
time_to_sleep = 1.0 / FLAGS.sensor_frequency
count = 0
try:
while True:
timestamp = erdos.Timestamp(coordinates=[count])
if not FLAGS.obstacle_detection:
obstacles_stream.send(ObstaclesMessage(timestamp, []))
obstacles_stream.send(erdos.WatermarkMessage(timestamp))
if not FLAGS.traffic_light_detection:
traffic_lights_stream.send(TrafficLightsMessage(timestamp, []))
traffic_lights_stream.send(erdos.WatermarkMessage(timestamp))
if not FLAGS.obstacle_tracking:
obstacles_tracking_stream.send(
ObstacleTrajectoriesMessage(timestamp, []))
obstacles_tracking_stream.send(
erdos.WatermarkMessage(timestamp))
count += 1
if pylot.flags.must_visualize():
import pygame
from pygame.locals import K_n
events = pygame.event.get()
for event in events:
if event.type == pygame.KEYUP:
if event.key == K_n:
control_display_stream.send(
erdos.Message(erdos.Timestamp(coordinates=[0]),
event.key))
elif event.type == pygame.QUIT:
raise KeyboardInterrupt
elif event.type == pygame.KEYDOWN:
if (event.key == pygame.K_c
and pygame.key.get_mods() & pygame.KMOD_CTRL):
raise KeyboardInterrupt
# NOTE: We should offset sleep time by the time it takes to send
# the messages.
time.sleep(time_to_sleep)
except KeyboardInterrupt:
node_handle.shutdown()
if pylot.flags.must_visualize():
import pygame
pygame.quit()