def __init__(
self,
client: carla.Client,
target_size: PixelDimensions,
render_lanes_on_junctions: bool,
pixels_per_meter: int = 4,
crop_type: BirdViewCropType = BirdViewCropType.FRONT_AND_REAR_AREA,
) -> None:
self.client = client
self.target_size = target_size
self.pixels_per_meter = pixels_per_meter
self._crop_type = crop_type
if crop_type is BirdViewCropType.FRONT_AND_REAR_AREA:
rendering_square_size = round(
square_fitting_rect_at_any_rotation(self.target_size))
elif crop_type is BirdViewCropType.FRONT_AREA_ONLY:
# We must keep rendering size from FRONT_AND_REAR_AREA (in order to avoid rotation issues)
enlarged_size = PixelDimensions(width=target_size.width,
height=target_size.height * 2)
rendering_square_size = round(
square_fitting_rect_at_any_rotation(enlarged_size))
else:
raise NotImplementedError
self.rendering_area = PixelDimensions(width=rendering_square_size,
height=rendering_square_size)
self._world = client.get_world()
self._map = self._world.get_map()
self.masks_generator = MapMaskGenerator(
client,
pixels_per_meter=pixels_per_meter,
render_lanes_on_junctions=render_lanes_on_junctions,
)
cache_path = self.parametrized_cache_path()
with FileLock(f"{cache_path}.lock"):
if Path(cache_path).is_file():
LOGGER.info(f"Loading cache from {cache_path}")
static_cache = np.load(cache_path)
self.full_road_cache = static_cache[0]
self.full_lanes_cache = static_cache[1]
self.full_centerlines_cache = static_cache[2]
LOGGER.info(
f"Loaded static layers from cache file: {cache_path}")
else:
LOGGER.warning(
f"Cache file does not exist, generating cache at {cache_path}"
)
self.full_road_cache = self.masks_generator.road_mask()
self.full_lanes_cache = self.masks_generator.lanes_mask()
self.full_centerlines_cache = self.masks_generator.centerlines_mask(
)
static_cache = np.stack([
self.full_road_cache,
self.full_lanes_cache,
self.full_centerlines_cache,
])
np.save(cache_path, static_cache, allow_pickle=False)
LOGGER.info(f"Saved static layers to cache file: {cache_path}")
class BirdViewProducer:
"""Responsible for producing top-down view on the map, following agent's vehicle.
About BirdView:
- top-down view, fixed directly above the agent (including vehicle rotation), cropped to desired size
- consists of stacked layers (masks), each filled with ones and zeros (depends on MaskMaskGenerator implementation).
Example layers: road, vehicles, pedestrians. 0 indicates -> no presence in that pixel, 1 -> presence
- convertible to RGB image
- Rendering full road and lanes masks is computationally expensive, hence caching mechanism is used
"""
def __init__(
self,
client: carla.Client,
target_size: PixelDimensions,
render_lanes_on_junctions: bool,
pixels_per_meter: int = 4,
crop_type: BirdViewCropType = BirdViewCropType.FRONT_AND_REAR_AREA,
) -> None:
self.client = client
self.target_size = target_size
self.pixels_per_meter = pixels_per_meter
self._crop_type = crop_type
if crop_type is BirdViewCropType.FRONT_AND_REAR_AREA:
rendering_square_size = round(
square_fitting_rect_at_any_rotation(self.target_size))
elif crop_type is BirdViewCropType.FRONT_AREA_ONLY:
# We must keep rendering size from FRONT_AND_REAR_AREA (in order to avoid rotation issues)
enlarged_size = PixelDimensions(width=target_size.width,
height=target_size.height * 2)
rendering_square_size = round(
square_fitting_rect_at_any_rotation(enlarged_size))
else:
raise NotImplementedError
self.rendering_area = PixelDimensions(width=rendering_square_size,
height=rendering_square_size)
self._world = client.get_world()
self._map = self._world.get_map()
self.masks_generator = MapMaskGenerator(
client,
pixels_per_meter=pixels_per_meter,
render_lanes_on_junctions=render_lanes_on_junctions,
)
cache_path = self.parametrized_cache_path()
with FileLock(f"{cache_path}.lock"):
if Path(cache_path).is_file():
LOGGER.info(f"Loading cache from {cache_path}")
static_cache = np.load(cache_path)
self.full_road_cache = static_cache[0]
self.full_lanes_cache = static_cache[1]
self.full_centerlines_cache = static_cache[2]
self.full_buildings_cache = static_cache[3]
LOGGER.info(
f"Loaded static layers from cache file: {cache_path}")
else:
LOGGER.warning(
f"Cache file does not exist, generating cache at {cache_path}"
)
self.full_road_cache = self.masks_generator.road_mask()
self.full_lanes_cache = self.masks_generator.lanes_mask()
self.full_centerlines_cache = self.masks_generator.centerlines_mask(
)
self.full_buildings_cache = self.masks_generator.buildings_mask(
)
static_cache = np.stack([
self.full_road_cache,
self.full_lanes_cache,
self.full_centerlines_cache,
self.full_buildings_cache,
])
np.save(cache_path, static_cache, allow_pickle=False)
LOGGER.info(f"Saved static layers to cache file: {cache_path}")
def parametrized_cache_path(self) -> str:
cache_dir = Path("birdview_v4_cache")
cache_dir.mkdir(parents=True, exist_ok=True)
opendrive_content_hash = cache.generate_opendrive_content_hash(
self._map)
cache_filename = (f"{self._map.name}__"
f"px_per_meter={self.pixels_per_meter}__"
f"opendrive_hash={opendrive_content_hash}__"
f"margin={mask.MAP_BOUNDARY_MARGIN}.npy")
return str(cache_dir / cache_filename)
def produce(self, agent_vehicle: carla.Actor) -> BirdView:
all_actors = actors.query_all(world=self._world)
segregated_actors = actors.segregate_by_type(actors=all_actors)
agent_vehicle_loc = agent_vehicle.get_location()
# Reusing already generated static masks for whole map
self.masks_generator.disable_local_rendering_mode()
agent_global_px_pos = self.masks_generator.location_to_pixel(
agent_vehicle_loc)
cropping_rect = CroppingRect(
x=int(agent_global_px_pos.x - self.rendering_area.width / 2),
y=int(agent_global_px_pos.y - self.rendering_area.height / 2),
width=self.rendering_area.width,
height=self.rendering_area.height,
)
masks = np.zeros(
shape=(
len(BirdViewMasks),
self.rendering_area.height,
self.rendering_area.width,
),
dtype=np.uint8,
)
masks[BirdViewMasks.ROAD.value] = self.full_road_cache[
cropping_rect.vslice, cropping_rect.hslice]
masks[BirdViewMasks.LANES.value] = self.full_lanes_cache[
cropping_rect.vslice, cropping_rect.hslice]
masks[BirdViewMasks.CENTERLINES.value] = self.full_centerlines_cache[
cropping_rect.vslice, cropping_rect.hslice]
masks[BirdViewMasks.BUILDINGS.value] = self.full_buildings_cache[
cropping_rect.vslice, cropping_rect.hslice]
# Dynamic masks
rendering_window = RenderingWindow(origin=agent_vehicle_loc,
area=self.rendering_area)
self.masks_generator.enable_local_rendering_mode(rendering_window)
masks = self._render_actors_masks(agent_vehicle, segregated_actors,
masks)
cropped_masks = self.apply_agent_following_transformation_to_masks(
agent_vehicle, masks)
ordered_indices = [
mask.value for mask in BirdViewMasks.bottom_to_top()
]
return cropped_masks[:, :, ordered_indices]
@staticmethod
def as_rgb(birdview: BirdView) -> RgbCanvas:
h, w, d = birdview.shape
assert d == len(BirdViewMasks)
rgb_canvas = np.zeros(shape=(h, w, 3), dtype=np.uint8)
nonzero_indices = lambda arr: arr == COLOR_ON
for mask_type in BirdViewMasks.bottom_to_top():
rgb_color = RGB_BY_MASK[mask_type]
mask = birdview[:, :, mask_type]
# If mask above contains 0, don't overwrite content of canvas (0 indicates transparency)
rgb_canvas[nonzero_indices(mask)] = rgb_color
return rgb_canvas
def _render_actors_masks(
self,
agent_vehicle: carla.Actor,
segregated_actors: SegregatedActors,
masks: np.ndarray,
) -> np.ndarray:
"""Fill masks with ones and zeros (more precisely called as "bitmask").
Although numpy dtype is still the same, additional semantic meaning is being added.
"""
lights_masks = self.masks_generator.traffic_lights_masks(
segregated_actors.traffic_lights)
red_lights_mask, yellow_lights_mask, green_lights_mask = lights_masks
masks[BirdViewMasks.RED_LIGHTS.value] = red_lights_mask
masks[BirdViewMasks.YELLOW_LIGHTS.value] = yellow_lights_mask
masks[BirdViewMasks.GREEN_LIGHTS.value] = green_lights_mask
masks[BirdViewMasks.AGENT.
value] = self.masks_generator.agent_vehicle_mask(agent_vehicle)
masks[
BirdViewMasks.VEHICLES.value] = self.masks_generator.vehicles_mask(
segregated_actors.vehicles)
masks[BirdViewMasks.PEDESTRIANS.
value] = self.masks_generator.pedestrians_mask(
segregated_actors.pedestrians)
return masks
def apply_agent_following_transformation_to_masks(
self, agent_vehicle: carla.Actor, masks: np.ndarray) -> np.ndarray:
"""Returns image of shape: height, width, channels"""
agent_transform = agent_vehicle.get_transform()
angle = (agent_transform.rotation.yaw + 90
) # vehicle's front will point to the top
# Rotating around the center
crop_with_car_in_the_center = masks
masks_n, h, w = crop_with_car_in_the_center.shape
rotation_center = Coord(x=w // 2, y=h // 2)
# warpAffine from OpenCV requires the first two dimensions to be in order: height, width, channels
crop_with_centered_car = np.transpose(crop_with_car_in_the_center,
axes=(1, 2, 0))
rotated = rotate(crop_with_centered_car, angle, center=rotation_center)
half_width = self.target_size.width // 2
hslice = slice(rotation_center.x - half_width,
rotation_center.x + half_width)
if self._crop_type is BirdViewCropType.FRONT_AREA_ONLY:
vslice = slice(rotation_center.y - self.target_size.height,
rotation_center.y)
elif self._crop_type is BirdViewCropType.FRONT_AND_REAR_AREA:
half_height = self.target_size.height // 2
vslice = slice(rotation_center.y - half_height,
rotation_center.y + half_height)
else:
raise NotImplementedError
assert (
vslice.start > 0 and hslice.start > 0
), "Trying to access negative indexes is not allowed, check for calculation errors!"
car_on_the_bottom = rotated[vslice, hslice]
return car_on_the_bottom