def receive_image(self, img_msg: ImageMsg):
"""Process and republish new camera image.
The image is cropped to simulate the internal preprocessing of our real camera.
Args:
img_msg: Published image message.
"""
br = CvBridge()
cv_img = br.imgmsg_to_cv2(img_msg)
new_img = cv_img[self.param.output_start_y:self.param.output_end_y,
self.param.output_start_x:self.param.output_end_x, ]
if self.param.apply_gan:
new_img = self.gan_translator(
new_img,
f_keep_pixels=self.param.factor_keep_pixels,
f_keep_colored_pixels=self.param.factor_keep_colored_pixels,
)
else:
new_img = cv2.cvtColor(new_img, cv2.COLOR_BGR2GRAY)
out_msg = br.cv2_to_imgmsg(new_img, encoding="mono8")
out_msg.header = img_msg.header
self.publisher.publish(out_msg)
def __enter__(self):
bridge = CvBridge()
cv_img = bridge.imgmsg_to_cv2(self.image_msg,
desired_encoding=self.desired_encoding)
self.tmpfile = tempfile.NamedTemporaryFile(suffix=".jpg")
cv2.imwrite(self.tmpfile.name, cv_img)
return self.tmpfile.__enter__()
def __enter__(self):
bridge = CvBridge()
cv_img = bridge.imgmsg_to_cv2(self.image_msg, desired_encoding=self.desired_encoding)
self.tmpfile = tempfile.NamedTemporaryFile(suffix=".jpg")
cv2.imwrite(self.tmpfile.name, cv_img)
return self.tmpfile.__enter__()
def receive_image(self, msg: ImageMsg):
"""Receive new camera image and publish corresponding labels."""
try:
tf_transform = self.listener.lookup_transform(
"vehicle", "sim_world", msg.header.stamp, timeout=rospy.Duration(0.1)
)
current_tf = Transform(tf_transform.transform)
except Exception as e:
rospy.logerr(f"Could not lookup transform {e}")
return
# Pass latest car state to the speaker to ensure tf and
# car state are approximately synchronized
self.label_speaker.listen(self._latest_car_state_msg)
try:
# Pause physics to prevent the car from moving any further
# while calculating the bounding boxes
self.pause_physics_proxy()
image_size = (
self.camera_specs.output_size["width"],
self.camera_specs.output_size["height"],
)
# Update transformations
BoundingBox.set_tfs(current_tf, self.camera_specs.P)
visible_bbs = self.label_speaker.speak(
image_size, self.camera_specs.horizontal_fov
)
self.label_publisher.publish(
ImageLabelsMsg(
img_header=msg.header,
bounding_boxes=[bb.to_msg() for bb in visible_bbs],
)
)
# Optionally publish an image with bounding boxes drawn into it
if self.param.publish_debug_image:
colors = self.param.colors.as_dict()
visualization_bbs = (
VisualBoundingBox(
bb.get_bounds(),
label=bb.class_description,
color=colors[str(bb.class_id // 100)]
if str(bb.class_id // 100) in colors
# ID Namespaces are in steps of 100.
else colors[str(-1)],
)
for bb in visible_bbs
)
br = CvBridge()
cv_img = br.imgmsg_to_cv2(msg)
cv_img: np.ndarray = cv_img.copy()
cv_img = cv2.cvtColor(cv_img, cv2.COLOR_GRAY2RGB)
for bb in visualization_bbs:
bb.draw(cv_img)
out_msg = br.cv2_to_imgmsg(cv_img, encoding="rgb8")
out_msg.header = msg.header
self.image_publisher.publish(out_msg)
finally:
self.unpause_physics_proxy()
def image_parse(img, enc='passthrough'):
""" Converts a ros image to a numpy array
"""
br = CvBridge()
image = np.asarray(br.imgmsg_to_cv2(img, desired_encoding=enc))
return image