def visualize_ground_bboxes(op_name, timestamp, image_np, det_objs):
""" Creates a cv2 window to visualize detected objects."""
add_timestamp(timestamp, image_np)
for det_obj in det_objs:
det_obj.visualize_on_img(image_np, GROUND_COLOR_MAP)
cv2.imshow(op_name, image_np)
cv2.waitKey(1)
def on_msg_camera_stream(self, msg):
"""Camera stream callback method.
Invoked upon the receipt of a message on the camera stream.
"""
self._logger.info('{} received frame {}'.format(
self.name, msg.timestamp))
start_time = time.time()
assert msg.encoding == 'BGR', 'Expects BGR frames'
image = np.expand_dims(msg.frame.transpose([2, 0, 1]), axis=0)
tensor = torch.tensor(image).float().cuda() / 255.0
output = self._network(tensor)
# XXX(ionel): Check if the model outputs Carla Cityscapes values or
# correct Cityscapes values.
output = transform_to_cityscapes_palette(
torch.argmax(output, dim=1).cpu().numpy()[0])
output = rgb_to_bgr(output)
if self._flags.visualize_segmentation_output:
add_timestamp(msg.timestamp, output)
cv2.imshow(self.name, output)
cv2.waitKey(1)
# Get runtime in ms.
runtime = (time.time() - start_time) * 1000
self._csv_logger.info('{},{},"{}",{}'.format(time_epoch_ms(),
self.name, msg.timestamp,
runtime))
output_msg = SegmentedFrameMessage(output, runtime, msg.timestamp)
self.get_output_stream(self._output_stream_name).send(output_msg)
def on_msg_camera_stream(self, msg):
"""Camera stream callback method.
Invoked upon the receipt of a message on the camera stream.
"""
self._logger.info('{} received frame {}'.format(
self.name, msg.timestamp))
start_time = time.time()
assert msg.encoding == 'BGR', 'Expects BGR frames'
image = torch.from_numpy(msg.frame.transpose([2, 0, 1
])).unsqueeze(0).float()
image_var = Variable(image, requires_grad=False, volatile=True)
final = self._model(image_var)[0]
_, pred = torch.max(final, 1)
pred = pred.cpu().data.numpy()[0]
image_np = self._pallete[pred.squeeze()]
# After we apply the pallete, the image is in RGB format
image_np = rgb_to_bgr(image_np)
if self._flags.visualize_segmentation_output:
add_timestamp(msg.timestamp, image_np)
cv2.imshow(self.name, image_np)
cv2.waitKey(1)
# Get runtime in ms.
runtime = (time.time() - start_time) * 1000
self._csv_logger.info('{},{},"{}",{}'.format(time_epoch_ms(),
self.name, msg.timestamp,
runtime))
output_msg = SegmentedFrameMessage(image_np, runtime, msg.timestamp)
self.get_output_stream(self._output_stream_name).send(output_msg)
def visualize_bboxes(op_name, timestamp, image_np, detected_objs,
bbox_color_map):
""" Creates a cv2 window to visualize detected objects."""
add_timestamp(timestamp, image_np)
for detected_obj in detected_objs:
detected_obj.visualize_on_img(image_np, bbox_color_map)
cv2.imshow(op_name, image_np)
cv2.waitKey(1)
def visualize_bboxes(
op_name, timestamp, image_np, detected_objs, bbox_color_map):
# txt_font = cv2.FONT_HERSHEY_SIMPLEX
add_timestamp(timestamp, image_np)
for detected_obj in detected_objs:
detected_obj.visualize_on_img(image_np, bbox_color_map)
cv2.imshow(op_name, image_np)
cv2.waitKey(1)
def annotate_image_with_bboxes(
timestamp, image_np, detected_objs, bbox_color_map=GROUND_COLOR_MAP):
""" Adds bounding boxes to an image."""
# txt_font = cv2.FONT_HERSHEY_SIMPLEX
add_timestamp(timestamp, image_np)
for detected_obj in detected_objs:
detected_obj.visualize_on_img(image_np, bbox_color_map)
return image_np
def visualize(self, window_name, timestamp=None):
"""Creates a cv2 window to visualize the segmented frame."""
cityscapes_frame = self.as_cityscapes_palette()
# Convert to BGR
cityscapes_frame = cityscapes_frame[:, :, ::-1]
if timestamp is not None:
add_timestamp(cityscapes_frame, timestamp)
cv2.imshow(window_name, cityscapes_frame)
cv2.waitKey(1)
def visualize_no_colors_bboxes(op_name, timestamp, image_np, bboxes):
add_timestamp(timestamp, image_np)
for corners in bboxes:
(xmin, xmax, ymin, ymax) = corners
color = [128, 0, 0]
# Show bounding box.
cv2.rectangle(image_np, (xmin, ymin), (xmax, ymax), color, 2)
cv2.imshow(op_name, image_np)
cv2.waitKey(1)
def on_msg_camera_stream(self, msg):
start_time = time.time()
assert msg.encoding == 'BGR', 'Expects BGR frames'
image_np = msg.frame
# TODO(ionel): Implement lane detection.
edges = self.apply_canny(image_np)
lines_edges = self.apply_hough(image_np, edges)
kernel_size = 3
grad_x = self.apply_sobel(image_np,
orient='x',
sobel_kernel=kernel_size,
thresh_min=0,
thresh_max=255)
grad_y = self.apply_sobel(image_np,
orient='y',
sobel_kernel=kernel_size,
thresh_min=0,
thresh_max=255)
mag_binary = self.magnitude_threshold(image_np,
sobel_kernel=kernel_size,
thresh_min=0,
thresh_max=255)
dir_binary = self.direction_threshold(image_np,
sobel_kernel=kernel_size,
thresh_min=0,
thresh_max=np.pi / 2)
s_binary = self.extract_s_channel(image_np)
# Select the pixels where both x and y gradients meet the threshold
# criteria, or the gradient magnitude and direction are both with
# the threshold values.
combined = np.zeros_like(dir_binary)
combined[((grad_x == 1) & (grad_y == 1)) | ((mag_binary == 1) & (dir_binary == 1))] = 1
combined_binary = np.zeros_like(grad_x)
combined_binary[(s_binary == 1) | (grad_x == 1)] = 1
# Get runtime in ms.
runtime = (time.time() - start_time) * 1000
self._csv_logger.info('{},{},"{}",{}'.format(
time_epoch_ms(), self.name, msg.timestamp, runtime))
if self._flags.visualize_lane_detection:
add_timestamp(msg.timestamp, lines_edges)
cv2.imshow(self.name, lines_edges)
cv2.waitKey(1)
output_msg = Message(image_np, msg.timestamp)
self.get_output_stream(self._output_stream_name).send(output_msg)
def visualize_ground_bboxes(op_name, timestamp, image_np, det_objs):
add_timestamp(timestamp, image_np)
for det_obj in det_objs:
det_obj.visualize_on_img(image_np, GROUND_COLOR_MAP)
cv2.imshow(op_name, image_np)
cv2.waitKey(1)
def display_frame(self, msg):
add_timestamp(msg.timestamp, msg.frame)
cv2.imshow(self.name, msg.frame)
cv2.waitKey(1)
def display_frame(self, msg):
assert msg.encoding == 'BGR', 'Expects BGR frames'
add_timestamp(msg.timestamp, msg.frame)
cv2.imshow(self.name, msg.frame)
cv2.waitKey(1)