def sampling_points_from_frustum(height,
width,
focal,
principal_point,
depth_min=0.0,
depth_max=5.,
frustum_size=(256, 256, 256)):
"""Generates samples from a camera frustum."""
# ------------------ Get the rays from the camera ----------------------------
sampling_points = grid.generate((0., 0.),
(float(width) - 1, float(height) - 1),
(frustum_size[0], frustum_size[1]))
sampling_points = tf.reshape(sampling_points, [-1, 2]) # [h*w, 2]
rays = perspective.ray(sampling_points, focal, principal_point) # [h*w, 3]
# ------------------ Extract a volume in front of the camera -----------------
depth_tensor = grid.generate((depth_min, ), (depth_max, ),
(frustum_size[2], ))
sampling_volume = tf.multiply(tf.expand_dims(
rays, axis=-1), tf.transpose(depth_tensor)) # [h*w, 3, dstep]
sampling_volume = tf.transpose(sampling_volume,
[0, 2, 1]) # [h*w, dstep, 3]
sampling_volume = tf.reshape(sampling_volume, [-1, 3]) # [h*w*dstep, 3]
return sampling_volume
def test_generate_random(self):
"""Test the uniform grid generation."""
starts = np.array((0., 0.), dtype=np.float32)
stops = np.random.randint(1, 10, size=(2))
nums = stops + 1
stops = stops.astype(np.float32)
g = grid.generate(starts, stops, nums)
shape = nums.tolist() + [2]
xv, yv = np.meshgrid(range(shape[0]), range(shape[1]), indexing="ij")
gt = np.stack((xv, yv), axis=-1).astype(np.float32)
self.assertAllClose(g, gt)
def generate_ground_image(height,
width,
focal,
principal_point,
camera_rotation_matrix,
camera_translation_vector,
ground_color=(0.43, 0.43, 0.8)):
"""Generate an image depicting only the ground."""
batch_size = camera_rotation_matrix.shape[0]
background_image = np.ones((batch_size, height, width, 1, 1),
dtype=np.float32)
background_image[:, -1, ...] = 0 # Zero the bottom line for proper sampling
# The projection of the ground depends on the top right corner (approximation)
plane_point_np = np.tile(np.array([[3.077984, 2.905388, 0.]],
dtype=np.float32), (batch_size, 1))
plane_point_rotated = rotation_matrix_3d.rotate(plane_point_np,
camera_rotation_matrix)
plane_point_translated = plane_point_rotated + camera_translation_vector
plane_point2d = \
perspective.project(plane_point_translated, focal, principal_point)
_, y = tf.split(plane_point2d, [1, 1], axis=-1)
sfactor = height/y
helper_matrix1 = np.tile(np.array([[[1, 0, 0],
[0, 0, 0],
[0, 0, 0]]]), (batch_size, 1, 1))
helper_matrix2 = np.tile(np.array([[[0, 0, 0],
[0, 1, 0],
[0, 0, 1]]]), (batch_size, 1, 1))
transformation_matrix = tf.multiply(tf.expand_dims(sfactor, -1),
helper_matrix1) + helper_matrix2
plane_points = grid.generate((0., 0., 0.),
(float(height), float(width), 0.),
(height, width, 1))
plane_points = tf.reshape(plane_points, [-1, 3])
transf_plane_points = tf.matmul(transformation_matrix,
plane_points,
transpose_b=True)
interpolated_points = \
trilinear.interpolate(background_image,
tf.linalg.matrix_transpose(transf_plane_points))
ground_alpha = (1- tf.reshape(interpolated_points,
[batch_size, height, width, 1]))
ground_image = tf.ones((batch_size, height, width, 3))*ground_color
return ground_image, ground_alpha
def generate_ground_image(height,
width,
focal,
principal_point,
camera_rotation_matrix,
camera_translation_vector,
ground_color=(0.43, 0.43, 0.8)):
"""Generate an image depicting only the ground."""
background_image = np.ones((height, width, 1, 1), dtype=np.float32)
background_image[-1,
...] = 0 # Set the bottom line to 0 for proper sampling
# The projection of the ground depends on the top right corner (approximation)
plane_point_np = np.array([[3.077984, 2.905388, 0.]], dtype=np.float32)
plane_point2d = \
perspective.project(rotation_matrix_3d.rotate(plane_point_np,
camera_rotation_matrix) +
camera_translation_vector.T, focal, principal_point)
_, y = tf.split(plane_point2d, [1, 1], axis=-1)
sfactor = y / 256.
transformation_matrix = (1/sfactor)*np.array([[1, 0, 0],
[0, 0, 0],
[0, 0, 0]]) + \
np.array([[0, 0, 0],
[0, 1, 0],
[0, 0, 1]])
plane_points = grid.generate(
(0., 0., 0.), (float(height), float(width), 0.), (height, width, 1))
plane_points = tf.reshape(plane_points, [-1, 3])
transf_plane_points = tf.matmul(transformation_matrix,
plane_points,
transpose_b=True)
interpolated_points = \
trilinear.interpolate(background_image, tf.transpose(transf_plane_points))
ground_alpha = (1 - tf.reshape(interpolated_points, [256, 256, 1]))
ground_image = tf.ones((256, 256, 3)) * ground_color
return ground_image, ground_alpha
def perspective_transform(
image: type_alias.TensorLike,
transform_matrix: type_alias.TensorLike,
output_shape: Optional[type_alias.TensorLike] = None,
resampling_type: ResamplingType = ResamplingType.BILINEAR,
border_type: BorderType = BorderType.ZERO,
pixel_type: PixelType = PixelType.HALF_INTEGER,
name: Optional[str] = "perspective_transform",
) -> tf.Tensor:
"""Applies a projective transformation to an image.
The projective transformation is represented by a 3 x 3 matrix
[[a0, a1, a2], [b0, b1, b2], [c0, c1, c2]], mapping a point `[x, y]` to a
transformed point
`[x', y'] = [(a0 x + a1 y + a2) / k, (b0 x + b1 y + b2) / k]`, where
`k = c0 x + c1 y + c2`.
Note:
The transformation matrix maps target to source by transforming output
points to input points.
Args:
image: A tensor of shape `[B, H_i, W_i, C]`, where `B` is the batch size,
`H_i` the height of the image, `W_i` the width of the image, and `C` the
number of channels of the image.
transform_matrix: A tensor of shape `[B, 3, 3]` containing projective
transform matrices. The transformation maps target to source by
transforming output points to input points.
output_shape: The heigh `H_o` and width `W_o` output dimensions after the
transform. If None, output is the same size as input image.
resampling_type: Resampling mode. Supported values are
`ResamplingType.NEAREST` and `ResamplingType.BILINEAR`.
border_type: Border mode. Supported values are `BorderType.ZERO` and
`BorderType.DUPLICATE`.
pixel_type: Pixel mode. Supported values are `PixelType.INTEGER` and
`PixelType.HALF_INTEGER`.
name: A name for this op. Defaults to "perspective_transform".
Returns:
A tensor of shape `[B, H_o, W_o, C]` containing transformed images.
Raises:
ValueError: If `image` has rank != 4. If `transform_matrix` has rank < 3 or
its last two dimensions are not 3. If `image` and `transform_matrix` batch
dimension does not match.
"""
with tf.name_scope(name):
image = tf.convert_to_tensor(value=image, name="image")
transform_matrix = tf.convert_to_tensor(
value=transform_matrix, name="transform_matrix")
output_shape = tf.shape(
input=image)[-3:-1] if output_shape is None else tf.convert_to_tensor(
value=output_shape, name="output_shape")
shape.check_static(image, tensor_name="image", has_rank=4)
shape.check_static(
transform_matrix,
tensor_name="transform_matrix",
has_rank=3,
has_dim_equals=((-1, 3), (-2, 3)))
shape.compare_batch_dimensions(
tensors=(image, transform_matrix),
last_axes=0,
broadcast_compatible=False)
dtype = image.dtype
zero = tf.cast(0.0, dtype)
height, width = tf.unstack(output_shape, axis=-1)
warp = grid.generate(
starts=(zero, zero),
stops=(tf.cast(width, dtype) - 1.0, tf.cast(height, dtype) - 1.0),
nums=(width, height))
warp = tf.transpose(a=warp, perm=[1, 0, 2])
if pixel_type == PixelType.HALF_INTEGER:
warp += 0.5
padding = [[0, 0] for _ in range(warp.shape.ndims)]
padding[-1][-1] = 1
warp = tf.pad(
tensor=warp, paddings=padding, mode="CONSTANT", constant_values=1.0)
warp = warp[..., tf.newaxis]
transform_matrix = transform_matrix[:, tf.newaxis, tf.newaxis, ...]
warp = tf.linalg.matmul(transform_matrix, warp)
warp = warp[..., 0:2, 0] / warp[..., 2, :]
return sample(image, warp, resampling_type, border_type, pixel_type)
def sampling_points_from_3d_grid(grid_size, dtype=tf.float32):
"""Returns a tensor of shape `[M, 3]`, with M the number of sampling points."""
sampling_points = grid.generate((-1.0, -1.0, -1.0), (1.0, 1.0, 1.0),
grid_size)
sampling_points = tf.cast(sampling_points, dtype)
return tf.reshape(sampling_points, [-1, 3])
def test_rasterize_preset(self):
camera_origin = (0.0, 0.0, 0.0)
camera_up = (0.0, 1.0, 0.0)
look_at_point = (0.0, 0.0, 1.0)
field_of_view = (60 * np.math.pi / 180, )
near_plane = (0.01, )
far_plane = (400.0, )
# Construct the view projection matrix.
model_to_eye_matrix = look_at.right_handed(camera_origin,
look_at_point, camera_up)
perspective_matrix = perspective.right_handed(
field_of_view, (float(_IMAGE_WIDTH) / float(_IMAGE_HEIGHT), ),
near_plane, far_plane)
view_projection_matrix = tf.linalg.matmul(perspective_matrix,
model_to_eye_matrix)
view_projection_matrix = tf.expand_dims(view_projection_matrix, axis=0)
depth = 1.0
vertices = np.array([[(-2.0 * _TRIANGLE_SIZE, 0.0, depth),
(0.0, _TRIANGLE_SIZE, depth), (0.0, 0.0, depth),
(0.0, -_TRIANGLE_SIZE, depth)]],
dtype=np.float32)
triangles = np.array(((1, 2, 0), (0, 2, 3)), np.int32)
predicted_fb = rasterization_backend.rasterize(
vertices, triangles, view_projection_matrix,
(_IMAGE_WIDTH, _IMAGE_HEIGHT))
with self.subTest(name="triangle_index"):
groundtruth_triangle_index = np.zeros(
(1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1), dtype=np.int32)
groundtruth_triangle_index[..., :_IMAGE_WIDTH // 2, 0] = 0
groundtruth_triangle_index[..., :_IMAGE_HEIGHT // 2,
_IMAGE_WIDTH // 2:, 0] = 1
self.assertAllEqual(groundtruth_triangle_index,
predicted_fb.triangle_id)
with self.subTest(name="mask"):
groundtruth_mask = np.ones((1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1),
dtype=np.int32)
groundtruth_mask[..., :_IMAGE_WIDTH // 2, 0] = 0
self.assertAllEqual(groundtruth_mask, predicted_fb.foreground_mask)
attributes = np.array(((1.0, 0.0, 0.0), (0.0, 1.0, 0.0),
(0.0, 0.0, 1.0))).astype(np.float32)
perspective_correct_interpolation = lambda geometry, pixels: glm.perspective_correct_interpolation( # pylint: disable=g-long-lambda,line-too-long
geometry, attributes, pixels, model_to_eye_matrix,
perspective_matrix,
np.array((_IMAGE_WIDTH, _IMAGE_HEIGHT)).astype(np.float32),
np.array((0.0, 0.0)).astype(np.float32))
with self.subTest(name="barycentric_coordinates_triangle_0"):
geometry_0 = tf.gather(vertices, triangles[0, :], axis=1)
pixels_0 = tf.transpose(grid.generate((3.5, 2.5), (6.5, 4.5),
(4, 3)),
perm=(1, 0, 2))
barycentrics_gt_0 = perspective_correct_interpolation(
geometry_0, pixels_0)
self.assertAllClose(barycentrics_gt_0,
predicted_fb.barycentrics.value[0, 2:, 3:, :],
atol=1e-3)
with self.subTest(name="barycentric_coordinates_triangle_1"):
geometry_1 = tf.gather(vertices, triangles[1, :], axis=1)
pixels_1 = tf.transpose(grid.generate((3.5, 0.5), (6.5, 1.5),
(4, 2)),
perm=(1, 0, 2))
barycentrics_gt_1 = perspective_correct_interpolation(
geometry_1, pixels_1)
self.assertAllClose(barycentrics_gt_1,
predicted_fb.barycentrics.value[0, 0:2, 3:, :],
atol=1e-3)
def random_patches(focal: tf.Tensor,
principal_point: tf.Tensor,
height: int,
width: int,
patch_height: int,
patch_width: int,
scale: float = 1.0,
indexing: str = "ij",
name: str = None) -> Tuple[tf.Tensor, tf.Tensor]:
"""Sample patches at different scales and from an image.
Args:
focal: A tensor of shape `[A1, ..., An, 2]`
principal_point: A tensor of shape `[A1, ..., An, 2]`
height: The height of the image plane in pixels.
width: The width of the image plane in pixels.
patch_height: The height M of the patch in pixels.
patch_width: The width N of the patch in pixels.
scale: The scale of the patch.
indexing: Indexing of the patch ('ij' or 'xy')
name: A name for this op that defaults to "random_patches".
Returns:
A tensor of shape `[A1, ..., An, M*N, 3]` where the last dimension is the
ray directions in 3D passing from the M*N pixels of the patch and
a tensor of shape `[A1, ..., An, M*N, 2]` with the pixel x, y locations.
"""
with tf.compat.v1.name_scope(name, "random_patches",
[focal, principal_point]):
focal = tf.convert_to_tensor(value=focal)
principal_point = tf.convert_to_tensor(value=principal_point)
shape.check_static(tensor=focal,
tensor_name="focal",
has_dim_equals=(-1, 2))
shape.check_static(tensor=principal_point,
tensor_name="principal_point",
has_dim_equals=(-1, 2))
shape.compare_batch_dimensions(tensors=(focal, principal_point),
tensor_names=("focal",
"principal_point"),
last_axes=-2,
broadcast_compatible=True)
if indexing not in ["xy", "ij"]:
raise ValueError("'axis' needs to be 'xy' or 'ij'")
batch_shape = tf.shape(focal)[:-1]
patch = grid.generate([0, 0], [patch_width - 1, patch_height - 1],
[patch_width, patch_height])
if indexing == "xy":
patch = tf.reverse(patch, axis=[-1])
patch = tf.cast(patch, tf.float32)
patch = patch * scale
interm_shape = tf.concat([tf.ones_like(batch_shape),
tf.shape(patch)],
axis=0)
patch = tf.reshape(patch, interm_shape)
random_y = tf.random.uniform(batch_shape,
minval=0,
maxval=height -
int(patch_height * scale) + 1,
dtype=tf.int32)
random_x = tf.random.uniform(batch_shape,
minval=0,
maxval=width - int(patch_width * scale) +
1,
dtype=tf.int32)
patch_origins = tf.cast(tf.stack([random_x, random_y], axis=-1),
tf.float32)
patch_origins = tf.expand_dims(tf.expand_dims(patch_origins, -2), -2)
pixels = tf.cast(patch + patch_origins, tf.float32)
final_shape = tf.concat([batch_shape, [patch_height * patch_width, 2]],
axis=0)
pixels = tf.reshape(pixels, final_shape)
rays = ray(pixels, tf.expand_dims(focal, -2),
tf.expand_dims(principal_point, -2))
return rays, pixels
def test_rasterize_preset(self):
model_to_eye_matrix = rasterization_test_utils.make_look_at_matrix(
look_at_point=(0.0, 0.0, 1.0))
perspective_matrix = rasterization_test_utils.make_perspective_matrix(
_IMAGE_WIDTH, _IMAGE_HEIGHT)
view_projection_matrix = tf.linalg.matmul(perspective_matrix,
model_to_eye_matrix)
view_projection_matrix = tf.expand_dims(view_projection_matrix, axis=0)
depth = 1.0
vertices = np.array([[(-2.0 * _TRIANGLE_SIZE, 0.0, depth),
(0.0, _TRIANGLE_SIZE, depth), (0.0, 0.0, depth),
(0.0, -_TRIANGLE_SIZE, depth)]],
dtype=np.float32)
triangles = np.array(((1, 2, 0), (0, 2, 3)), np.int32)
predicted_fb = _proxy_rasterize(vertices, triangles,
view_projection_matrix)
with self.subTest(name="triangle_index"):
groundtruth_triangle_index = np.zeros(
(1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1), dtype=np.int32)
groundtruth_triangle_index[..., :_IMAGE_WIDTH // 2, 0] = 0
groundtruth_triangle_index[..., :_IMAGE_HEIGHT // 2,
_IMAGE_WIDTH // 2:, 0] = 1
self.assertAllEqual(groundtruth_triangle_index,
predicted_fb.triangle_id)
with self.subTest(name="mask"):
groundtruth_mask = np.ones((1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1),
dtype=np.int32)
groundtruth_mask[..., :_IMAGE_WIDTH // 2, 0] = 0
self.assertAllEqual(groundtruth_mask, predicted_fb.foreground_mask)
attributes = np.array(((1.0, 0.0, 0.0), (0.0, 1.0, 0.0),
(0.0, 0.0, 1.0))).astype(np.float32)
perspective_correct_interpolation = lambda geometry, pixels: glm.perspective_correct_interpolation( # pylint: disable=g-long-lambda,line-too-long
geometry, attributes, pixels, model_to_eye_matrix,
perspective_matrix,
np.array((_IMAGE_WIDTH, _IMAGE_HEIGHT)).astype(np.float32),
np.array((0.0, 0.0)).astype(np.float32))
with self.subTest(name="barycentric_coordinates_triangle_0"):
geometry_0 = tf.gather(vertices, triangles[0, :], axis=1)
pixels_0 = tf.transpose(grid.generate((3.5, 2.5), (6.5, 4.5),
(4, 3)),
perm=(1, 0, 2))
barycentrics_gt_0 = perspective_correct_interpolation(
geometry_0, pixels_0)
self.assertAllClose(barycentrics_gt_0,
predicted_fb.barycentrics.value[0, 2:, 3:, :],
atol=1e-3)
with self.subTest(name="barycentric_coordinates_triangle_1"):
geometry_1 = tf.gather(vertices, triangles[1, :], axis=1)
pixels_1 = tf.transpose(grid.generate((3.5, 0.5), (6.5, 1.5),
(4, 2)),
perm=(1, 0, 2))
barycentrics_gt_1 = perspective_correct_interpolation(
geometry_1, pixels_1)
self.assertAllClose(barycentrics_gt_1,
predicted_fb.barycentrics.value[0, 0:2, 3:, :],
atol=1e-3)
def update(self, labeled_sdfs, labeled_classes, labeled_poses,
predicted_sdfs, predicted_classes, predicted_poses):
"""Update."""
if labeled_sdfs or labeled_classes:
print(labeled_sdfs)
mean_x = tf.reduce_mean(labeled_poses[1][:, 0])
mean_z = tf.reduce_mean(labeled_poses[1][:, 2])
samples_world = grid.generate(
(mean_x - 0.5, 0.0, mean_z - 0.5),
(mean_x + 0.5, 1.0, mean_z + 0.5),
[self.resolution, self.resolution, self.resolution])
samples_world = tf.reshape(samples_world, [-1, 3])
status = False
if status:
_, axs = plt.subplots(3, 3)
fig_obj_count = 0
# Do the same for the ground truth and predictions
num_collisions = 0
prev_intersection = 0
sdf_values = tf.zeros_like(samples_world)[:, 0:1]
for classes, sdfs, poses in [(predicted_classes, predicted_sdfs,
predicted_poses)]:
for i in range(classes.shape[0]):
sdf = tf.expand_dims(sdfs[i], -1)
sdf = sdf * -1.0 # inside positive, outside zero
samples_object = centernet_utils.transform_pointcloud(
tf.reshape(samples_world, [1, 1, -1, 3]),
tf.reshape(poses[2][i], [1, 1, 3]),
tf.reshape(poses[0][i], [1, 1, 3, 3]),
tf.reshape(poses[1][i], [1, 1, 3]),
inverse=True) * 2.0
samples_object = (samples_object *
(29.0 / 32.0) / 2.0 + 0.5) * 32.0 - 0.5
samples = tf.squeeze(samples_object)
interpolated = trilinear.interpolate(sdf, samples)
occupancy_value = tf.math.sign(
tf.nn.relu(interpolated + self.tol))
sdf_values += occupancy_value
intersection = tf.reduce_sum(
tf.math.sign(tf.nn.relu(sdf_values - 1)))
if intersection > prev_intersection:
prev_intersection = intersection
num_collisions += 1
status2 = False
if status2:
a = 1
values = interpolated
inter = tf.reshape(
values,
[self.resolution, self.resolution, self.resolution])
inter = tf.transpose(tf.reduce_max(inter, axis=a))
im = axs[fig_obj_count, 0].matshow(inter.numpy())
plt.colorbar(im, ax=axs[fig_obj_count, 0])
values = tf.math.sign(tf.nn.relu(interpolated + self.tol))
inter = tf.reshape(
values,
[self.resolution, self.resolution, self.resolution])
inter = tf.transpose(tf.reduce_max(inter, axis=a))
im = axs[fig_obj_count, 1].matshow(inter.numpy())
plt.colorbar(im, ax=axs[fig_obj_count, 1])
values = sdf_values
inter = tf.reshape(
values,
[self.resolution, self.resolution, self.resolution])
inter = tf.transpose(tf.reduce_max(inter, axis=a))
im = axs[fig_obj_count, 2].matshow(inter.numpy())
plt.colorbar(im, ax=axs[fig_obj_count, 2])
fig_obj_count += 1
intersection = tf.reduce_sum(tf.math.sign(tf.nn.relu(sdf_values - 1)))
union = tf.reduce_sum(tf.math.sign(sdf_values))
iou = intersection / union
self.collisions.append(num_collisions)
self.intersections.append(intersection)
self.ious.append(iou)
return num_collisions, intersection, iou
def update(self, labeled_sdfs, labeled_classes, labeled_poses,
predicted_sdfs, predicted_classes, predicted_poses):
"""Update."""
labeled_rotations = labeled_poses[0]
labeled_translations = labeled_poses[1]
labeled_sizes = labeled_poses[2]
status = True
if status:
box_limits_x = [100, -100]
# box_limits_y = [100, -100]
box_limits_z = [100, -100]
for i in range(labeled_translations.shape[0]):
rot = tf.reshape(tf.gather(labeled_rotations[i], [0, 2, 6, 8]),
[2, 2])
min_x = tf.cast(0.0 - labeled_sizes[i][0] / 2.0,
dtype=tf.float32)
max_x = tf.cast(0.0 + labeled_sizes[i][0] / 2.0,
dtype=tf.float32)
# min_y = tf.cast(0.0 - labeled_sizes[i][1] / 2.0, dtype=tf.float32)
# max_y = tf.cast(0.0 + labeled_sizes[i][1] / 2.0, dtype=tf.float32)
min_z = tf.cast(0.0 - labeled_sizes[i][2] / 2.0,
dtype=tf.float32)
max_z = tf.cast(0.0 + labeled_sizes[i][2] / 2.0,
dtype=tf.float32)
translation = tf.reshape(
[labeled_translations[i][0], labeled_translations[i][2]],
[2, 1])
pt_0 = rot @ tf.reshape([min_x, min_z], [2, 1]) + translation
pt_1 = rot @ tf.reshape([min_x, max_z], [2, 1]) + translation
pt_2 = rot @ tf.reshape([max_x, min_z], [2, 1]) + translation
pt_3 = rot @ tf.reshape([max_x, max_z], [2, 1]) + translation
for pt in [pt_0, pt_1, pt_2, pt_3]:
if pt[0] < box_limits_x[0]:
box_limits_x[0] = pt[0]
if pt[0] > box_limits_x[1]:
box_limits_x[1] = pt[0]
if pt[1] < box_limits_z[0]:
box_limits_z[0] = pt[1]
if pt[1] > box_limits_z[1]:
box_limits_z[1] = pt[1]
mean_x = tf.reduce_mean(box_limits_x)
mean_z = tf.reduce_mean(box_limits_z)
else:
mean_x = tf.reduce_mean(labeled_translations[:, 0])
mean_z = tf.reduce_mean(labeled_translations[:, 2])
samples_world = grid.generate(
(mean_x - 0.5, 0.0, mean_z - 0.5),
(mean_x + 0.5, 1.0, mean_z + 0.5),
[self.resolution, self.resolution, self.resolution])
# samples_world = grid.generate(
# (box_limits_x[0][0], box_limits_y[0], box_limits_z[0][0]),
# (box_limits_x[1][0], box_limits_y[1], box_limits_z[1][0]),
# [self.resolution, self.resolution, self.resolution])
# samples_world = grid.generate(
# (-5.0, -5.0, -5.0),
# (5.0, 5.0, 5.0),
# [self.resolution, self.resolution, self.resolution])
samples_world = tf.reshape(samples_world, [-1, 3])
ious = []
status = False
if status:
_, axs = plt.subplots(labeled_translations.shape[0], 5)
fig_obj_count = 0
for class_id in range(self.max_num_classes):
# Do the same for the ground truth and predictions
sdf_values = tf.zeros_like(samples_world)[:, 0:1]
for mtype, (classes, sdfs, poses) in enumerate([
(labeled_classes, labeled_sdfs, labeled_poses),
(predicted_classes, predicted_sdfs, predicted_poses)
]):
for i in range(classes.shape[0]):
if class_id == classes[i]:
sdf = tf.expand_dims(sdfs[i], -1)
sdf = sdf * -1.0 # inside positive, outside zero
samples_object = centernet_utils.transform_pointcloud(
tf.reshape(samples_world, [1, 1, -1, 3]),
tf.reshape(poses[2][i], [1, 1, 3]),
tf.reshape(poses[0][i], [1, 1, 3, 3]),
tf.reshape(poses[1][i], [1, 1, 3]),
inverse=True) * 2.0
samples_object = \
(samples_object * (29.0/32.0) / 2.0 + 0.5) * 32.0 - 0.5
samples = tf.squeeze(samples_object)
interpolated = trilinear.interpolate(sdf, samples)
sdf_values += tf.math.sign(
tf.nn.relu(interpolated + self.tol))
status2 = False
if status2:
a = 2
values = interpolated
inter = tf.reshape(values, [
self.resolution, self.resolution,
self.resolution
])
inter = tf.transpose(tf.reduce_max(inter, axis=a))
im = axs[fig_obj_count,
mtype * 2 + 0].matshow(inter.numpy())
plt.colorbar(im,
ax=axs[fig_obj_count, mtype * 2 + 0])
print(mtype, fig_obj_count, 0)
values = tf.math.sign(
tf.nn.relu(interpolated + self.tol))
inter = tf.reshape(values, [
self.resolution, self.resolution,
self.resolution
])
inter = tf.transpose(tf.reduce_max(inter, axis=a))
im = axs[fig_obj_count,
mtype * 2 + 1].matshow(inter.numpy())
plt.colorbar(im,
ax=axs[fig_obj_count, mtype * 2 + 1])
print(mtype, fig_obj_count, 1)
if mtype == 1:
values = sdf_values
inter = tf.reshape(values, [
self.resolution, self.resolution,
self.resolution
])
inter = tf.transpose(
tf.reduce_max(inter, axis=a))
im = axs[fig_obj_count,
4].matshow(inter.numpy())
plt.colorbar(im, ax=axs[fig_obj_count, 4])
print(mtype, fig_obj_count, 2)
fig_obj_count += 1
intersection = tf.reduce_sum(
tf.math.sign(tf.nn.relu(sdf_values - 1)))
union = tf.reduce_sum(tf.math.sign(sdf_values))
iou = intersection / union
if not tf.math.is_nan(iou):
ious.append(iou)
status3 = False
if status3:
_ = plt.figure(figsize=(5, 5))
plt.clf()
# mask = (sdf_values.numpy() > 0)[:, 0]
# plt.scatter(samples_world.numpy()[mask, 0],
# samples_world.numpy()[mask, 1],
# marker='.', c=sdf_values.numpy()[mask, 0])
plt.scatter(samples_world.numpy()[:, 0],
samples_world.numpy()[:, 1],
marker='.',
c=sdf_values.numpy()[:, 0])
plt.colorbar()
if not tf.math.is_nan(iou):
self.iou_per_class[class_id].append(iou)
if ious:
ious = [0]
return np.mean(ious), np.min(ious)
