def _point_conv(self,
kernel_inputs,
neighborhood,
pdf,
features,
non_linearity_type='relu'):
""" Method to compute a PointConv convolution using a single
MLP with one hidden layer as implicit convolution kernel function.
Args:
kernel_inputs: A `float` `Tensor` of shape `[M, L]`, the input to the
kernel MLP.
neighborhood: A `Neighborhood` instance.
pdf: A `float` `Tensor` of shape `[M]`, the point densities.
features: A `float` `Tensor` of shape `[N, C1]`, the input features.
non_linearity_type: An `string`, specifies the type of the activation
function used inside the kernel MLP.
Possible: `'ReLU', 'lReLU', 'ELU'`, defaults to leaky ReLU. (optional)
Returns:
A `float` `Tensor` of shape `[N,C2]`, the output features.
"""
# Compute the hidden layer MLP
basis_neighs = tf.matmul(kernel_inputs, self._basis_axis_tf) + \
self._basis_bias_tf
basis_neighs = \
non_linearity_types[non_linearity_type.lower()](basis_neighs)
# Normalize the pdf
max_pdf = tf.math.unsorted_segment_max(
pdf,
neighborhood._original_neigh_ids[:, 1],
tf.shape(neighborhood._samples_neigh_ranges)[0])
neigh_max_pdfs = tf.gather(max_pdf, neighborhood._original_neigh_ids[:, 1])
cur_pdf = pdf / neigh_max_pdfs
cur_pdf = tf.reshape(cur_pdf, [-1, 1])
# Non-linear transform pdf
cur_pdf = tf.nn.relu(tf.matmul(cur_pdf, self._weights_pdf[0]) +\
self._biases_pdf[0])
cur_pdf = tf.matmul(cur_pdf, self._weights_pdf[1]) + self._biases_pdf[1]
# Scale features
basis_neighs = basis_neighs / cur_pdf
# Compute the projection to the samples.
weighted_features = basis_proj(
basis_neighs,
features,
neighborhood)
#Compute convolution - hidden layer to output (linear)
convolution_result = tf.matmul(
tf.reshape(weighted_features,
[-1, self._num_features_in * self._size_hidden]),
self._weights)
return convolution_result
def _monte_carlo_conv(self,
kernel_inputs,
neighborhood,
pdf,
features,
non_linearity_type='leaky_relu'):
""" Method to compute a Monte-Carlo integrated convolution using multiple
MLPs as implicit convolution kernel functions.
Args:
kernel_inputs: A `float` `Tensor` of shape `[M, L]`, the input to the
kernel MLP.
neighborhood: A `Neighborhood` instance.
pdf: A `float` `Tensor` of shape `[M]`, the point densities.
features: A `float` `Tensor` of shape `[N, C1]`, the input features.
non_linearity_type: An `string`, specifies the type of the activation
function used inside the kernel MLP.
Possible: `'ReLU', 'leaky_ReLU', 'ELU'`, defaults to leaky ReLU.
(optional)
Returns:
A `float` `Tensor` of shape `[N,C2]`, the output features.
"""
# Compute the hidden layer MLP
cur_inputs = tf.tile(
tf.reshape(kernel_inputs, [1, -1, self._num_dims]),
[self._num_mlps, 1, 1])
for cur_layer_iter in range(len(self._weights_tf)):
cur_inputs = tf.matmul(cur_inputs, self._weights_tf[cur_layer_iter]) + \
self._bias_tf[cur_layer_iter]
cur_inputs = non_linearity_types[non_linearity_type.lower()](
cur_inputs)
cur_inputs = tf.reshape(tf.transpose(cur_inputs, [1, 0, 2]),
[-1, self._mlp_size[-1] * self._num_mlps]) \
/ tf.reshape(pdf, [-1, 1])
# Compute the projection to the samples.
weighted_features = basis_proj(cur_inputs, features, neighborhood)
# Reshape features
weighted_features = tf.transpose(
tf.reshape(weighted_features, [
-1, self._num_features_in, self._num_mlps, self._mlp_size[-1]
]), [2, 0, 1, 3])
#Compute convolution - hidden layer to output (linear)
convolution_result = tf.matmul(
tf.reshape(weighted_features, [
self._num_mlps, -1, self._num_features_in * self._mlp_size[-1]
]), self._final_weights_tf)
return tf.reshape(tf.transpose(convolution_result, [1, 0, 2]),
[-1, self._num_features_out])
def test_basis_proj(self, num_points, num_samples, num_features,
batch_size, radius, hidden_size, dimension):
cell_sizes = np.float32(np.repeat(radius, dimension))
points, batch_ids = utils._create_random_point_cloud_segmented(
batch_size, num_points, dimension=dimension)
features = np.random.rand(num_points, num_features[0])
point_cloud = PointCloud(points, batch_ids)
point_samples, batch_ids_samples = \
utils._create_random_point_cloud_segmented(
batch_size, num_samples, dimension=dimension)
point_cloud_samples = PointCloud(point_samples, batch_ids_samples)
grid = Grid(point_cloud, cell_sizes)
neighborhood = Neighborhood(grid, cell_sizes, point_cloud_samples)
nb_ids = neighborhood._original_neigh_ids
# tf
conv_layer = MCConv(num_features[0], num_features[1], dimension, 1,
[hidden_size])
basis_weights_tf = tf.reshape(conv_layer._weights_tf[0],
[dimension, hidden_size])
basis_biases_tf = tf.reshape(conv_layer._bias_tf[0], [1, hidden_size])
neigh_point_coords = points[nb_ids[:, 0]]
center_point_coords = point_samples[nb_ids[:, 1]]
kernel_input = (neigh_point_coords - center_point_coords) / radius
basis_neighs = \
tf.matmul(kernel_input.astype(np.float32), basis_weights_tf) + \
basis_biases_tf
basis_neighs = tf.nn.relu(basis_neighs)
weighted_latent_per_sample_tf = basis_proj(basis_neighs, features,
neighborhood)
# numpy
neighbor_ids = neighborhood._original_neigh_ids.numpy()
nb_ranges = neighborhood._samples_neigh_ranges.numpy()
# extract variables
hidden_weights = basis_weights_tf.numpy()
hidden_biases = basis_biases_tf.numpy()
features_on_neighbors = features[neighbor_ids[:, 0]]
# compute first layer of kernel MLP
point_diff = (points[neighbor_ids[:, 0]] -\
point_samples[neighbor_ids[:, 1]])\
/ np.expand_dims(cell_sizes, 0)
latent_per_nb = np.dot(point_diff, hidden_weights) + hidden_biases
latent_relu_per_nb = np.maximum(latent_per_nb, 0)
# Monte-Carlo integration after first layer
# weighting with pdf
weighted_features_per_nb = np.expand_dims(features_on_neighbors, 2) * \
np.expand_dims(latent_relu_per_nb, 1)
nb_ranges = np.concatenate(([0], nb_ranges), axis=0)
# sum (integration)
weighted_latent_per_sample = \
np.zeros([num_samples, num_features[0], hidden_size])
for i in range(num_samples):
weighted_latent_per_sample[i] = \
np.sum(weighted_features_per_nb[nb_ranges[i]:nb_ranges[i + 1]],
axis=0)
self.assertAllClose(weighted_latent_per_sample_tf,
weighted_latent_per_sample)
def basis_proj_basis_neighs(basis_neighs_in):
return basis_proj(basis_neighs_in, features,
neighborhood) / (max_num_nb)
def basis_proj_features(features_in):
return basis_proj(basis_neighs, features_in,
neighborhood) / (max_num_nb)
def _kernel_offsets(self,
kernel_input,
neighborhood,
features):
""" Method to compute the kernel offsets for deformable KPConv
using a rigid KPConv.
As described in Section 3.2 of [KPConv: Flexible and Deformable Convolution
for Point Clouds. Thomas et al., 2019](https://arxiv.org/abs/1904.08889).
Note: In the following
`D` is the dimensionality of the point cloud (=3)
`M` is the number of neighbor pairs
'C1`is the number of input features
`N1' is the number of input points
`N2' is the number of ouput points
`K` is the number of kernel points
Args:
kernel_inputs: A `float` `Tensor` of shape `[M, D]`, the input to the
kernel, i.e. the distances between neighbor pairs.
neighborhood: A `Neighborhood` instance.
features: A `float` `Tensor` of shape `[N1, C1]`, the input features.
Returns:
A `float` `Tensor` of shape `[K, M, D]`, the offsets.
"""
# neighbor pairs ids
neighbors = neighborhood._original_neigh_ids
# kernel weights from distances, shape [M, K]
points_diff = tf.expand_dims(kernel_input, 1) - \
tf.expand_dims(self._kernel_points, 0)
points_dist = tf.linalg.norm(points_diff, axis=2)
kernel_weights = self._weighting(points_dist, self._sigma)
# Pad zeros to fullfil requirements of the basis_proj custom op,
# 8, 16, or 32 basis are allowed.
if self._num_kernel_points < 8:
kernel_weights = tf.pad(kernel_weights,
[[0, 0], [0, 8 - self._num_kernel_points]])
elif self._num_kernel_points > 8 and self._num_kernel_points < 16:
kernel_weights = tf.pad(kernel_weights,
[[0, 0], [0, 16 - self._num_kernel_points]])
elif self._num_kernel_points > 16 and self._num_kernel_points < 32:
kernel_weights = tf.pad(kernel_weights,
[[0, 0], [0, 32 - self._num_kernel_points]])
elif self._num_kernel_points > 32 and self._num_kernel_points < 64:
kernel_weights = tf.pad(kernel_weights,
[[0, 0], [0, 64 - self._num_kernel_points]])
# Compute the projection to the samples.
weighted_features = basis_proj(
kernel_weights,
features,
neighborhood)
# remove padding
weighted_features = weighted_features[:, :, 0:self._num_kernel_points]
# Compute convolution - hidden layer to output (linear)
offset_per_center = tf.matmul(
tf.reshape(weighted_features,
[-1, self._num_features_in * self._num_kernel_points]),
self._kernel_offsets_weights)
# save for regularization loss computation
self._offsets = tf.reshape(offset_per_center, [self._num_output_points,
self._num_kernel_points,
self._num_dims])
# project back onto neighbor pairs, shape [M, D*K]
offset_per_nb = tf.gather(offset_per_center, neighbors[:, 1])
# reshape to shape [M, K, self._num_dims]
return tf.reshape(offset_per_nb,
[neighbors.shape[0],
self._num_kernel_points,
self._num_dims])
def _kp_conv(self,
kernel_input,
neighborhood,
features):
""" Method to compute a kernel point convolution using linear interpolation
of the kernel weights.
Note: In the following
`D` is the dimensionality of the points cloud (=3)
`M` is the number of neighbor pairs
'C1`is the number of input features
`C2` is the number of output features
`N1' is the number of input points
`N2' is the number of ouput points
Args:
kernel_inputs: A `float` `Tensor` of shape `[M, D]`, the input to the
kernel, i.e. the distances between neighbor pairs.
neighborhood: A `Neighborhood` instance.
features: A `float` `Tensor` of shape `[N1, C1]`, the input features.
Returns:
A `float` `Tensor` of shape `[N2, C2]`, the output features.
"""
# neighbor pairs ids
neighbors = neighborhood._original_neigh_ids
# kernel weights from distances, shape [M, K]
kernel_offsets = self._get_offsets(kernel_input, neighborhood, features)
points_diff = tf.expand_dims(kernel_input, 1) - \
(tf.expand_dims(self._kernel_points, 0) + kernel_offsets)
points_dist = tf.linalg.norm(points_diff, axis=2)
kernel_weights = self._weighting(points_dist, self._sigma)
# Pad zeros to fullfil requirements of the basis_proj custom op,
# 8, 16, 32, or 64 basis are allowed.
if self._num_kernel_points < 8:
kernel_weights = tf.pad(kernel_weights,
[[0, 0], [0, 8 - self._num_kernel_points]])
elif self._num_kernel_points > 8 and self._num_kernel_points < 16:
kernel_weights = tf.pad(kernel_weights,
[[0, 0], [0, 16 - self._num_kernel_points]])
elif self._num_kernel_points > 16 and self._num_kernel_points < 32:
kernel_weights = tf.pad(kernel_weights,
[[0, 0], [0, 32 - self._num_kernel_points]])
elif self._num_kernel_points > 32 and self._num_kernel_points < 64:
kernel_weights = tf.pad(kernel_weights,
[[0, 0], [0, 64 - self._num_kernel_points]])
# save values for regularization loss computation
self._cur_point_dist = points_dist
self._cur_neighbors = neighbors
# Compute the projection to the samples.
weighted_features = basis_proj(
kernel_weights,
features,
neighborhood)
# remove padding
weighted_features = weighted_features[:, :, 0:self._num_kernel_points]
#Compute convolution - hidden layer to output (linear)
convolution_result = tf.matmul(
tf.reshape(weighted_features,
[-1, self._num_features_in * self._num_kernel_points]),
self._weights)
return convolution_result