def _point_conv(self, kernel_inputs, neighborhood, pdf, features, non_linearity_type='relu'): """ Method to compute a PointConv convolution using a single MLP with one hidden layer as implicit convolution kernel function. Args: kernel_inputs: A `float` `Tensor` of shape `[M, L]`, the input to the kernel MLP. neighborhood: A `Neighborhood` instance. pdf: A `float` `Tensor` of shape `[M]`, the point densities. features: A `float` `Tensor` of shape `[N, C1]`, the input features. non_linearity_type: An `string`, specifies the type of the activation function used inside the kernel MLP. Possible: `'ReLU', 'lReLU', 'ELU'`, defaults to leaky ReLU. (optional) Returns: A `float` `Tensor` of shape `[N,C2]`, the output features. """ # Compute the hidden layer MLP basis_neighs = tf.matmul(kernel_inputs, self._basis_axis_tf) + \ self._basis_bias_tf basis_neighs = \ non_linearity_types[non_linearity_type.lower()](basis_neighs) # Normalize the pdf max_pdf = tf.math.unsorted_segment_max( pdf, neighborhood._original_neigh_ids[:, 1], tf.shape(neighborhood._samples_neigh_ranges)[0]) neigh_max_pdfs = tf.gather(max_pdf, neighborhood._original_neigh_ids[:, 1]) cur_pdf = pdf / neigh_max_pdfs cur_pdf = tf.reshape(cur_pdf, [-1, 1]) # Non-linear transform pdf cur_pdf = tf.nn.relu(tf.matmul(cur_pdf, self._weights_pdf[0]) +\ self._biases_pdf[0]) cur_pdf = tf.matmul(cur_pdf, self._weights_pdf[1]) + self._biases_pdf[1] # Scale features basis_neighs = basis_neighs / cur_pdf # Compute the projection to the samples. weighted_features = basis_proj( basis_neighs, features, neighborhood) #Compute convolution - hidden layer to output (linear) convolution_result = tf.matmul( tf.reshape(weighted_features, [-1, self._num_features_in * self._size_hidden]), self._weights) return convolution_result
def _monte_carlo_conv(self, kernel_inputs, neighborhood, pdf, features, non_linearity_type='leaky_relu'): """ Method to compute a Monte-Carlo integrated convolution using multiple MLPs as implicit convolution kernel functions. Args: kernel_inputs: A `float` `Tensor` of shape `[M, L]`, the input to the kernel MLP. neighborhood: A `Neighborhood` instance. pdf: A `float` `Tensor` of shape `[M]`, the point densities. features: A `float` `Tensor` of shape `[N, C1]`, the input features. non_linearity_type: An `string`, specifies the type of the activation function used inside the kernel MLP. Possible: `'ReLU', 'leaky_ReLU', 'ELU'`, defaults to leaky ReLU. (optional) Returns: A `float` `Tensor` of shape `[N,C2]`, the output features. """ # Compute the hidden layer MLP cur_inputs = tf.tile( tf.reshape(kernel_inputs, [1, -1, self._num_dims]), [self._num_mlps, 1, 1]) for cur_layer_iter in range(len(self._weights_tf)): cur_inputs = tf.matmul(cur_inputs, self._weights_tf[cur_layer_iter]) + \ self._bias_tf[cur_layer_iter] cur_inputs = non_linearity_types[non_linearity_type.lower()]( cur_inputs) cur_inputs = tf.reshape(tf.transpose(cur_inputs, [1, 0, 2]), [-1, self._mlp_size[-1] * self._num_mlps]) \ / tf.reshape(pdf, [-1, 1]) # Compute the projection to the samples. weighted_features = basis_proj(cur_inputs, features, neighborhood) # Reshape features weighted_features = tf.transpose( tf.reshape(weighted_features, [ -1, self._num_features_in, self._num_mlps, self._mlp_size[-1] ]), [2, 0, 1, 3]) #Compute convolution - hidden layer to output (linear) convolution_result = tf.matmul( tf.reshape(weighted_features, [ self._num_mlps, -1, self._num_features_in * self._mlp_size[-1] ]), self._final_weights_tf) return tf.reshape(tf.transpose(convolution_result, [1, 0, 2]), [-1, self._num_features_out])
def test_basis_proj(self, num_points, num_samples, num_features, batch_size, radius, hidden_size, dimension): cell_sizes = np.float32(np.repeat(radius, dimension)) points, batch_ids = utils._create_random_point_cloud_segmented( batch_size, num_points, dimension=dimension) features = np.random.rand(num_points, num_features[0]) point_cloud = PointCloud(points, batch_ids) point_samples, batch_ids_samples = \ utils._create_random_point_cloud_segmented( batch_size, num_samples, dimension=dimension) point_cloud_samples = PointCloud(point_samples, batch_ids_samples) grid = Grid(point_cloud, cell_sizes) neighborhood = Neighborhood(grid, cell_sizes, point_cloud_samples) nb_ids = neighborhood._original_neigh_ids # tf conv_layer = MCConv(num_features[0], num_features[1], dimension, 1, [hidden_size]) basis_weights_tf = tf.reshape(conv_layer._weights_tf[0], [dimension, hidden_size]) basis_biases_tf = tf.reshape(conv_layer._bias_tf[0], [1, hidden_size]) neigh_point_coords = points[nb_ids[:, 0]] center_point_coords = point_samples[nb_ids[:, 1]] kernel_input = (neigh_point_coords - center_point_coords) / radius basis_neighs = \ tf.matmul(kernel_input.astype(np.float32), basis_weights_tf) + \ basis_biases_tf basis_neighs = tf.nn.relu(basis_neighs) weighted_latent_per_sample_tf = basis_proj(basis_neighs, features, neighborhood) # numpy neighbor_ids = neighborhood._original_neigh_ids.numpy() nb_ranges = neighborhood._samples_neigh_ranges.numpy() # extract variables hidden_weights = basis_weights_tf.numpy() hidden_biases = basis_biases_tf.numpy() features_on_neighbors = features[neighbor_ids[:, 0]] # compute first layer of kernel MLP point_diff = (points[neighbor_ids[:, 0]] -\ point_samples[neighbor_ids[:, 1]])\ / np.expand_dims(cell_sizes, 0) latent_per_nb = np.dot(point_diff, hidden_weights) + hidden_biases latent_relu_per_nb = np.maximum(latent_per_nb, 0) # Monte-Carlo integration after first layer # weighting with pdf weighted_features_per_nb = np.expand_dims(features_on_neighbors, 2) * \ np.expand_dims(latent_relu_per_nb, 1) nb_ranges = np.concatenate(([0], nb_ranges), axis=0) # sum (integration) weighted_latent_per_sample = \ np.zeros([num_samples, num_features[0], hidden_size]) for i in range(num_samples): weighted_latent_per_sample[i] = \ np.sum(weighted_features_per_nb[nb_ranges[i]:nb_ranges[i + 1]], axis=0) self.assertAllClose(weighted_latent_per_sample_tf, weighted_latent_per_sample)
def basis_proj_basis_neighs(basis_neighs_in): return basis_proj(basis_neighs_in, features, neighborhood) / (max_num_nb)
def basis_proj_features(features_in): return basis_proj(basis_neighs, features_in, neighborhood) / (max_num_nb)
def _kernel_offsets(self, kernel_input, neighborhood, features): """ Method to compute the kernel offsets for deformable KPConv using a rigid KPConv. As described in Section 3.2 of [KPConv: Flexible and Deformable Convolution for Point Clouds. Thomas et al., 2019](https://arxiv.org/abs/1904.08889). Note: In the following `D` is the dimensionality of the point cloud (=3) `M` is the number of neighbor pairs 'C1`is the number of input features `N1' is the number of input points `N2' is the number of ouput points `K` is the number of kernel points Args: kernel_inputs: A `float` `Tensor` of shape `[M, D]`, the input to the kernel, i.e. the distances between neighbor pairs. neighborhood: A `Neighborhood` instance. features: A `float` `Tensor` of shape `[N1, C1]`, the input features. Returns: A `float` `Tensor` of shape `[K, M, D]`, the offsets. """ # neighbor pairs ids neighbors = neighborhood._original_neigh_ids # kernel weights from distances, shape [M, K] points_diff = tf.expand_dims(kernel_input, 1) - \ tf.expand_dims(self._kernel_points, 0) points_dist = tf.linalg.norm(points_diff, axis=2) kernel_weights = self._weighting(points_dist, self._sigma) # Pad zeros to fullfil requirements of the basis_proj custom op, # 8, 16, or 32 basis are allowed. if self._num_kernel_points < 8: kernel_weights = tf.pad(kernel_weights, [[0, 0], [0, 8 - self._num_kernel_points]]) elif self._num_kernel_points > 8 and self._num_kernel_points < 16: kernel_weights = tf.pad(kernel_weights, [[0, 0], [0, 16 - self._num_kernel_points]]) elif self._num_kernel_points > 16 and self._num_kernel_points < 32: kernel_weights = tf.pad(kernel_weights, [[0, 0], [0, 32 - self._num_kernel_points]]) elif self._num_kernel_points > 32 and self._num_kernel_points < 64: kernel_weights = tf.pad(kernel_weights, [[0, 0], [0, 64 - self._num_kernel_points]]) # Compute the projection to the samples. weighted_features = basis_proj( kernel_weights, features, neighborhood) # remove padding weighted_features = weighted_features[:, :, 0:self._num_kernel_points] # Compute convolution - hidden layer to output (linear) offset_per_center = tf.matmul( tf.reshape(weighted_features, [-1, self._num_features_in * self._num_kernel_points]), self._kernel_offsets_weights) # save for regularization loss computation self._offsets = tf.reshape(offset_per_center, [self._num_output_points, self._num_kernel_points, self._num_dims]) # project back onto neighbor pairs, shape [M, D*K] offset_per_nb = tf.gather(offset_per_center, neighbors[:, 1]) # reshape to shape [M, K, self._num_dims] return tf.reshape(offset_per_nb, [neighbors.shape[0], self._num_kernel_points, self._num_dims])
def _kp_conv(self, kernel_input, neighborhood, features): """ Method to compute a kernel point convolution using linear interpolation of the kernel weights. Note: In the following `D` is the dimensionality of the points cloud (=3) `M` is the number of neighbor pairs 'C1`is the number of input features `C2` is the number of output features `N1' is the number of input points `N2' is the number of ouput points Args: kernel_inputs: A `float` `Tensor` of shape `[M, D]`, the input to the kernel, i.e. the distances between neighbor pairs. neighborhood: A `Neighborhood` instance. features: A `float` `Tensor` of shape `[N1, C1]`, the input features. Returns: A `float` `Tensor` of shape `[N2, C2]`, the output features. """ # neighbor pairs ids neighbors = neighborhood._original_neigh_ids # kernel weights from distances, shape [M, K] kernel_offsets = self._get_offsets(kernel_input, neighborhood, features) points_diff = tf.expand_dims(kernel_input, 1) - \ (tf.expand_dims(self._kernel_points, 0) + kernel_offsets) points_dist = tf.linalg.norm(points_diff, axis=2) kernel_weights = self._weighting(points_dist, self._sigma) # Pad zeros to fullfil requirements of the basis_proj custom op, # 8, 16, 32, or 64 basis are allowed. if self._num_kernel_points < 8: kernel_weights = tf.pad(kernel_weights, [[0, 0], [0, 8 - self._num_kernel_points]]) elif self._num_kernel_points > 8 and self._num_kernel_points < 16: kernel_weights = tf.pad(kernel_weights, [[0, 0], [0, 16 - self._num_kernel_points]]) elif self._num_kernel_points > 16 and self._num_kernel_points < 32: kernel_weights = tf.pad(kernel_weights, [[0, 0], [0, 32 - self._num_kernel_points]]) elif self._num_kernel_points > 32 and self._num_kernel_points < 64: kernel_weights = tf.pad(kernel_weights, [[0, 0], [0, 64 - self._num_kernel_points]]) # save values for regularization loss computation self._cur_point_dist = points_dist self._cur_neighbors = neighbors # Compute the projection to the samples. weighted_features = basis_proj( kernel_weights, features, neighborhood) # remove padding weighted_features = weighted_features[:, :, 0:self._num_kernel_points] #Compute convolution - hidden layer to output (linear) convolution_result = tf.matmul( tf.reshape(weighted_features, [-1, self._num_features_in * self._num_kernel_points]), self._weights) return convolution_result