def forward(self, x, lidar_mask, bilateral_filters): mc = self.mc size_z, size_a = mc.LCN_HEIGHT, mc.LCN_WIDTH # initialize compatibilty matrices compat_kernel_init = torch.from_numpy( np.reshape( np.ones((mc.NUM_CLASS, mc.NUM_CLASS), dtype="float32") - np.identity(mc.NUM_CLASS, dtype="float32"), [mc.NUM_CLASS, mc.NUM_CLASS, 1, 1])) bi_compat_kernel = compat_kernel_init * mc.BI_FILTER_COEF bi_compat_kernel.requires_grad_() angular_compat_kernel = compat_kernel_init * mc.ANG_FILTER_COEF angular_compat_kernel.requires_grad_() condensing_kernel = torch.from_numpy( util.condensing_matrix(mc.NUM_CLASS, size_z, size_a)).float() angular_filters = torch.from_numpy( util.angular_filter_kernel(mc.NUM_CLASS, size_z, size_a, mc.ANG_THETA_A**2)).float() bi_angular_filters = torch.from_numpy( util.angular_filter_kernel(mc.NUM_CLASS, size_z, size_a, mc.BILATERAL_THETA_A**2)).float() # GPU bi_compat_kernel, angular_compat_kernel, condensing_kernel, angular_filters, bi_angular_filters = \ bi_compat_kernel.to(device), angular_compat_kernel.to(device), condensing_kernel.to(device), angular_filters.to(device), bi_angular_filters.to(device) for it in range(mc.RCRF_ITER): unary = F.softmax(x, dim=-1) ang_output, bi_output = self.locally_connected_layer( unary, lidar_mask, bilateral_filters, angular_filters, bi_angular_filters, condensing_kernel) ang_output = F.conv2d(ang_output, weight=angular_compat_kernel, stride=self.stride, padding=0) bi_output = F.conv2d(bi_output, weight=bi_compat_kernel, stride=self.stride, padding=0) pairwise = torch.add(ang_output, bi_output) outputs = torch.add(unary, pairwise) x = outputs return outputs
def _recurrent_crf_layer(self, layer_name, inputs, bilateral_filters, sizes=[3, 5], num_iterations=1, padding='SAME'): """Recurrent conditional random field layer. Iterative meanfield inference is implemented as a reccurent neural network. Args: layer_name: layer name inputs: input tensor with shape [batch_size, zenith, azimuth, num_class]. bilateral_filters: filter weight with shape [batch_size, zenith, azimuth, sizes[0]*size[1]-1]. sizes: size of the local region to be filtered. num_iterations: number of meanfield inferences. padding: padding strategy Returns: outputs: tensor with shape [batch_size, zenith, azimuth, num_class]. """ assert num_iterations >= 1, 'number of iterations should >= 1' mc = self.mc with tf.variable_scope(layer_name) as scope: # initialize compatibilty matrices compat_kernel_init = tf.constant(np.reshape( np.ones( (mc.NUM_CLASS, mc.NUM_CLASS)) - np.identity(mc.NUM_CLASS), [1, 1, mc.NUM_CLASS, mc.NUM_CLASS]), dtype=tf.float32) bi_compat_kernel = _variable_on_device( name='bilateral_compatibility_matrix', shape=[1, 1, mc.NUM_CLASS, mc.NUM_CLASS], initializer=compat_kernel_init * mc.BI_FILTER_COEF, trainable=True) self._activation_summary(bi_compat_kernel, 'bilateral_compat_mat') angular_compat_kernel = _variable_on_device( name='angular_compatibility_matrix', shape=[1, 1, mc.NUM_CLASS, mc.NUM_CLASS], initializer=compat_kernel_init * mc.ANG_FILTER_COEF, trainable=True) self._activation_summary(angular_compat_kernel, 'angular_compat_mat') self.model_params += [bi_compat_kernel, angular_compat_kernel] condensing_kernel = tf.constant(util.condensing_matrix( sizes[0], sizes[1], mc.NUM_CLASS), dtype=tf.float32, name='condensing_kernel') angular_filters = tf.constant(util.angular_filter_kernel( sizes[0], sizes[1], mc.NUM_CLASS, mc.ANG_THETA_A**2), dtype=tf.float32, name='angular_kernel') bi_angular_filters = tf.constant(util.angular_filter_kernel( sizes[0], sizes[1], mc.NUM_CLASS, mc.BILATERAL_THETA_A**2), dtype=tf.float32, name='bi_angular_kernel') for it in range(num_iterations): unary = tf.nn.softmax(inputs, dim=-1, name='unary_term_at_iter_{}'.format(it)) ang_output, bi_output = self._locally_connected_layer( 'message_passing_iter_{}'.format(it), unary, bilateral_filters, angular_filters, bi_angular_filters, condensing_kernel, sizes=sizes, padding=padding) # 1x1 convolution as compatibility transform ang_output = tf.nn.conv2d( ang_output, angular_compat_kernel, strides=[1, 1, 1, 1], padding='SAME', name='angular_compatibility_transformation') self._activation_summary(ang_output, 'ang_transfer_iter_{}'.format(it)) bi_output = tf.nn.conv2d( bi_output, bi_compat_kernel, strides=[1, 1, 1, 1], padding='SAME', name='bilateral_compatibility_transformation') self._activation_summary(bi_output, 'bi_transfer_iter_{}'.format(it)) pairwise = tf.add(ang_output, bi_output, name='pairwise_term_at_iter_{}'.format(it)) outputs = tf.add(unary, pairwise, name='energy_at_iter_{}'.format(it)) inputs = outputs return outputs