def forward(self, x, lidar_mask, bilateral_filters):
mc = self.mc
size_z, size_a = mc.LCN_HEIGHT, mc.LCN_WIDTH
# initialize compatibilty matrices
compat_kernel_init = torch.from_numpy(
np.reshape(
np.ones((mc.NUM_CLASS, mc.NUM_CLASS), dtype="float32") -
np.identity(mc.NUM_CLASS, dtype="float32"),
[mc.NUM_CLASS, mc.NUM_CLASS, 1, 1]))
bi_compat_kernel = compat_kernel_init * mc.BI_FILTER_COEF
bi_compat_kernel.requires_grad_()
angular_compat_kernel = compat_kernel_init * mc.ANG_FILTER_COEF
angular_compat_kernel.requires_grad_()
condensing_kernel = torch.from_numpy(
util.condensing_matrix(mc.NUM_CLASS, size_z, size_a)).float()
angular_filters = torch.from_numpy(
util.angular_filter_kernel(mc.NUM_CLASS, size_z, size_a,
mc.ANG_THETA_A**2)).float()
bi_angular_filters = torch.from_numpy(
util.angular_filter_kernel(mc.NUM_CLASS, size_z, size_a,
mc.BILATERAL_THETA_A**2)).float()
# GPU
bi_compat_kernel, angular_compat_kernel, condensing_kernel, angular_filters, bi_angular_filters = \
bi_compat_kernel.to(device), angular_compat_kernel.to(device), condensing_kernel.to(device), angular_filters.to(device), bi_angular_filters.to(device)
for it in range(mc.RCRF_ITER):
unary = F.softmax(x, dim=-1)
ang_output, bi_output = self.locally_connected_layer(
unary, lidar_mask, bilateral_filters, angular_filters,
bi_angular_filters, condensing_kernel)
ang_output = F.conv2d(ang_output,
weight=angular_compat_kernel,
stride=self.stride,
padding=0)
bi_output = F.conv2d(bi_output,
weight=bi_compat_kernel,
stride=self.stride,
padding=0)
pairwise = torch.add(ang_output, bi_output)
outputs = torch.add(unary, pairwise)
x = outputs
return outputs
def _recurrent_crf_layer(self,
layer_name,
inputs,
bilateral_filters,
sizes=[3, 5],
num_iterations=1,
padding='SAME'):
"""Recurrent conditional random field layer. Iterative meanfield inference is
implemented as a reccurent neural network.
Args:
layer_name: layer name
inputs: input tensor with shape [batch_size, zenith, azimuth, num_class].
bilateral_filters: filter weight with shape
[batch_size, zenith, azimuth, sizes[0]*size[1]-1].
sizes: size of the local region to be filtered.
num_iterations: number of meanfield inferences.
padding: padding strategy
Returns:
outputs: tensor with shape [batch_size, zenith, azimuth, num_class].
"""
assert num_iterations >= 1, 'number of iterations should >= 1'
mc = self.mc
with tf.variable_scope(layer_name) as scope:
# initialize compatibilty matrices
compat_kernel_init = tf.constant(np.reshape(
np.ones(
(mc.NUM_CLASS, mc.NUM_CLASS)) - np.identity(mc.NUM_CLASS),
[1, 1, mc.NUM_CLASS, mc.NUM_CLASS]),
dtype=tf.float32)
bi_compat_kernel = _variable_on_device(
name='bilateral_compatibility_matrix',
shape=[1, 1, mc.NUM_CLASS, mc.NUM_CLASS],
initializer=compat_kernel_init * mc.BI_FILTER_COEF,
trainable=True)
self._activation_summary(bi_compat_kernel, 'bilateral_compat_mat')
angular_compat_kernel = _variable_on_device(
name='angular_compatibility_matrix',
shape=[1, 1, mc.NUM_CLASS, mc.NUM_CLASS],
initializer=compat_kernel_init * mc.ANG_FILTER_COEF,
trainable=True)
self._activation_summary(angular_compat_kernel,
'angular_compat_mat')
self.model_params += [bi_compat_kernel, angular_compat_kernel]
condensing_kernel = tf.constant(util.condensing_matrix(
sizes[0], sizes[1], mc.NUM_CLASS),
dtype=tf.float32,
name='condensing_kernel')
angular_filters = tf.constant(util.angular_filter_kernel(
sizes[0], sizes[1], mc.NUM_CLASS, mc.ANG_THETA_A**2),
dtype=tf.float32,
name='angular_kernel')
bi_angular_filters = tf.constant(util.angular_filter_kernel(
sizes[0], sizes[1], mc.NUM_CLASS, mc.BILATERAL_THETA_A**2),
dtype=tf.float32,
name='bi_angular_kernel')
for it in range(num_iterations):
unary = tf.nn.softmax(inputs,
dim=-1,
name='unary_term_at_iter_{}'.format(it))
ang_output, bi_output = self._locally_connected_layer(
'message_passing_iter_{}'.format(it),
unary,
bilateral_filters,
angular_filters,
bi_angular_filters,
condensing_kernel,
sizes=sizes,
padding=padding)
# 1x1 convolution as compatibility transform
ang_output = tf.nn.conv2d(
ang_output,
angular_compat_kernel,
strides=[1, 1, 1, 1],
padding='SAME',
name='angular_compatibility_transformation')
self._activation_summary(ang_output,
'ang_transfer_iter_{}'.format(it))
bi_output = tf.nn.conv2d(
bi_output,
bi_compat_kernel,
strides=[1, 1, 1, 1],
padding='SAME',
name='bilateral_compatibility_transformation')
self._activation_summary(bi_output,
'bi_transfer_iter_{}'.format(it))
pairwise = tf.add(ang_output,
bi_output,
name='pairwise_term_at_iter_{}'.format(it))
outputs = tf.add(unary,
pairwise,
name='energy_at_iter_{}'.format(it))
inputs = outputs
return outputs
