def feature_decoding_layer(xyz1, xyz2, points1, points2, radius, sigma, K, mlp, is_training, bn_decay, weight_decay, scope, bn=True, use_xyz=True, boundary_label=None): ''' Input: xyz1: (batch_size, ndataset1, 3) TF tensor xyz2: (batch_size, ndataset2, 3) TF tensor, sparser than xyz1 points1: (batch_size, ndataset1, nchannel1) TF tensor points2: (batch_size, ndataset2, nchannel2) TF tensor sigma: float32 -- KDE bandwidth K: int32 -- how many points in each local region mlp: list of int32 -- output size for MLP on each point Return: new_points: (batch_size, ndataset1, mlp[-1]) TF tensor ''' with tf.variable_scope(scope) as sc: dist, idx = three_nn(xyz1, xyz2) dist = tf.maximum(dist, 1e-10) norm = tf.reduce_sum((1.0 / dist), axis=2, keepdims=True) norm = tf.tile(norm, [1, 1, 3]) weight = (1.0 / dist) / norm interpolated_points = three_interpolate(points2, idx, weight) #setup for deConv if boundary_label is None: pass else: tmp_boundary_label = tf.tile(tf.expand_dims(boundary_label, [-1]), [1, 1, interpolated_points.shape[2]]) interpolated_points = interpolated_points * tmp_boundary_label grouped_xyz, grouped_feature, idx, grouped_boundary_label = pointconv_util.grouping( interpolated_points, K, xyz1, xyz1, use_xyz=use_xyz, boundary_label=boundary_label) weight = weight_net_hidden(grouped_xyz, [32], scope='decode_weight_net', is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay) new_points = tf.transpose(grouped_feature, [0, 1, 3, 2]) new_points = tf.matmul(new_points, weight) new_points = tf_util.conv2d(new_points, mlp[0], [1, new_points.get_shape()[2].value], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='decode_after_conv', bn_decay=bn_decay, weight_decay=weight_decay) if points1 is not None: new_points1 = tf.concat(axis=-1, values=[ new_points, tf.expand_dims(points1, axis=2) ]) # B,ndataset1,nchannel1+nchannel2 else: new_points1 = new_points for i, num_out_channel in enumerate(mlp): if i != 0: new_points1 = tf_util.conv2d(new_points1, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv_%d' % (i), bn_decay=bn_decay, weight_decay=weight_decay) new_points1 = tf.squeeze(new_points1, [2]) # B,ndataset1,mlp[-1] return new_points1
def feature_encoding_layer_extra(xyz, sample_xyz, sample_boundary, feature, npoint, radius, sigma, K, mlp, local_num_out_channel, is_training, bn_decay, weight_decay, scope, bn=True, use_xyz=True, boundary_label=None): ''' Input: xyz: (batch_size, ndataset, 3) TF tensor feature: (batch_size, ndataset, channel) TF tensor npoint: int32 -- #points sampled in farthest point sampling sigma: float32 -- KDE bandwidth K: int32 -- how many points in each local region mlp: list of int32 -- output size for MLP on each point use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features Return: new_xyz: (batch_size, npoint, 3) TF tensor new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor ''' batch_size = xyz.shape[0] with tf.variable_scope(scope) as sc: num_points = xyz.get_shape()[1] if num_points == npoint: new_xyz = xyz sub_boundary_label = boundary_label else: if boundary_label is None: #new_xyz = pointconv_util.sampling(npoint, xyz) new_xyz = sample_xyz sub_boundary_label = None else: #new_xyz, sub_boundary_label = pointconv_util.sampling_with_boundary_label(npoint, xyz, boundary_label) new_xyz = sample_xyz sub_boundary_label = sample_boundary if boundary_label is None: pass else: tmp_boundary_label = tf.tile(tf.expand_dims(boundary_label, [-1]), [1, 1, feature.shape[2]]) feature = feature * tmp_boundary_label if local_num_out_channel is not None: xyz_4n, feature_4n, idx_4n, _ = pointconv_util.grouping( feature, 4, xyz, xyz) xyz_3n = xyz_4n[:, :, 1:4, :] local_feature = tf_util.local_feature(xyz_3n, local_num_out_channel, bn, is_training, 0, bn_decay, weight_decay) feature = tf.concat([feature, local_feature], axis=2) grouped_xyz, grouped_feature, idx, grouped_boundary_label = pointconv_util.grouping( feature, K, xyz, new_xyz, boundary_label=boundary_label) for i, num_out_channel in enumerate(mlp): if i != len(mlp) - 1: grouped_feature = tf_util.conv2d(grouped_feature, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv%d' % (i), bn_decay=bn_decay, weight_decay=weight_decay) if grouped_boundary_label is not None: tmp_grouped_boundary_label = tf.tile( tf.expand_dims(grouped_boundary_label, [-1]), [1, 1, 1, grouped_feature.shape[3]]) grouped_feature = grouped_feature * tmp_grouped_boundary_label weight = weight_net_hidden(grouped_xyz, [32], scope='weight_net', is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay) new_points = tf.transpose(grouped_feature, [0, 1, 3, 2]) new_points = tf.matmul(new_points, weight) new_points = tf_util.conv2d(new_points, mlp[-1], [1, new_points.get_shape()[2].value], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='after_conv', bn_decay=bn_decay, weight_decay=weight_decay) new_points = tf.squeeze(new_points, [2]) # (batch_size, npoints, mlp2[-1]) return new_xyz, new_points, sub_boundary_label
def feature_encoding_layer_msg(xyz, feature, npoint, radius, sigma, K, mlp, is_training, bn_decay, weight_decay, scope, bn=True, use_xyz=True): ''' Input: xyz: (batch_size, ndataset, 3) TF tensor feature: (batch_size, ndataset, channel) TF tensor npoint: int32 -- #points sampled in farthest point sampling sigma: float32 -- KDE bandwidth radius_list: list of radius for local region K_list: int32 -- list of how many points in each local region mlp: list of int32 -- output size for MLP on each point use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features Return: new_xyz: (batch_size, npoint, 3) TF tensor new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor ''' radius_list = radius K_list = K sigma_list = sigma num_points = xyz.get_shape()[1] if num_points == npoint: new_xyz = xyz else: new_xyz = pointconv_util.sampling(npoint, xyz) new_points_list = [] for i in range(len(radius_list)): with tf.variable_scope("%s_%d" % (scope, i)) as sc: radius = radius_list[i] K = K_list[i] sigma = sigma_list[i] grouped_xyz, grouped_feature, idx = pointconv_util.grouping( feature, K, xyz, new_xyz) density = pointconv_util.kernel_density_estimation_ball( xyz, radius, sigma) inverse_density = tf.div(1.0, density) grouped_density = tf.gather_nd( inverse_density, idx) # (batch_size, npoint, nsample, 1) #grouped_density = tf_grouping.group_point(inverse_density, idx) inverse_max_density = tf.reduce_max(grouped_density, axis=2, keep_dims=True) density_scale = tf.div(grouped_density, inverse_max_density) #density_scale = tf_grouping.group_point(density, idx) for j, num_out_channel in enumerate(mlp): if j != len(mlp) - 1: grouped_feature = tf_util.conv2d(grouped_feature, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv%d' % j, bn_decay=bn_decay, weight_decay=weight_decay) weight = weight_net_hidden(grouped_xyz, [32], scope='weight_net', is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay) density_scale = nonlinear_transform(density_scale, [16, 1], scope='density_net', is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay) new_points = tf.multiply(grouped_feature, density_scale) new_points = tf.transpose(new_points, [0, 1, 3, 2]) new_points = tf.matmul(new_points, weight) new_points = tf_util.conv2d(new_points, mlp[-1], [1, new_points.get_shape()[2].value], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='after_conv', bn_decay=bn_decay, weight_decay=weight_decay) new_points = tf.squeeze(new_points, [2]) # (batch_size, npoints, mlp2[-1]) new_points_list.append(new_points) new_points = tf.concat(new_points_list, axis=-1) return new_xyz, new_points
def feature_decoding_layer_depthwise(xyz1, xyz2, points1, points2, radius, sigma, K, mlp, is_training, bn_decay, weight_decay, scope, bn=True, use_xyz=True): ''' Input: depthwise version of pointconv xyz1: (batch_size, ndataset1, 3) TF tensor xyz2: (batch_size, ndataset2, 3) TF tensor, sparser than xyz1 points1: (batch_size, ndataset1, nchannel1) TF tensor points2: (batch_size, ndataset2, nchannel2) TF tensor sigma: float32 -- KDE bandwidth K: int32 -- how many points in each local region mlp: list of int32 -- output size for MLP on each point Return: new_points: (batch_size, ndataset1, mlp[-1]) TF tensor ''' with tf.variable_scope(scope) as sc: dist, idx = three_nn(xyz1, xyz2) dist = tf.maximum(dist, 1e-10) norm = tf.reduce_sum((1.0 / dist), axis=2, keep_dims=True) norm = tf.tile(norm, [1, 1, 3]) weight = (1.0 / dist) / norm interpolated_points = three_interpolate(points2, idx, weight) #setup for deConv grouped_xyz, grouped_feature, idx = pointconv_util.grouping( interpolated_points, K, xyz1, xyz1, use_xyz=use_xyz) density = pointconv_util.kernel_density_estimation_ball( xyz1, radius, sigma) inverse_density = tf.div(1.0, density) grouped_density = tf.gather_nd(inverse_density, idx) # (batch_size, npoint, nsample, 1) #grouped_density = tf_grouping.group_point(inverse_density, idx) inverse_max_density = tf.reduce_max(grouped_density, axis=2, keep_dims=True) density_scale = tf.div(grouped_density, inverse_max_density) #density_scale = tf_grouping.group_point(density, idx) weight = weight_net(grouped_xyz, [32, grouped_feature.get_shape()[3].value], scope='decode_weight_net', is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay) density_scale = nonlinear_transform(density_scale, [16, 1], scope='decode_density_net', is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay) new_points = tf.multiply(grouped_feature, density_scale) new_points = tf.multiply(grouped_feature, weight) new_points = tf_util.reduce_sum2d_conv(new_points, axis=2, scope='fp_sumpool', bn=True, bn_decay=bn_decay, is_training=is_training, keep_dims=False) if points1 is not None: new_points1 = tf.concat( axis=-1, values=[new_points, points1]) # B,ndataset1,nchannel1+nchannel2 else: new_points1 = new_points new_points1 = tf.expand_dims(new_points1, 2) for i, num_out_channel in enumerate(mlp): new_points1 = tf_util.conv2d(new_points1, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv_%d' % (i), bn_decay=bn_decay, weight_decay=weight_decay) new_points1 = tf.squeeze(new_points1, [2]) # B,ndataset1,mlp[-1] return new_points1
def get_boundary_model_loss(labels_pl, point_cloud, is_training, num_class, sigma, bn_decay=None, weight_decay=None): # generate boundary 0 if boundary else 1 num_neighbor = 16 ratio = 0.6 g_xyz, g_labels, g_idx, _ = pointconv_util.grouping(tf.cast( tf.expand_dims(labels_pl, [-1]), tf.float32), num_neighbor, point_cloud[:, :, :3], point_cloud[:, :, :3], use_xyz=False) g_labels = tf.squeeze(g_labels, [-1]) self_labels = tf.cast( tf.tile(tf.expand_dims(labels_pl, [-1]), [1, 1, num_neighbor]), tf.float32) same_label_num = tf.reduce_sum(tf.cast(tf.equal(g_labels, self_labels), tf.float32), axis=2) boundary_points = tf.cast( tf.greater_equal(same_label_num, (num_neighbor * ratio)), tf.int32) boundary_points = tf.cast(boundary_points, tf.float32) target_boundary_label = tf.stop_gradient(boundary_points) # ======================================== batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud[:, :, :3] l0_points = point_cloud[:, :, 3:6] #difference extraction _, tmp_grouped_feature, _, _ = pointconv_util.grouping(l0_points, 8, l0_xyz, l0_xyz, boundary_label=None, use_xyz=False) mean, var = tf.nn.moments(tmp_grouped_feature, axes=2) print(var.shape) # tmp_average_feature = tf.reduce_mean(tmp_grouped_feature, axis=2) l0_points = var #Feature encoding layer # l1_xyz, l1_points, _ = feature_encoding_layer(l0_xyz, l0_points, npoint=2048, radius = 0.1, sigma = sigma, K=8, mlp=[32,32,64], is_training=is_training, bn_decay=bn_decay, weight_decay = weight_decay, scope='boundary_layer1') #Feature decoding layer # l0_points = feature_decoding_layer(l0_xyz, l1_xyz, l0_points, l1_points, 0.1, sigma, 8, [128,128,128], is_training, bn_decay, weight_decay, scope='boundary_fa_layer2') # FC layers net = tf_util.conv1d(l0_points, 32, 1, padding='VALID', bn=True, is_training=is_training, scope='boundary_fc1', bn_decay=bn_decay, weight_decay=weight_decay) net = tf_util.conv1d(net, 64, 1, padding='VALID', bn=True, is_training=is_training, scope='boundary_fc2', bn_decay=bn_decay, weight_decay=weight_decay) net = tf_util.conv1d(net, 64, 1, padding='VALID', bn=True, is_training=is_training, scope='boundary_fc3', bn_decay=bn_decay, weight_decay=weight_decay) net = tf_util.conv1d(net, 128, 1, padding='VALID', bn=True, is_training=is_training, scope='boundary_fc4', bn_decay=bn_decay, weight_decay=weight_decay) end_points['feats'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv1d(net, 1, 1, padding='VALID', activation_fn=None, weight_decay=weight_decay, scope='boundary_fc5') net = tf.squeeze(net, [2]) boundary_weight = 2 - 1 * target_boundary_label boundary_loss = tf.losses.sigmoid_cross_entropy(tf.cast( target_boundary_label, tf.int32), net, weights=boundary_weight) return net, boundary_loss
def feature_encoding_layer_extra(xyz, sample_xyz, feature, npoint, radius, sigma, K, mlp, akc_channel, is_training, bn_decay, weight_decay, scope, bn=True, use_xyz=True): ''' Input: xyz: (batch_size, ndataset, 3) TF tensor feature: (batch_size, ndataset, channel) TF tensor npoint: int32 -- #points sampled in farthest point sampling sigma: float32 -- KDE bandwidth K: int32 -- how many points in each local region mlp: list of int32 -- output size for MLP on each point use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features Return: new_xyz: (batch_size, npoint, 3) TF tensor new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor ''' with tf.variable_scope(scope) as sc: num_points = xyz.get_shape()[1] if num_points == npoint: new_xyz = xyz else: new_xyz = sample_xyz if akc_channel is not None: xyz_4n, feature_4n, idx_4n = pointconv_util.grouping( feature, 4, xyz, xyz) xyz_3n = xyz_4n[:, :, 1:4, :] akc_feature = tf_util.akc(xyz_3n, akc_channel, bn, is_training, 0, bn_decay, weight_decay) feature = tf.concat([feature, akc_feature], axis=2) grouped_xyz, grouped_feature, idx = pointconv_util.grouping( feature, K, xyz, new_xyz) density = pointconv_util.kernel_density_estimation_ball( xyz, radius, sigma) inverse_density = tf.div(1.0, density) grouped_density = tf.gather_nd(inverse_density, idx) # (batch_size, npoint, nsample, 1) #grouped_density = tf_grouping.group_point(inverse_density, idx) inverse_max_density = tf.reduce_max(grouped_density, axis=2, keepdims=True) density_scale = tf.div(grouped_density, inverse_max_density) #density_scale = tf_grouping.group_point(density, idx) for i, num_out_channel in enumerate(mlp): if i != len(mlp) - 1: grouped_feature = tf_util.conv2d(grouped_feature, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv%d' % (i), bn_decay=bn_decay, weight_decay=weight_decay) weight = weight_net_hidden(grouped_xyz, [32], scope='weight_net', is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay) density_scale = nonlinear_transform(density_scale, [16, 1], scope='density_net', is_training=is_training, bn_decay=bn_decay, weight_decay=weight_decay) new_points = tf.multiply(grouped_feature, density_scale) new_points = tf.transpose(new_points, [0, 1, 3, 2]) new_points = tf.matmul(new_points, weight) new_points = tf_util.conv2d(new_points, mlp[-1], [1, new_points.get_shape()[2].value], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='after_conv', bn_decay=bn_decay, weight_decay=weight_decay) new_points = tf.squeeze(new_points, [2]) # (batch_size, npoints, mlp2[-1]) return new_xyz, new_points