示例#1
0
def trans_net(point_cloud,
              k=20,
              is_training=False,
              bn_decay=None,
              scope='transform_net1'):
    # input (B, N, 3)  return(B, N, 3)
    adj_matrix = tf_util.pairwise_distance(point_cloud)
    nn_idx = tf_util.knn(adj_matrix, k=k)
    edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=k)

    with tf.variable_scope(scope) as sc:
        transform = input_transform_net(edge_feature,
                                        is_training,
                                        bn_decay,
                                        K=3)

    point_cloud_transformed = tf.matmul(point_cloud, transform)

    return point_cloud_transformed
示例#2
0
def get_model_other(point_cloud, is_training, bn_decay=None):
    """
      B: batch size;
      N: number of points,
      C: channels;
      k: number of nearest neighbors
      point_cloud: B*N*C
    """

    end_points = {}
    minSF = tf.reshape(tf.math.argmin(point_cloud[:, :, 0], axis=1), (-1, 1))
    batch_size = point_cloud.get_shape()[0]  # .value

    # # 1. graph for first EdgeConv B N C=6

    adj_matrix = tf_util.pairwise_distance(
        point_cloud[:, :, :para.dim])  # B N C=6 => B*N*N
    # adj_matrix = tf_util.pairwise_distance(point_cloud[:, :, 1:para.dim])  # B N C=6 => B*N*N
    nn_idx = tf_util.knn(adj_matrix, k=20)

    # get the distance to minSF of 1024 points
    allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
    end_points['knn1'] = allSF_dist

    point_cloud = tf.expand_dims(point_cloud[:, :, :para.dim], axis=-2)
    # point_cloud = tf.expand_dims(point_cloud[:, :, 1:para.dim], axis=-2)
    edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=20)
    net = tf_util.conv2d(edge_feature,
                         64, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='dgcnn1',
                         bn_decay=bn_decay)
    net = tf.reduce_max(net, axis=-2, keepdims=True)
    net1 = net

    # # 2. graph for second EdgeConv B N C=64
    adj_matrix = tf_util.pairwise_distance(net)
    nn_idx = tf_util.knn(adj_matrix, k=20)

    # get the distance to minSF of 1024 points
    allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
    end_points['knn2'] = allSF_dist

    # net: B*N*1*6+64=71
    net = tf.concat([point_cloud, net1], axis=-1)

    # edge_feature: B*N*k*142
    edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20)
    net = tf_util.conv2d(edge_feature,
                         64, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='dgcnn2',
                         bn_decay=bn_decay)
    net = tf.reduce_max(net, axis=-2, keepdims=True)
    net2 = net

    # 3. graph for third EdgeConv B N C=64
    adj_matrix = tf_util.pairwise_distance(net)
    nn_idx = tf_util.knn(adj_matrix, k=20)

    # get the distance to minSF of 1024 points
    allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
    end_points['knn3'] = allSF_dist

    # net: B*N*1*6+64+64=134
    net = tf.concat([point_cloud, net1, net2], axis=-1)

    # edge_feature: B*N*k*268
    edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20)
    net = tf_util.conv2d(edge_feature,
                         64, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='dgcnn3',
                         bn_decay=bn_decay)
    net = tf.reduce_max(net, axis=-2, keepdims=True)
    net3 = net

    # 4. graph for fourth EdgeConv B N C=64
    adj_matrix = tf_util.pairwise_distance(net)
    nn_idx = tf_util.knn(adj_matrix, k=20)

    # get the distance to minSF of 1024 points
    allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
    end_points['knn4'] = allSF_dist

    # net: B*N*1*6+64+64+64=198
    net = tf.concat([point_cloud, net1, net2, net3], axis=-1)

    # edge_feature: B*N*k*396
    edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20)
    net = tf_util.conv2d(edge_feature,
                         128, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='dgcnn4',
                         bn_decay=bn_decay)
    net = tf.reduce_max(net, axis=-2, keepdims=True)
    net4 = net

    # input: B*N*1*6+64+64+128+128 = 326  => net: B*N*1*1024
    net = tf_util.conv2d(tf.concat([point_cloud, net1, net2, net3, net4],
                                   axis=-1),
                         1024, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='agg',
                         bn_decay=bn_decay)
    # net: B*1*1*1024
    # SF_features = tf.gather(net, indices=minSF, axis=1, batch_dims=1)
    net = tf.reduce_max(net, axis=1, keepdims=True)
    # SF_all = tf.concat([SF_features,net], axis=-1)
    # net: B*1024
    net = tf.squeeze(net)
    # net: B*2048
    # net = tf.squeeze(SF_all)

    # MLP on global point cloud vector
    net = tf.reshape(net, [batch_size, -1])
    print(net.get_shape())
    end_points['global_feature'] = net

    # Fully connected end_points: classifier
    net = tf_util.fully_connected(net,
                                  512,
                                  bn=True,
                                  is_training=is_training,
                                  scope='fc1',
                                  bn_decay=bn_decay)
    end_points['fc1'] = net
    net = tf_util.dropout(net,
                          keep_prob=0.5,
                          is_training=is_training,
                          scope='dp1')
    net = tf_util.fully_connected(net,
                                  256,
                                  bn=True,
                                  is_training=is_training,
                                  scope='fc2',
                                  bn_decay=bn_decay)
    end_points['fc2'] = net
    net = tf_util.dropout(net,
                          keep_prob=0.5,
                          is_training=is_training,
                          scope='dp2')
    net = tf_util.fully_connected(net,
                                  para.outputClassN,
                                  activation_fn=None,
                                  scope='fc3')
    end_points['fc3'] = net
    return net, end_points
示例#3
0
    def create_encoder(self, point_cloud, npts):

        point_cloud = tf.reshape(point_cloud, (BATCH_SIZE, NUM_POINT, 3))

        adj_matrix = tf_util.pairwise_distance(point_cloud)
        nn_idx = tf_util.knn(adj_matrix, k=self.knn)
        edge_feature = tf_util.get_edge_feature(point_cloud,
                                                nn_idx=nn_idx,
                                                k=self.knn)

        with tf.variable_scope('transform_net1') as sc:
            transform = input_transform_net_dgcnn(edge_feature,
                                                  self.is_training,
                                                  self.bn_decay,
                                                  K=3)

        point_cloud_transformed = tf.matmul(point_cloud, transform)
        adj_matrix = tf_util.pairwise_distance(point_cloud_transformed)
        nn_idx = tf_util.knn(adj_matrix, k=self.knn)
        edge_feature = tf_util.get_edge_feature(point_cloud_transformed,
                                                nn_idx=nn_idx,
                                                k=self.knn)

        net = tf_util.conv2d(edge_feature,
                             64, [1, 1],
                             padding='VALID',
                             stride=[1, 1],
                             bn=True,
                             is_training=self.is_training,
                             scope='dgcnn1',
                             bn_decay=self.bn_decay)
        net = tf.reduce_max(net, axis=-2, keep_dims=True)
        net1 = net

        adj_matrix = tf_util.pairwise_distance(net)
        nn_idx = tf_util.knn(adj_matrix, k=self.knn)
        edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=self.knn)

        net = tf_util.conv2d(edge_feature,
                             64, [1, 1],
                             padding='VALID',
                             stride=[1, 1],
                             bn=True,
                             is_training=self.is_training,
                             scope='dgcnn2',
                             bn_decay=self.bn_decay)
        net = tf.reduce_max(net, axis=-2, keep_dims=True)
        net2 = net

        adj_matrix = tf_util.pairwise_distance(net)
        nn_idx = tf_util.knn(adj_matrix, k=self.knn)
        edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=self.knn)

        net = tf_util.conv2d(edge_feature,
                             64, [1, 1],
                             padding='VALID',
                             stride=[1, 1],
                             bn=True,
                             is_training=self.is_training,
                             scope='dgcnn3',
                             bn_decay=self.bn_decay)
        net = tf.reduce_max(net, axis=-2, keep_dims=True)
        net3 = net

        adj_matrix = tf_util.pairwise_distance(net)
        nn_idx = tf_util.knn(adj_matrix, k=self.knn)
        edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=self.knn)

        net = tf_util.conv2d(edge_feature,
                             128, [1, 1],
                             padding='VALID',
                             stride=[1, 1],
                             bn=True,
                             is_training=self.is_training,
                             scope='dgcnn4',
                             bn_decay=self.bn_decay)
        net = tf.reduce_max(net, axis=-2, keep_dims=True)
        net4 = net

        net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1),
                             1024, [1, 1],
                             padding='VALID',
                             stride=[1, 1],
                             bn=True,
                             is_training=self.is_training,
                             scope='agg',
                             bn_decay=self.bn_decay)

        net = tf.reduce_max(net, axis=1, keep_dims=True)

        features = tf.reshape(net, [BATCH_SIZE, -1])
        return features
示例#4
0
 def get_graph_feature(x, k):
     """Torch: get_graph_feature = TF: adj_matrix + nn_idx + edge_feature"""
     adj_matrix = tf_util.pairwise_distance(x)
     nn_idx = tf_util.knn(adj_matrix, k=k)
     x = tf_util.get_edge_feature(x, nn_idx=nn_idx, k=k)
     return x
def get_model_other(point_cloud, is_training, bn_decay=None):
    """ Classification DGCNN, input is BxNxC, output BxCls
        B batch size
        N number of points per pointcloud
        C input channel: eg. x,y,z,SF,distance, minSF...
        Cls output class number
    """
    batch_size = point_cloud.get_shape()[0]  # .value
    end_points = {}

    # get MinSF index
    minSF = tf.reshape(tf.math.argmin(point_cloud[:, :, 0], axis=1), (-1, 1))

    # # 1. graph for transform net with only x,y,z
    adj_matrix = tf_util.pairwise_distance(point_cloud[:, :, 1:4])  # B N C=3 => B N N
    nn_idx = tf_util.knn(adj_matrix, k=para.k)
    edge_feature = tf_util.get_edge_feature(point_cloud[:, :, 1:4], nn_idx=nn_idx, k=para.k)
    with tf.compat.v1.variable_scope('transform_net1') as sc:
        transform = input_transform_net_dgcnn(edge_feature, is_training, bn_decay, K=3)
    point_cloud_transform = tf.matmul(point_cloud[:, :, 1:4], transform)

    # get the distance to minSF of 1024 points
    allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)  # B N 1
    end_points['knn1'] = allSF_dist

    point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0], axis=-1)
    # # 2. graph for first EdgeConv with transform(x,y,z), SF, distance, minSF
    point_cloud_all = tf.concat(axis=2, values=[point_cloud_SF,
                                                point_cloud_transform,
                                                point_cloud[:, :, 4:para.dim]])

    adj_matrix = tf_util.pairwise_distance(point_cloud_all)  # B N C=6
    nn_idx = tf_util.knn(adj_matrix, k=para.k)
    edge_feature = tf_util.get_edge_feature(point_cloud_all, nn_idx=nn_idx, k=para.k)
    net = tf_util.conv2d(edge_feature, 64, [1, 1],
                         padding='VALID', stride=[1, 1],
                         bn=True, is_training=is_training,
                         scope='dgcnn1', bn_decay=bn_decay)
    net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
    net1 = net

    # get the distance to minSF of 1024 points
    allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
    end_points['knn2'] = allSF_dist

    # # 3. graph for second EdgeConv with C = 64
    adj_matrix = tf_util.pairwise_distance(net)
    nn_idx = tf_util.knn(adj_matrix, k=para.k)
    edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k)
    net = tf_util.conv2d(edge_feature, 64, [1, 1],
                         padding='VALID', stride=[1, 1],
                         bn=True, is_training=is_training,
                         scope='dgcnn2', bn_decay=bn_decay)
    net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
    net2 = net

    # get the distance to minSF of 1024 points
    allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
    end_points['knn3'] = allSF_dist

    # # 4. graph for third EdgeConv with C = 64
    adj_matrix = tf_util.pairwise_distance(net)
    nn_idx = tf_util.knn(adj_matrix, k=para.k)
    edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k)
    net = tf_util.conv2d(edge_feature, 64, [1, 1],
                         padding='VALID', stride=[1, 1],
                         bn=True, is_training=is_training,
                         scope='dgcnn3', bn_decay=bn_decay)
    net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
    net3 = net

    # get the distance to minSF of 1024 points
    allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
    end_points['knn4'] = allSF_dist

    # # 5. graph for fourth EdgeConv with C = 64
    adj_matrix = tf_util.pairwise_distance(net)
    nn_idx = tf_util.knn(adj_matrix, k=para.k)

    edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k)
    net = tf_util.conv2d(edge_feature, 128, [1, 1],
                         padding='VALID', stride=[1, 1],
                         bn=True, is_training=is_training,
                         scope='dgcnn4', bn_decay=bn_decay)
    net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True)
    net4 = net

    # get the distance to minSF of 1024 points
    allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1)
    end_points['knn5'] = allSF_dist

    # # 6. MLP for all concatenate features 64+64+64+128
    net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1), 1024, [1, 1],
                         padding='VALID', stride=[1, 1],
                         bn=True, is_training=is_training,
                         scope='agg', bn_decay=bn_decay)
    net = tf.reduce_max(input_tensor=net, axis=1, keepdims=True)  # maxpooling B N C=1024 => B 1 1024

    # # 7. MLP on global point cloud vector B 1 1024
    net = tf.reshape(net, [batch_size, -1])
    net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay)
    net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
    net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay)
    net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2')
    net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3')

    return net, end_points
示例#6
0
def get_model(point_cloud,
              up_ratio,
              is_training,
              bradius=1.0,
              knn=30,
              scope='generator',
              weight_decay=0.0,
              bn_decay=None,
              bn=True,
              fd=64,
              fD=1024):
    with tf.variable_scope(scope, reuse=tf.AUTO_REUSE) as sc:
        batch_size = point_cloud.get_shape()[0].value
        num_point = point_cloud.get_shape()[1].value
        input_point_cloud = tf.expand_dims(point_cloud, -1)

        adj = tf_util.pairwise_distance(point_cloud)
        nn_idx = tf_util.knn(adj, k=knn)
        edge_feature = tf_util.get_edge_feature(input_point_cloud,
                                                nn_idx=nn_idx,
                                                k=knn)

        with tf.variable_scope('transform_net1') as sc:
            transform = input_transform_net(edge_feature,
                                            is_training,
                                            bn_decay,
                                            K=3)
        point_cloud_transformed = tf.matmul(point_cloud, transform)
        input_point_cloud = tf.expand_dims(point_cloud_transformed, -1)
        adj = tf_util.pairwise_distance(point_cloud_transformed)
        nn_idx = tf_util.knn(adj, k=knn)
        edge_feature = tf_util.get_edge_feature(input_point_cloud,
                                                nn_idx=nn_idx,
                                                k=knn)

        out1 = tf_util.conv2d(edge_feature,
                              fd, [1, 1],
                              padding='VALID',
                              stride=[1, 1],
                              bn=bn,
                              is_training=is_training,
                              weight_decay=weight_decay,
                              scope='dgcnn_conv1',
                              bn_decay=bn_decay)

        out2 = tf_util.conv2d(out1,
                              fd, [1, 1],
                              padding='VALID',
                              stride=[1, 1],
                              bn=bn,
                              is_training=is_training,
                              weight_decay=weight_decay,
                              scope='dgcnn_conv2',
                              bn_decay=bn_decay)

        net_max_1 = tf.reduce_max(out2, axis=-2, keep_dims=True)
        net_mean_1 = tf.reduce_mean(out2, axis=-2, keep_dims=True)

        out3 = tf_util.conv2d(tf.concat([net_max_1, net_mean_1], axis=-1),
                              fd, [1, 1],
                              padding='VALID',
                              stride=[1, 1],
                              bn=bn,
                              is_training=is_training,
                              weight_decay=weight_decay,
                              scope='dgcnn_conv3',
                              bn_decay=bn_decay)

        adj = tf_util.pairwise_distance(tf.squeeze(out3, axis=-2))
        nn_idx = tf_util.knn(adj, k=knn)
        edge_feature = tf_util.get_edge_feature(out3, nn_idx=nn_idx, k=knn)

        out4 = tf_util.conv2d(edge_feature,
                              fd, [1, 1],
                              padding='VALID',
                              stride=[1, 1],
                              bn=bn,
                              is_training=is_training,
                              weight_decay=weight_decay,
                              scope='dgcnn_conv4',
                              bn_decay=bn_decay)

        net_max_2 = tf.reduce_max(out4, axis=-2, keep_dims=True)
        net_mean_2 = tf.reduce_mean(out4, axis=-2, keep_dims=True)

        out5 = tf_util.conv2d(tf.concat([net_max_2, net_mean_2], axis=-1),
                              fd, [1, 1],
                              padding='VALID',
                              stride=[1, 1],
                              bn=bn,
                              is_training=is_training,
                              weight_decay=weight_decay,
                              scope='dgcnn_conv5',
                              bn_decay=bn_decay)

        adj = tf_util.pairwise_distance(tf.squeeze(out5, axis=-2))
        nn_idx = tf_util.knn(adj, k=knn)
        edge_feature = tf_util.get_edge_feature(out5, nn_idx=nn_idx, k=knn)

        out6 = tf_util.conv2d(edge_feature,
                              fd, [1, 1],
                              padding='VALID',
                              stride=[1, 1],
                              bn=bn,
                              is_training=is_training,
                              weight_decay=weight_decay,
                              scope='dgcnn_conv6',
                              bn_decay=bn_decay)

        net_max_3 = tf.reduce_max(out6, axis=-2, keep_dims=True)
        net_mean_3 = tf.reduce_mean(out6, axis=-2, keep_dims=True)

        out7 = tf_util.conv2d(tf.concat([net_max_3, net_mean_3], axis=-1),
                              fd, [1, 1],
                              padding='VALID',
                              stride=[1, 1],
                              bn=bn,
                              is_training=is_training,
                              weight_decay=weight_decay,
                              scope='dgcnn_conv7',
                              bn_decay=bn_decay)

        out8 = tf_util.conv2d(tf.concat([out3, out5, out7], axis=-1),
                              fD, [1, 1],
                              padding='VALID',
                              stride=[1, 1],
                              bn=bn,
                              is_training=is_training,
                              scope='dgcnn_conv8',
                              bn_decay=bn_decay)

        out_max = tf_util.max_pool2d(out8, [num_point, 1],
                                     padding='VALID',
                                     scope='maxpool')

        expand = tf.tile(out_max, [1, num_point, 1, 1])

        concat_unweight = tf.concat(axis=3,
                                    values=[
                                        expand, net_max_1, net_mean_1, out3,
                                        net_max_2, net_mean_2, out5, net_max_3,
                                        net_mean_3, out7, out8
                                    ])

        feat_list = [
            "expand", "net_max_1", "net_mean_1", "out3", "net_max_2",
            "net_mean_2", "out5", "net_max_3", "net_mean_3", "out7", "out8"
        ]

        out_attention = tf_util.conv2d(concat_unweight,
                                       128, [1, 1],
                                       padding='VALID',
                                       stride=[1, 1],
                                       bn_decay=bn_decay,
                                       bn=True,
                                       is_training=is_training,
                                       scope='attention_conv1',
                                       weight_decay=weight_decay)
        out_attention = tf_util.conv2d(out_attention,
                                       64, [1, 1],
                                       padding='VALID',
                                       stride=[1, 1],
                                       bn_decay=bn_decay,
                                       bn=True,
                                       is_training=is_training,
                                       scope='attention_conv2',
                                       weight_decay=weight_decay)
        out_attention = tf_util.conv2d(out_attention,
                                       len(feat_list), [1, 1],
                                       padding='VALID',
                                       stride=[1, 1],
                                       bn_decay=bn_decay,
                                       bn=True,
                                       is_training=is_training,
                                       scope='attention_conv3',
                                       weight_decay=weight_decay)

        out_attention = tf_util.max_pool2d(out_attention, [num_point, 1],
                                           padding='VALID',
                                           scope='attention_maxpool')
        out_attention = tf.nn.softmax(out_attention)
        tmp_attention = tf.squeeze(out_attention)

        for i in range(len(feat_list)):
            tmp1 = tf.slice(out_attention, [0, 0, 0, i], [1, 1, 1, 1])
            exec('dim = %s.get_shape()[-1].value' % feat_list[i])
            tmp2 = tf.tile(tmp1, [1, 1, 1, dim])
            if i == 0:
                attention_weight = tmp2
            else:
                attention_weight = tf.concat([attention_weight, tmp2], axis=-1)
        attention_weight = tf.tile(attention_weight, [1, num_point, 1, 1])
        concat = tf.multiply(concat_unweight, attention_weight)

        concat = tf_util.conv2d(concat,
                                256, [1, 1],
                                padding='VALID',
                                stride=[1, 1],
                                bn_decay=bn_decay,
                                bn=True,
                                is_training=is_training,
                                scope='concat_conv',
                                weight_decay=weight_decay)
        concat = tf_util.dropout(concat,
                                 keep_prob=0.6,
                                 is_training=is_training,
                                 scope='dg1')

        with tf.variable_scope('uv_predict'):
            uv_2d = tf_util.conv2d(concat,
                                   up_ratio * 2, [1, 1],
                                   padding='VALID',
                                   stride=[1, 1],
                                   bn_decay=bn_decay,
                                   activation_fn=None,
                                   bn=None,
                                   is_training=is_training,
                                   scope='uv_conv1',
                                   weight_decay=weight_decay)
            uv_2d = tf.reshape(uv_2d, [batch_size, num_point, up_ratio, 2])
            uv_2d = tf.concat(
                [uv_2d, tf.zeros([batch_size, num_point, up_ratio, 1])],
                axis=-1)

        with tf.variable_scope('T_predict'):
            affine_T = tf_util.conv2d(concat,
                                      9, [1, 1],
                                      padding='VALID',
                                      stride=[1, 1],
                                      bn_decay=bn_decay,
                                      activation_fn=None,
                                      bn=None,
                                      is_training=is_training,
                                      scope='patch_conv1',
                                      weight_decay=weight_decay)
            affine_T = tf.reshape(affine_T, [batch_size, num_point, 3, 3])

            uv_3d = tf.matmul(uv_2d, affine_T)
            uv_3d = uv_3d + tf.tile(tf.expand_dims(point_cloud, axis=-2),
                                    [1, 1, up_ratio, 1])
            uv_3d = tf.transpose(uv_3d, perm=[0, 2, 1, 3])
            uv_3d = tf.reshape(uv_3d, [batch_size, num_point * up_ratio, -1])

        with tf.variable_scope('normal_predict'):
            dense_normal_offset = tf_util.conv2d(concat,
                                                 up_ratio * 3, [1, 1],
                                                 padding='VALID',
                                                 stride=[1, 1],
                                                 bn_decay=bn_decay,
                                                 activation_fn=None,
                                                 bn=None,
                                                 is_training=is_training,
                                                 scope='normal_offset_conv1',
                                                 weight_decay=weight_decay)
            dense_normal_offset = tf.reshape(
                dense_normal_offset, [batch_size, num_point, up_ratio, 3])

            sparse_normal = tf.convert_to_tensor([0, 0, 1], dtype=tf.float32)
            sparse_normal = tf.expand_dims(sparse_normal, axis=0)
            sparse_normal = tf.expand_dims(sparse_normal, axis=0)
            sparse_normal = tf.expand_dims(sparse_normal, axis=0)
            sparse_normal = tf.tile(sparse_normal,
                                    [batch_size, num_point, 1, 1])
            sparse_normal = tf.matmul(sparse_normal, affine_T)
            sparse_normal = tf.nn.l2_normalize(sparse_normal, axis=-1)

            dense_normal = tf.tile(sparse_normal,
                                   [1, 1, up_ratio, 1]) + dense_normal_offset

            dense_normal = tf.nn.l2_normalize(dense_normal, axis=-1)
            dense_normal = tf.transpose(dense_normal, perm=[0, 2, 1, 3])
            dense_normal = tf.reshape(dense_normal,
                                      [batch_size, num_point * up_ratio, -1])

        with tf.variable_scope('up_layer'):
            if not np.isscalar(bradius):
                bradius_expand = tf.expand_dims(tf.expand_dims(bradius,
                                                               axis=-1),
                                                axis=-1)
            else:
                bradius_expand = bradius
            bradius_expand = bradius_expand
            grid = tf.expand_dims(uv_3d * bradius_expand, axis=2)

            concat_up = tf.tile(concat, (1, up_ratio, 1, 1))
            concat_up = tf.concat([concat_up, grid], axis=-1)

            concat_up = tf_util.conv2d(concat_up,
                                       128, [1, 1],
                                       padding='VALID',
                                       stride=[1, 1],
                                       bn=True,
                                       is_training=is_training,
                                       scope='up_layer1',
                                       bn_decay=bn_decay,
                                       weight_decay=weight_decay)

            concat_up = tf_util.dropout(concat_up,
                                        keep_prob=0.6,
                                        is_training=is_training,
                                        scope='up_dg1')

            concat_up = tf_util.conv2d(concat_up,
                                       128, [1, 1],
                                       padding='VALID',
                                       stride=[1, 1],
                                       bn=True,
                                       is_training=is_training,
                                       scope='up_layer2',
                                       bn_decay=bn_decay,
                                       weight_decay=weight_decay)
            concat_up = tf_util.dropout(concat_up,
                                        keep_prob=0.6,
                                        is_training=is_training,
                                        scope='up_dg2')

        # get xyz
        coord_z = tf_util.conv2d(concat_up,
                                 1, [1, 1],
                                 padding='VALID',
                                 stride=[1, 1],
                                 activation_fn=None,
                                 bn=False,
                                 is_training=is_training,
                                 scope='fc_layer',
                                 weight_decay=weight_decay)
        coord_z = tf.reshape(coord_z, [batch_size, up_ratio, num_point, 1])
        coord_z = tf.transpose(coord_z, perm=[0, 2, 1, 3])
        coord_z = tf.concat(
            [tf.zeros_like(coord_z),
             tf.zeros_like(coord_z), coord_z], axis=-1)

        coord_z = tf.matmul(coord_z, affine_T)
        coord_z = tf.transpose(coord_z, perm=[0, 2, 1, 3])
        coord_z = tf.reshape(coord_z, [batch_size, num_point * up_ratio, -1])

        coord = uv_3d + coord_z

    return coord, dense_normal, tf.squeeze(sparse_normal, [2])
def get_model(point_cloud,
              is_training,
              normals,
              use_local_frame=True,
              add_normals=False,
              bn=True,
              bn_decay=None,
              use_xavier=True,
              align_pointclouds=False,
              drop_prob=0.5,
              n_classes=1,
              k=20):
    """ Part segmentation DGCNN, input is BxNxnFeatures, output BxnClasses """

    batch_size = point_cloud.get_shape()[0].value
    num_point = point_cloud.get_shape()[1].value

    # Input xyz coordinates
    input_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])

    # Add tangent vectors and normals for local frames calculation
    local_frame_data = None
    if use_local_frame:
        tangent_vec1 = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3],
                                name="tangent_vec1")
        tangent_vec2 = tf.slice(point_cloud, [0, 0, 6], [-1, -1, 3],
                                name="tangent_vec2")
        local_frame_data = (tangent_vec1, tangent_vec2, normals)

    # Point clouds global alignment
    if align_pointclouds:
        # Calculate pairwise distance on global coordinates and find k-nn's for each point
        adj = tf_util.pairwise_distance(input_xyz)
        nn_idx = tf_util.knn(adj, k=k)
        input_xyz = tf.expand_dims(input_xyz, -1)
        edge_feature = tf_util.get_edge_feature(input_xyz, nn_idx=nn_idx, k=k)

        with tf.variable_scope('transform_net_global') as sc:
            global_transform = global_spatial_transformer(
                point_cloud=edge_feature,
                is_training=is_training,
                bn=bn,
                bn_decay=bn_decay,
                is_dist=True)
        input_xyz = tf.matmul(tf.squeeze(input_xyz, axis=-1), global_transform)

    if add_normals:
        if input_xyz.shape.ndims == 4:
            input_xyz = tf.squeeze(input_xyz, axis=-1)
        input_xyz = tf.concat([input_xyz, normals], axis=-1)

    input_image = tf.expand_dims(input_xyz, -1)
    adj = tf_util.pairwise_distance(input_xyz)
    nn_idx = tf_util.knn(adj, k=k)
    edge_feature = tf_util.get_edge_feature(input_image,
                                            nn_idx=nn_idx,
                                            k=k,
                                            use_local_frame=use_local_frame,
                                            local_frame_data=local_frame_data,
                                            add_normals=add_normals)

    # EdgeConv layer 1 {64, 64}
    out1 = tf_util.conv2d(edge_feature,
                          64, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          bn=bn,
                          is_training=is_training,
                          scope='adj_conv1',
                          bn_decay=bn_decay,
                          is_dist=True,
                          use_xavier=use_xavier)

    out2 = tf_util.conv2d(out1,
                          64, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          bn=bn,
                          is_training=is_training,
                          scope='adj_conv2',
                          bn_decay=bn_decay,
                          is_dist=True,
                          use_xavier=use_xavier)

    net_1 = tf.reduce_max(out2, axis=-2, keep_dims=True)

    # EdgeConv layer 2 {64, 64}
    adj = tf_util.pairwise_distance(net_1)
    nn_idx = tf_util.knn(adj, k=k)
    edge_feature = tf_util.get_edge_feature(net_1, nn_idx=nn_idx, k=k)

    out3 = tf_util.conv2d(edge_feature,
                          64, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          bn=bn,
                          is_training=is_training,
                          scope='adj_conv3',
                          bn_decay=bn_decay,
                          is_dist=True,
                          use_xavier=use_xavier)

    out4 = tf_util.conv2d(out3,
                          64, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          bn=bn,
                          is_training=is_training,
                          scope='adj_conv4',
                          bn_decay=bn_decay,
                          is_dist=True,
                          use_xavier=use_xavier)

    net_2 = tf.reduce_max(out4, axis=-2, keep_dims=True)

    # EdgeConv layer 3 {64}
    adj = tf_util.pairwise_distance(net_2)
    nn_idx = tf_util.knn(adj, k=k)
    edge_feature = tf_util.get_edge_feature(net_2, nn_idx=nn_idx, k=k)

    out5 = tf_util.conv2d(edge_feature,
                          64, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          bn=bn,
                          is_training=is_training,
                          scope='adj_conv5',
                          bn_decay=bn_decay,
                          is_dist=True,
                          use_xavier=use_xavier)

    net_3 = tf.reduce_max(out5, axis=-2, keep_dims=True)

    # [EdgeConv1, EdgeConv2, EdgeConv3] -> MLP {64}
    out7 = tf_util.conv2d(tf.concat([net_1, net_2, net_3], axis=-1),
                          1024, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          bn=bn,
                          is_training=is_training,
                          scope='adj_conv7',
                          bn_decay=bn_decay,
                          is_dist=True,
                          use_xavier=use_xavier)

    out_max = tf_util.max_pool2d(out7, [num_point, 1],
                                 padding='VALID',
                                 scope='maxpool')
    expand = tf.tile(out_max, [1, num_point, 1, 1])

    # Concat [global_feature, EdgeConv1, EdgeConv2, EdgeConv3]
    concat = tf.concat(axis=3, values=[expand, net_1, net_2, net_3])

    # FC layer - MLP{256, 256, 128, n_classes}
    net2 = tf_util.conv2d(concat,
                          256, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          bn_decay=bn_decay,
                          bn=bn,
                          is_training=is_training,
                          scope='seg/conv1',
                          is_dist=True,
                          use_xavier=use_xavier)
    net2 = tf_util.dropout(net2,
                           keep_prob=1 - drop_prob,
                           is_training=is_training,
                           scope='seg/dp1')
    net2 = tf_util.conv2d(net2,
                          256, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          bn_decay=bn_decay,
                          bn=bn,
                          is_training=is_training,
                          scope='seg/conv2',
                          is_dist=True,
                          use_xavier=use_xavier)
    net2 = tf_util.dropout(net2,
                           keep_prob=1 - drop_prob,
                           is_training=is_training,
                           scope='seg/dp2')
    net2 = tf_util.conv2d(net2,
                          128, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          bn_decay=bn_decay,
                          bn=bn,
                          is_training=is_training,
                          scope='seg/conv3',
                          is_dist=True,
                          use_xavier=use_xavier)
    net2 = tf_util.conv2d(net2,
                          n_classes, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          activation_fn=None,
                          bn=False,
                          scope='seg/conv4',
                          is_dist=False,
                          use_xavier=use_xavier)

    net2 = tf.reshape(net2, [batch_size, num_point, n_classes])

    return net2
示例#8
0
def residual_attn_block(point_cloud,
                        mv_fc,
                        k=20,
                        get_mask=False,
                        C_out=64,
                        C_attn=256,
                        is_training=True,
                        bn_decay=None,
                        scope='attn1_'):
    """

    :param point_cloud: (N, P, 1, C_in)
    :param mv_fc: (N, C(1024))
    :param is_training: training state
    :param k: k neighbors
    :param C_out: output channel
    :param bn_decay: bn decay
    :param scope: scope name
    :return: (N, P, 1, C_out)
    """

    point_cloud_sq = tf.squeeze(point_cloud)
    batch_size = point_cloud_sq.shape[0].value
    num_points = point_cloud_sq.shape[1].value
    num_dims_in = point_cloud_sq.shape[2].value

    adj_matrix = tf_util.pairwise_distance(point_cloud)
    nn_idx = tf_util.knn(adj_matrix, k=k)
    edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=k)
    res1 = tf_util.conv2d(edge_feature,
                          C_out, [1, 1],
                          padding='VALID',
                          stride=[1, 1],
                          bn=True,
                          is_training=is_training,
                          scope=scope + '_res1',
                          bn_decay=bn_decay)
    res1 = tf.reduce_max(res1, axis=-2, keep_dims=True)

    # res2 = tf_util.conv2d(point_cloud, C_attn, [1,1],
    #                       padding='VALID', stride=[1, 1],
    #                       bn=True, is_training=is_training,
    #                       scope=scope+'res2_conv1', bn_decay=bn_decay)
    # res2_global = tf_util.max_pool2d(res2, [num_points, 1], padding='VALID', scope=scope+'maxpool')
    # print (res2_global.get_shape())
    res2_global = tf.expand_dims(tf.expand_dims(mv_fc, axis=1), axis=1)
    res2_global = tf.tile(res2_global, [1, num_points, 1, 1])
    # print (res2_global.get_shape())
    res2_concat = tf.concat([res2_global, point_cloud], axis=-1)
    res2_out = tf_util.conv2d(res2_concat,
                              C_out, [1, 1],
                              padding='VALID',
                              stride=[1, 1],
                              bn=True,
                              is_training=is_training,
                              scope=scope + '_res2',
                              bn_decay=bn_decay)
    res2_mask = tf.nn.sigmoid(tf.log(res2_out))
    # print (res2_mask.get_shape())

    res2_attn = tf.multiply(res2_mask, res1)
    # print (res2_attn.get_shape())

    net = tf.add(res2_attn, res1)
    # print (net.get_shape())

    if get_mask:
        return net, res2_mask
    else:
        return net
示例#9
0
def multi_modal(point_cloud,
                view_images,
                is_training=False,
                bn_decay=None,
                n_classes=40,
                get_ft=False,
                get_mask=False):
    """

    :param point_cloud: (B, N, 3)
    :param view_images: (B, V, W, H, C)
    :param is_training: is_training for dropout and bn
    :param bn_decay: bn_decay
    :param n_classes: 40
    :return: multi-modal logit and mvcnn logit
    """

    fc6_b = MVCNN.inference_multiview(view_images,
                                      n_classes,
                                      is_training=is_training)

    mv_global = tf_util.fully_connected(fc6_b,
                                        1024,
                                        bn=True,
                                        is_training=is_training,
                                        scope='pc_mv_t',
                                        bn_decay=bn_decay)

    batch_size = point_cloud.get_shape()[0].value
    num_point = point_cloud.get_shape()[1].value
    k = 20

    point_cloud_transformed = trans_net(point_cloud,
                                        k=k,
                                        is_training=is_training,
                                        bn_decay=bn_decay,
                                        scope='pc_transform_net1')

    adj_matrix = tf_util.pairwise_distance(point_cloud_transformed)
    nn_idx = tf_util.knn(adj_matrix, k=k)
    edge_feature = tf_util.get_edge_feature(point_cloud_transformed,
                                            nn_idx=nn_idx,
                                            k=k)
    net = tf_util.conv2d(edge_feature,
                         64, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='pc1',
                         bn_decay=bn_decay)
    net = tf.reduce_max(net, axis=-2, keep_dims=True)
    net1 = net

    adj_matrix = tf_util.pairwise_distance(net)
    nn_idx = tf_util.knn(adj_matrix, k=k)
    edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=k)
    net = tf_util.conv2d(edge_feature,
                         64, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='pc2',
                         bn_decay=bn_decay)
    net = tf.reduce_max(net, axis=-2, keep_dims=True)
    net2 = net

    net = residual_attn_block(net,
                              mv_global,
                              k=k,
                              C_out=64,
                              get_mask=get_mask,
                              C_attn=256,
                              is_training=is_training,
                              bn_decay=bn_decay,
                              scope='pc_attn_1')
    if get_mask:
        net3, mask1 = net
        net = net3
    else:
        net3 = net

    net = residual_attn_block(net,
                              mv_global,
                              k=k,
                              C_out=128,
                              get_mask=get_mask,
                              C_attn=256,
                              is_training=is_training,
                              bn_decay=bn_decay,
                              scope='pc_attn_2')
    if get_mask:
        net4, mask2 = net
        net = net3
    else:
        net4 = net

    net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1),
                         1024, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='pc_agg',
                         bn_decay=bn_decay)

    net = tf.reduce_max(net, axis=1, keep_dims=True)

    # MLP on global point cloud vector
    net = tf.reshape(net, [batch_size, -1])
    net = tf.concat([net, mv_global], axis=-1)

    net = tf_util.fully_connected(net,
                                  512,
                                  bn=True,
                                  is_training=is_training,
                                  scope='pc_fc1',
                                  bn_decay=bn_decay)
    net = tf_util.dropout(net,
                          keep_prob=0.5,
                          is_training=is_training,
                          scope='pc_dp1')
    net = tf_util.fully_connected(net,
                                  256,
                                  bn=True,
                                  is_training=is_training,
                                  scope='pc_fc2',
                                  bn_decay=bn_decay)
    ft = net
    net = tf_util.dropout(net,
                          keep_prob=0.5,
                          is_training=is_training,
                          scope='pc_dp2')
    net = tf_util.fully_connected(net, 40, activation_fn=None, scope='pc_fc3')

    if get_ft:
        return net, ft
    elif get_mask:
        return net, mask1, mask2
    else:
        return net