def get_transform(point_cloud, is_training, bn_decay=None, K = 3): """ Transform Net, input is BxNx3 gray image Return: Transformation matrix of size 3xK """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(input_image, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='tconv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='tconv4', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 128, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 128, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: assert(K==3) weights = tf.get_variable('weights', [128, 3*K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [3*K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) + tf.constant([1,0,0,0,1,0,0,0,1], dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) #transform = tf_util.fully_connected(net, 3*K, activation_fn=None, scope='tfc3') transform = tf.reshape(transform, [batch_size, 3, K]) return transform
def get_transform_K(inputs, is_training, bn_decay=None, K = 3): """ Transform Net, input is BxNx1xK gray image Return: Transformation matrix of size KxK """ batch_size = inputs.get_shape()[0].value num_point = inputs.get_shape()[1].value net = tf_util.conv2d(inputs, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='tconv2', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_feat') as sc: weights = tf.get_variable('weights', [256, K*K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [K*K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) + tf.constant(np.eye(K).flatten(), dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) #transform = tf_util.fully_connected(net, 3*K, activation_fn=None, scope='tfc3') transform = tf.reshape(transform, [batch_size, K, K]) return transform
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_3d_box_estimation_v1_net(object_point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D Box Estimation PointNet v1 network. Input: object_point_cloud: TF tensor in shape (B,M,C) point clouds in object coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type Output: output: TF tensor in shape (B,3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER*4) including box centers, heading bin class scores and residuals, and size cluster scores and residuals ''' num_point = object_point_cloud.get_shape()[1].value net = tf.expand_dims(object_point_cloud, 2) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg2', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg3', bn_decay=bn_decay) net = tf_util.conv2d(net, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg4', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool2') net = tf.squeeze(net, axis=[1,2]) net = tf.concat([net, one_hot_vec], axis=1) net = tf_util.fully_connected(net, 512, scope='fc1', bn=True, is_training=is_training, bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, scope='fc2', bn=True, is_training=is_training, bn_decay=bn_decay) # The first 3 numbers: box center coordinates (cx,cy,cz), # the next NUM_HEADING_BIN*2: heading bin class scores and bin residuals # next NUM_SIZE_CLUSTER*4: box cluster scores and residuals output = tf_util.fully_connected(net, 3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER*4, activation_fn=None, scope='fc3') return output, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Point functions (MLP implemented as conv2d) net = tf_util.conv2d(input_image, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_center_regression_net(object_point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' Regression network for center delta. a.k.a. T-Net. Input: object_point_cloud: TF tensor in shape (B,M,C) point clouds in 3D mask coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type Output: predicted_center: TF tensor in shape (B,3) ''' num_point = object_point_cloud.get_shape()[1].value net = tf.expand_dims(object_point_cloud, 2) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg1-stage1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg2-stage1', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg3-stage1', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool-stage1') net = tf.squeeze(net, axis=[1,2]) net = tf.concat([net, one_hot_vec], axis=1) net = tf_util.fully_connected(net, 256, scope='fc1-stage1', bn=True, is_training=is_training, bn_decay=bn_decay) net = tf_util.fully_connected(net, 128, scope='fc2-stage1', bn=True, is_training=is_training, bn_decay=bn_decay) predicted_center = tf_util.fully_connected(net, 3, activation_fn=None, scope='fc3-stage1') return predicted_center, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output BxNx50 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) point_feat = tf.expand_dims(net_transformed, [2]) print(point_feat) net = tf_util.conv2d(point_feat, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') print(global_feat) global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1]) concat_feat = tf.concat(3, [point_feat, global_feat_expand]) print(concat_feat) net = tf_util.conv2d(concat_feat, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv6', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv7', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv8', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv9', bn_decay=bn_decay) net = tf_util.conv2d(net, 50, [1,1], padding='VALID', stride=[1,1], activation_fn=None, scope='conv10') net = tf.squeeze(net, [2]) # BxNxC return net, end_points
def get_instance_seg_v1_net(point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v1 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value net = tf.expand_dims(point_cloud, 2) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) point_feat = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(point_feat, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') global_feat = tf.concat([global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3) global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1]) concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand]) net = tf_util.conv2d(concat_feat, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv6', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv7', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv8', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv9', bn_decay=bn_decay) net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5) logits = tf_util.conv2d(net, 2, [1,1], padding='VALID', stride=[1,1], activation_fn=None, scope='conv10') logits = tf.squeeze(logits, [2]) # BxNxC return logits, end_points
def feature_transform_net(inputs, is_training, bn_decay=None, K=64): #特征变换网络,输入是BxNx1xK #返回:大小为KxK的变换矩阵 """ Feature Transform Net, input is BxNx1xK Return: Transformation matrix of size KxK """ batch_size = inputs.get_shape()[0].value num_point = inputs.get_shape()[1].value net = tf_util.conv2d(inputs, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv3', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_feat') as sc: weights = tf.get_variable('weights', [256, K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant(np.eye(K).flatten(), dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, K, K]) return transform
def pointnet_sa_module(xyz, points, npoint, radius, nsample, mlp, mlp2, group_all, is_training, bn_decay, scope, bn=True, pooling='max', tnet_spec=None, knn=False, use_xyz=True): ''' PointNet Set Abstraction (SA) Module Input: xyz: (batch_size, ndataset, 3) TF tensor points: (batch_size, ndataset, channel) TF tensor npoint: int32 -- #points sampled in farthest point sampling radius: float32 -- search radius in local region nsample: int32 -- how many points in each local region mlp: list of int32 -- output size for MLP on each point mlp2: list of int32 -- output size for MLP on each region group_all: bool -- group all points into one PC if set true, OVERRIDE npoint, radius and nsample settings use_xyz: bool, if True concat XYZ with local point features, otherwise just use point features Return: new_xyz: (batch_size, npoint, 3) TF tensor new_points: (batch_size, npoint, mlp[-1] or mlp2[-1]) TF tensor idx: (batch_size, npoint, nsample) int32 -- indices for local regions ''' with tf.variable_scope(scope) as sc: if group_all: nsample = xyz.get_shape()[1].value new_xyz, new_points, idx, grouped_xyz = sample_and_group_all( xyz, points, use_xyz) else: new_xyz, new_points, idx, grouped_xyz = sample_and_group( npoint, radius, nsample, xyz, points, tnet_spec, knn, use_xyz) for i, num_out_channel in enumerate(mlp): new_points = tf_util.conv2d(new_points, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv%d' % (i), bn_decay=bn_decay) if pooling == 'avg': new_points = tf_util.avg_pool2d(new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='avgpool1') elif pooling == 'weighted_avg': with tf.variable_scope('weighted_avg1'): dists = tf.norm(grouped_xyz, axis=-1, ord=2, keep_dims=True) exp_dists = tf.exp(-dists * 5) weights = exp_dists / tf.reduce_sum( exp_dists, axis=2, keep_dims=True) # (batch_size, npoint, nsample, 1) new_points *= weights # (batch_size, npoint, nsample, mlp[-1]) new_points = tf.reduce_sum(new_points, axis=2, keep_dims=True) elif pooling == 'max': new_points = tf.reduce_max(new_points, axis=[2], keep_dims=True) elif pooling == 'min': new_points = tf_util.max_pool2d(-1 * new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='minpool1') elif pooling == 'max_and_avg': avg_points = tf_util.max_pool2d(new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='maxpool1') max_points = tf_util.avg_pool2d(new_points, [1, nsample], stride=[1, 1], padding='VALID', scope='avgpool1') new_points = tf.concat([avg_points, max_points], axis=-1) if mlp2 is None: mlp2 = [] for i, num_out_channel in enumerate(mlp2): new_points = tf_util.conv2d(new_points, num_out_channel, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv_post_%d' % (i), bn_decay=bn_decay) new_points = tf.squeeze(new_points, [2]) # (batch_size, npoints, mlp2[-1]) return new_xyz, new_points, idx
def get_model_rbf0_gan(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net_no_bn(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) point_cloud_transformed = tf.expand_dims(point_cloud_transformed, 3) #centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32) centroids = tf.get_variable('centroids', [1, 1, 3, 1024], initializer=tf.constant_initializer( 0.2 * np.random.randn(1, 1, 3, 1024)), dtype=tf.float32) feature = tf.tile(point_cloud_transformed, [1, 1, 1, 1024]) bias = tf.tile(centroids, [batch_size, 1024, 1, 1]) net = tf.subtract(feature, bias) net = tf.norm(net, axis=2, keep_dims=True) net = tf.exp(-net) # Symmetric function: max pooling features = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(features, [batch_size, -1]) #net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training, # scope='fc0', bn_decay=bn_decay) #net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp1') net = tf_util.fully_connected(net, 512, bn=False, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=False, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 41, activation_fn=None, scope='fc3') return net, end_points, features, centroids
def get_model_rbf_transform(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) c1 = 64 #centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32) centroids = tf.get_variable('centroids', [1, 1, 3, c1], initializer=tf.constant_initializer( 0.5 * np.random.randn(1, 1, 3, c1)), dtype=tf.float32) net = tf.subtract( tf.tile(tf.expand_dims(point_cloud_transformed, 3), [1, 1, 1, c1]), tf.tile(centroids, [batch_size, num_point, 1, 1])) net = tf.norm(net, axis=2, keep_dims=True) net = tf.exp(-net) with tf.variable_scope('transform_net2', reuse=tf.AUTO_REUSE) as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) c2 = 256 #centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32) centroids2 = tf.get_variable('centroids2', [1, 1, c2, 64], initializer=tf.constant_initializer( 0.5 * np.random.randn(1, 1, c2, 64)), dtype=tf.float32) net = tf.subtract(tf.tile(net_transformed, [1, 1, c2, 1]), tf.tile(centroids2, [batch_size, num_point, 1, 1])) net = tf.norm(net, axis=3, keep_dims=True) net = tf.exp(-net) # Symmetric function: max pooling features = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(features, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points, features, centroids
def get_model_rbf3(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) point_cloud_transformed = tf.expand_dims(point_cloud_transformed, 3) #centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32) c1 = 512 c2 = 8 centroids = tf.get_variable('centroids', [1, 1, 3, c1], initializer=tf.constant_initializer( np.random.randn(1, 1, 3, c1)), dtype=tf.float32) sub_centroids = tf.get_variable('sub_centroids', [1, 1, 3, c2], initializer=tf.constant_initializer( 0.05 * np.random.randn(1, 1, 3, c2)), dtype=tf.float32) #sub_centroids = tf.constant(0.05*np.random.randn(1, 1, 3, c2), dtype=tf.float32) sub_bias = tf.add( tf.tile(tf.expand_dims(sub_centroids, 4), [1, 1, 1, 1, c1]), tf.tile(tf.expand_dims(centroids, 3), [1, 1, 1, c2, 1])) sub_bias = tf.tile(sub_bias, [batch_size, 1024, 1, 1, 1]) sub_feature = tf.tile(tf.expand_dims(point_cloud_transformed, 4), [1, 1, 1, c2, c1]) sub_net = tf.exp(-tf.square( tf.norm( tf.subtract(sub_feature, sub_bias), ord=3, axis=2, keep_dims=True)) ) sub_net = tf.squeeze(sub_net) sub_net = tf.transpose(sub_net, perm=[0, 1, 3, 2]) sub_net = tf_util.max_pool2d(sub_net, [num_point, 1], stride=[1, 1], padding='VALID', scope='maxpool') #sub_net = tf_util.conv2d(sub_net, 16, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, # scope='mini_conv1', bn_decay=bn_decay) sub_net = tf_util.conv2d(sub_net, 2, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='mini_conv2', bn_decay=bn_decay) sub_net = tf.squeeze(sub_net) feature = tf.tile(point_cloud_transformed, [1, 1, 1, c1]) bias = tf.tile(centroids, [batch_size, 1024, 1, 1]) net = tf.subtract(feature, bias) #net = tf.exp(net) net = tf.norm(net, ord=3, axis=2, keep_dims=True) net = tf.exp(-tf.square(net)) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.expand_dims(tf.squeeze(net), 2) features = tf.concat([net, sub_net], axis=2) # Symmetric function: max pooling #features = tf_util.max_pool2d(net, [num_point,1], # padding='VALID', scope='maxpool') net = tf.reshape(features, [batch_size, -1]) #net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training, # scope='fc0', bn_decay=bn_decay) #net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp1') net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points, features, centroids
def get_encoder_network(images, is_training): # Args: # images: is tensor of size BxHxWx1 where B is batch size, H and W are image dimensions # Returns: # encoder: is a Python dictionary containing all tensors net = {} net['conv1'] = tf_util.conv2d(images, 32, [3, 3], activation_fn=leak_relu, scope='encoder/conv1', bn=True, is_training=is_training) # H * W net['conv2'] = tf_util.conv2d(net['conv1'], 32, [3, 3], activation_fn=leak_relu, scope='encoder/conv2', bn=True, is_training=is_training) # H * W net['pool3'] = tf_util.max_pool2d(net['conv2'], [2, 2], scope='encoder/pool3', stride=[2, 2], padding='VALID') # H/2 * W/2 net['conv4'] = tf_util.conv2d(net['pool3'], 32, [3, 3], activation_fn=leak_relu, scope='encoder/conv4', bn=True, is_training=is_training) # H/2 * W/2 net['conv5'] = tf_util.conv2d(net['conv4'], 32, [3, 3], activation_fn=leak_relu, scope='encoder/conv5', bn=True, is_training=is_training) # H/2 * W/2 net['pool6'] = tf_util.max_pool2d(net['conv5'], [2, 2], scope='encoder/pool6', stride=[2, 2], padding='VALID') # H/4 * W/4 net['conv7'] = tf_util.conv2d(net['pool6'], 64, [3, 3], activation_fn=leak_relu, scope='encoder/conv7', bn=True, is_training=is_training) # H/4 * W/4 net['conv8'] = tf_util.conv2d(net['conv7'], 64, [3, 3], activation_fn=leak_relu, scope='encoder/conv8', bn=True, is_training=is_training) # H/4 * W/4 net['conv9'] = tf_util.conv2d(net['conv8'], 64, [3, 3], activation_fn=leak_relu, scope='encoder/conv9', bn=True, is_training=is_training) # H/4 * W/4 net['pool10'] = tf_util.max_pool2d(net['conv9'], [2, 2], scope='encoder/pool10', stride=[2, 2], padding='VALID') # H/8 * W/8 net['conv11'] = tf_util.conv2d(net['pool10'], 64, [3, 3], activation_fn=leak_relu, scope='encoder/conv11', bn=True, is_training=is_training) # H/8 * W/8 net['conv12'] = tf_util.conv2d(net['conv11'], 64, [3, 3], activation_fn=leak_relu, scope='encoder/conv12', bn=True, is_training=is_training) # H/8 * W/8 net['conv13'] = tf_util.conv2d(net['conv12'], 64, [3, 3], activation_fn=leak_relu, scope='encoder/conv13', bn=True, is_training=is_training) # H/8 * W/8 net['pool14'] = tf_util.max_pool2d(net['conv13'], [2, 2], scope='encoder/pool14', stride=[2, 2], padding='VALID') # H/16 * W/16 net['conv15'] = tf_util.conv2d(net['pool14'], 128, [3, 3], activation_fn=leak_relu, scope='encoder/conv15', bn=True, is_training=is_training) # H/16 * W/16 net['conv16'] = tf_util.conv2d(net['conv15'], 128, [3, 3], activation_fn=leak_relu, scope='encoder/conv16', bn=True, is_training=is_training) # H/16 * W/16 net['conv17'] = tf_util.conv2d(net['conv16'], 128, [3, 3], activation_fn=leak_relu, scope='encoder/conv17', bn=True, is_training=is_training) # H/16 * W/16 net['pool18'] = tf_util.max_pool2d(net['conv17'], [2, 2], scope='encoder/pool18', stride=[2, 2], padding='VALID') # H/32 * W/32 net['conv19'] = tf_util.conv2d(net['pool18'], 128, [3, 3], activation_fn=leak_relu, scope='encoder/conv19', bn=True, is_training=is_training) # H/32 * W/32 return net
def get_model(point_cloud, is_training, autoencoder=False, bn_decay=None): """ Autoencoder for point clouds. Input: point_cloud: TF tensor BxNx3 is_training: boolean bn_decay: float between 0 and 1 Output: net: TF tensor BxNx3, reconstructed point clouds end_points: dict """ batch_size = point_cloud.get_shape()[0].value num_pointclouds_per_query = point_cloud.get_shape()[1].value num_point = point_cloud.get_shape()[2].value point_cloud = tf.reshape( point_cloud, [batch_size * num_pointclouds_per_query, num_point, 3]) end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Encoder net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) point_feat = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(point_feat, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(global_feat, [batch_size * num_pointclouds_per_query, -1]) feature = net net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, bn_decay=bn_decay, scope='fc1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, bn_decay=bn_decay, scope='fc2') net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf.reshape(net, [batch_size, num_pointclouds_per_query, -1]) ### net = (batch_size, num_pointclouds_per_query, -1) return net, feature
def get_model_w_ae(self, input_x, is_training, reuse=False, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = self.configuration.batch_size num_point = self.configuration.n_input[0] end_points = {} # with tf.variable_scope('Classifier', reuse=tf.AUTO_REUSE) as scope: with tf.variable_scope('transform_net1') as sc: transform = self.input_transform_net(input_x, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(input_x, transform) end_points['first'] = point_cloud_transformed input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) end_points['second'] = net net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = self.feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) #print("before maxpool") #print(net.get_shape()) end_points['pre_max'] = net # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') #print("after maxpool") #print(net.get_shape()) net = tf.reshape(net, [batch_size, -1]) end_points['post_max'] = net #print("after reshape") #print(net.get_shape()) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') end_points['final'] = net net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') #print(end_points['pre_max'].get_shape()) return net, end_points
def get_model(input_tensor, is_training, bn_decay = None): weight_decay = 0.0 num_point = input_tensor.get_shape()[1].value k = 40 #Transform Net adj_matrix = tf_util.pairwise_distance(input_tensor) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(input_tensor, nn_idx=nn_idx, k=k) #Transform Net with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(edge_feature, is_training, bn_decay, K=input_tensor.get_shape()[2], is_dist=True) input_tensor_transformed = tf.matmul(input_tensor, transform) adj_matrix = tf_util.pairwise_distance(input_tensor_transformed) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(input_tensor_transformed, nn_idx=nn_idx, k=k) out1_1 = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='one/adj_conv1', bn_decay=bn_decay, is_dist=True) out1_2 = tf_util.conv2d(out1_1, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='one/adj_conv2', bn_decay=bn_decay, is_dist=True) out1_3 = tf_util.conv2d(out1_2, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='one/adj_conv3', bn_decay=bn_decay, is_dist=True) net_1 = tf.reduce_max(out1_3, axis=-2, keepdims=True) adj = tf_util.pairwise_distance(net_1) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(net_1, nn_idx=nn_idx, k=k) out2_1 = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='two/adj_conv1', bn_decay=bn_decay, is_dist=True) out2_2 = tf_util.conv2d(out2_1, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='two/adj_conv2', bn_decay=bn_decay, is_dist=True) out2_3 = tf_util.conv2d(out2_2, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='two/adj_conv3', bn_decay=bn_decay, is_dist=True) net_2 = tf.reduce_max(out2_3, axis=-2, keepdims=True) adj = tf_util.pairwise_distance(net_2) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(net_2, nn_idx=nn_idx, k=k) out3_1 = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='three/adj_conv1', bn_decay=bn_decay, is_dist=True) out3_2 = tf_util.conv2d(out3_1, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='three/adj_conv2', bn_decay=bn_decay, is_dist=True) net_3 = tf.reduce_max(out3_2, axis=-2, keepdims=True) out7 = tf_util.conv2d(tf.concat([net_1, net_2, net_3], axis=-1), 1024, [1, 1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='adj_conv7', bn_decay=bn_decay, is_dist=True) out_max = tf_util.max_pool2d(out7, [num_point, 1], padding='VALID', scope='maxpool') expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[expand, net_1, net_2, net_3]) # CONV net = tf_util.conv2d(concat, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='seg/conv1', is_dist=True) # net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], # bn=True, is_training=is_training, scope='seg/conv2', is_dist=True) # net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], # bn=True, is_training=is_training, scope='seg/conv3', is_dist=True) # net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], # bn=True, is_training=is_training, scope='seg/conv4', is_dist=True) # net = tf_util.conv2d(net, 32, [1,1], padding='VALID', stride=[1,1], # bn=True, is_training=is_training, scope='seg/conv5', is_dist=True) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, 16, [1,1], padding='VALID', stride=[1,1], activation_fn=None, scope='seg/output', is_dist=True) net = tf.squeeze(net, [2]) return net
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output BxNx50 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64 // 4, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64 // 4, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64 // 4) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) point_feat = tf.expand_dims(net_transformed, [2]) print(point_feat) net = tf_util.conv2d(point_feat, 64 // 4, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128 // 4, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024 // 4, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') print(global_feat) global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1]) concat_feat = tf.concat([point_feat, global_feat_expand], 3) print(concat_feat) net = tf_util.conv2d(concat_feat, 512 // 4, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv6', bn_decay=bn_decay) net = tf_util.conv2d(net, 256 // 4, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv7', bn_decay=bn_decay) net = tf_util.conv2d(net, 128 // 4, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv8', bn_decay=bn_decay) net = tf_util.conv2d(net, 128 // 4, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv9', bn_decay=bn_decay) net = tf_util.conv2d(net, 3, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conv10') #was hard-coded 50 net = tf.squeeze(net, [2]) # BxNxC return net, end_points
def get_model(pc_xyz, pc_features, is_training, num_class, bn_decay=None, use_t_net=True, pc_cls=None, cls_num=1, **kwargs): """ ConvNet baseline, input is BxNx3 gray image """ end_points = {} num_point = pc_xyz.shape[1] if use_t_net: with tf.variable_scope('transform_net1') as sc: transform = get_transform(pc_xyz, is_training, bn_decay) pc_xyz = tf.matmul(pc_xyz, transform) # B x N x 3 x 1 pc_xyz = tf.expand_dims(pc_xyz, -1) ## to B * N * 1 * 1 pc_features = tf.expand_dims(pc_features, -1) # to B x N x 4 x 1 pc_xyz = tf.concat(axis=2, values=[pc_xyz, pc_features]) # block one [64, 128, 128] K = 4 out1 = tf_util.conv2d(pc_xyz, 64, [1, K], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) out2 = tf_util.conv2d(out1, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) out3 = tf_util.conv2d(out2, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) if use_t_net: with tf.variable_scope('transform_net2') as sc: K = 128 transform = get_transform_K(out3, is_training, bn_decay, K) end_points['transform'] = transform net_transformed = tf.matmul(tf.reshape(out3, [-1, num_point, 128]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) else: net_transformed = out3 # block two [512, 2048] out4 = tf_util.conv2d(net_transformed, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) out5 = tf_util.conv2d(out4, 2048, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) out_max = tf_util.max_pool2d(out5, [num_point, 1], padding='VALID', scope='maxpool') # segmentation network if pc_cls is not None: # pc_labels.shape (batch_size, cls_num) one_hot_label_expand = tf.reshape(pc_cls, [-1, 1, 1, cls_num]) out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand]) out_max = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[out_max, out1, out2, out3, out4, out5]) # block [256, 256, 128] -> 2 net2 = tf_util.conv2d(concat, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv1') net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp1') net2 = tf_util.conv2d(net2, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2') net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp2') net2 = tf_util.conv2d(net2, 128, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3') net2 = tf_util.conv2d(net2, num_class, [1,1], padding='VALID', stride=[1,1], activation_fn=None, bn=False, scope='seg/conv4') net2 = tf.reshape(net2, [-1, num_point, num_class]) return net2, end_points
def get_model_my_model(point_cloud, is_training, bn_decay=None): batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} k = 20 adj_matrix = tf_util.pairwise_distance(point_cloud) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=k) with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(edge_feature, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) # Conv 1 adj_matrix = tf_util.pairwise_distance(point_cloud_transformed) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(point_cloud_transformed, nn_idx=nn_idx, k=k) net_local1 = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='dgcnn1', bn_decay=bn_decay) net_local1 = tf.reduce_max(net_local1, axis=-2, keep_dims=True) net_local1_intermediate = net_local1 net_local1 = tf_util.conv2d(net_local1, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='dgcnn1', bn_decay=bn_decay, activation_fn=None) net_local1 = tf.reduce_max(net_local1, axis=-2, keep_dims=True) net_local1 += net_local1_intermediate net_local1 = tf.nn.relu(net_local1) #net1 = net_local1 net_local_vector1 = tf_util.max_pool2d(net_local1, [num_point,1], padding='VALID', scope='maxpool1') input_image = tf.expand_dims(point_cloud_transformed, -1) net_global1 = tf_util.conv2d(input_image, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net_global_vector1 = tf_util.max_pool2d(net_global1, [num_point,1], padding='VALID', scope='maxpool1') points_feat1_concat = tf.concat(axis=-1, values=[net_global_vector1, net_local_vector1]) points_feat1_concat = tf.reduce_max(points_feat1_concat, axis=-2, keep_dims=True) # Conv 2 adj_matrix = tf_util.pairwise_distance(points_feat1_concat) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(points_feat1_concat, nn_idx=nn_idx, k=k) net_local2 = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='dgcnn2', bn_decay=bn_decay) net_local2 = tf.reduce_max(net_local2, axis=-2, keep_dims=True) #net2 = net_local2 net_local_vector2 = tf_util.max_pool2d(net_local2, [num_point,1], padding='VALID', scope='maxpool2') net_global2 = tf_util.conv2d(points_feat1_concat, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net_global_vector2 = tf_util.max_pool2d(net_global2, [num_point,1], padding='VALID', scope='maxpool2') points_feat2_concat = tf.concat(axis=-1, values=[net_global_vector2, net_local_vector2]) # Conv 3 adj_matrix = tf_util.pairwise_distance(points_feat2_concat) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(points_feat2_concat, nn_idx=nn_idx, k=k) net_local3 = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='dgcnn3', bn_decay=bn_decay) net_local3 = tf.reduce_max(net_local3, axis=-2, keep_dims=True) #net3 = net_local3 net_local_vector3 = tf_util.max_pool2d(net_local3, [num_point,1], padding='VALID', scope='maxpool3') net_global3 = tf_util.conv2d(points_feat2_concat, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net_global_vector3 = tf_util.max_pool2d(net_global3, [num_point,1], padding='VALID', scope='maxpool3') points_feat3_concat = tf.concat(axis=-1, values=[net_global_vector3, net_local_vector3]) # Conv 4 adj_matrix = tf_util.pairwise_distance(points_feat3_concat) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(points_feat3_concat, nn_idx=nn_idx, k=k) net_local4 = tf_util.conv2d(edge_feature, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='dgcnn4', bn_decay=bn_decay) net_local4 = tf.reduce_max(net_local4, axis=-2, keep_dims=True) #net4 = net_local4 net_local_vector4 = tf_util.max_pool2d(net_local4, [num_point,1], padding='VALID', scope='maxpool4') net_global4 = tf_util.conv2d(points_feat3_concat, 128, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net_global_vector4 = tf_util.max_pool2d(net_global4, [num_point,1], padding='VALID', scope='maxpool4') points_feat4_concat = tf.concat(axis=-1, values=[net_global_vector4, net_local_vector4]) # Conv 5 net_concat = tf_util.conv2d(tf.concat([points_feat1_concat, points_feat2_concat, points_feat3_concat, points_feat4_concat], axis=-1), 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetry Aggregation net_agg = tf_util.max_pool2d(net_concat, [num_point,1], padding='VALID', scope='maxpool_agg') net = tf.reshape(net_agg, [batch_size, -1]) #net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, # scope='fc1', bn_decay=bn_decay) #net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp1') #net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, # scope='fc2', bn_decay=bn_decay) #net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model_rbf(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) point_cloud_transformed = tf.expand_dims(point_cloud_transformed, 3) c1 = 1024 #centroids = tf.constant(np.random.randn(1, 1, 3, 1024), dtype=tf.float32) centroids = tf.get_variable('centroids', [1, 1, 3, c1], initializer=tf.constant_initializer( 0.5 * np.random.randn(1, 1, 3, c1)), dtype=tf.float32) #the per-centroids weights to change the shape of the multi-norm weights = tf.get_variable( 'weights', [1, 1, 4, c1], initializer=tf.constant_initializer(0.01 * np.random.randn(1, 1, 3, c1)), ) feature = tf.tile(point_cloud_transformed, [1, 1, 1, c1]) bias = tf.tile(centroids, [batch_size, num_point, 1, 1]) net = tf.subtract(feature, bias) net = tf.exp(net) net = tf.exp(-tf.concat( [ tf.norm(net, ord=0.5, axis=2, keep_dims=True), #tf.norm(net, ord=0.8, axis=2, keep_dims=True), tf.norm(net, ord=1, axis=2, keep_dims=True), #tf.norm(net, ord=1.5, axis=2, keep_dims=True), tf.norm(net, ord=2, axis=2, keep_dims=True), #tf.norm(net, ord=3, axis=2, keep_dims=True), #tf.norm(net, ord=4, axis=2, keep_dims=True), tf.norm(net, ord=np.inf, axis=2, keep_dims=True), ], axis=2)) net = tf.multiply(net, tf.tile(weights, [batch_size, num_point, 1, 1])) #net = tf.exp(-net) # Symmetric function: max pooling features = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.transpose(features, perm=[0, 1, 3, 2]) net = tf_util.conv2d(net, 3, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='mini_conv1', bn_decay=bn_decay) net = tf.reshape(net, [batch_size, -1]) #net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training, # scope='fc0', bn_decay=bn_decay) #net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp1') net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points, features, centroids
def get_model(point_cloud, is_training, bn_decay=None): """ ConvNet baseline, input is BxNx9 gray image """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) point_cloud_xyz = point_cloud[:, :, 6:] edge_feature, idx = tf_util.pointSIFT_KNN(0.15, point_cloud_xyz) _, xyz = tf_util.group(point_cloud_xyz, idx) feature, _ = tf_util.group(point_cloud, idx) net = tf_util.ngcn_chebyshev(inputs=feature, num_output_channels=64, local_cord=xyz, bn=False, is_training=is_training, scope='spec_conv%d' % (0), bn_decay=bn_decay) net_1 = tf.reduce_max(net, axis=-2, keep_dims=True) feature, _ = tf_util.group(tf.squeeze(net_1, axis=-2), idx) net = tf_util.ngcn_chebyshev(inputs=feature, num_output_channels=64, local_cord=xyz, bn=False, is_training=is_training, scope='spec_conv%d' % (1), bn_decay=bn_decay) net_2 = tf.reduce_max(net, axis=-2, keep_dims=True) feature, _ = tf_util.group(tf.squeeze(net_2, axis=-2), idx) net = tf_util.ngcn_chebyshev(inputs=feature, num_output_channels=128, local_cord=xyz, bn=False, is_training=is_training, scope='spec_conv%d' % (2), bn_decay=bn_decay) net_3 = tf.reduce_max(net, axis=-2, keep_dims=True) out7 = tf_util.conv2d(tf.concat([net_1, net_2, net_3], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='adj_conv7', bn_decay=bn_decay, is_dist=True) out_max = tf_util.max_pool2d(out7, [num_point, 1], padding='VALID', scope='maxpool') expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[expand, net_1, net_2, net_3]) # CONV net = tf_util.conv2d(concat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='seg/conv1', is_dist=True) net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='seg/conv2', is_dist=True) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, 13, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='seg/conv3', is_dist=True) net = tf.squeeze(net, [2]) return net
def get_model_elm(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) random_weights = tf.constant(np.random.randn(3, 4096), dtype=tf.float32) random_weights1 = tf.expand_dims(random_weights, 0) random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #2 random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #4 random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #8 random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #16 random_weights1 = tf.concat([random_weights1, random_weights1], axis=0) #32 net = tf.matmul(point_cloud, random_weights1) net = tf.expand_dims(net, 2) # Symmetric function: max pooling features = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(features, [batch_size, -1]) net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training, scope='fc0', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points, features
def get_model(point_cloud, input_label, is_training, cat_num, part_num, \ batch_size, num_point, weight_decay, graphnum, featnum, bn_decay=None): batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) k = 30 adj = tf_util.pairwise_distance(point_cloud) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(input_image, nn_idx=nn_idx, k=k) with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(edge_feature, is_training, bn_decay, K=3, is_dist=True) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -2) adj = tf_util.pairwise_distance(point_cloud_transformed) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(input_image, nn_idx=nn_idx, k=k) out1 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv1', bn_decay=bn_decay, is_dist=True) out2 = tf_util.conv2d(out1, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv2', bn_decay=bn_decay, is_dist=True) net_max_1 = tf.reduce_max(out2, axis=-2, keep_dims=True) net_mean_1 = tf.reduce_mean(out2, axis=-2, keep_dims=True) out3 = tf_util.conv2d(tf.concat([net_max_1, net_mean_1], axis=-1), 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv3', bn_decay=bn_decay, is_dist=True) out3A, net_max_1A, net_mean_1A = tf_util.offset_deform( input_image, out3, scope="trans_conv0", num_neighbor=k, num_graph=graphnum[0], num_feat=featnum[0], weight_decay=weight_decay, is_training=is_training, bn_decay=bn_decay) out5, net_max_2, net_mean_2 = tf_util.offset_deform( input_image, out3A, scope="trans_conv1", num_neighbor=k, num_graph=graphnum[0], num_feat=featnum[0], weight_decay=weight_decay, is_training=is_training, bn_decay=bn_decay) out7, net_max_3, net_mean_3 = tf_util.offset_deform( input_image, out5, scope="trans_conv2", num_neighbor=k, num_graph=graphnum[1], num_feat=featnum[1], weight_decay=weight_decay, is_training=is_training, bn_decay=bn_decay) '''adj = tf_util.pairwise_distance(tf.squeeze(trans2, axis=-2)) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(tf.concat([out5,trans2], axis = -1), nn_idx=nn_idx, k=k) out6 = tf_util.conv2d(edge_feature, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv6', bn_decay=bn_decay, is_dist=True) net_max_3 = tf.reduce_max(out6, axis=-2, keep_dims=True) net_mean_3 = tf.reduce_mean(out6, axis=-2, keep_dims=True) out7 = tf_util.conv2d(tf.concat([net_max_3, net_mean_3], axis=-1), 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv7', bn_decay=bn_decay, is_dist=True)''' out8 = tf_util.conv2d(tf.concat([out3, out5, out7], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='adj_conv13', bn_decay=bn_decay, is_dist=True) out_max = tf_util.max_pool2d(out8, [num_point, 1], padding='VALID', scope='maxpool') one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num]) one_hot_label_expand = tf_util.conv2d(one_hot_label_expand, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='one_hot_label_expand', bn_decay=bn_decay, is_dist=True) out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand]) expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[ expand, net_max_1, net_mean_1, out3, net_max_2, net_mean_2, out5, net_max_3, net_mean_3, out7, out8 ]) net2 = tf_util.conv2d(concat, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay, is_dist=True) net2 = tf_util.dropout(net2, keep_prob=0.6, is_training=is_training, scope='seg/dp1') net2 = tf_util.conv2d(net2, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay, is_dist=True) net2 = tf_util.dropout(net2, keep_prob=0.6, is_training=is_training, scope='seg/dp2') net2 = tf_util.conv2d(net2, 128, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay, is_dist=True) net2 = tf_util.conv2d(net2, part_num, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, bn=False, scope='seg/conv4', weight_decay=weight_decay, is_dist=True) net2 = tf.reshape(net2, [batch_size, num_point, part_num]) return net2
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2', reuse=tf.AUTO_REUSE) as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling features = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(features, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points, features
def get_3d_box_estimation_v1_net(object_point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D Box Estimation PointNet v1 network. Input: object_point_cloud: TF tensor in shape (B,M,C) point clouds in object coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type Output: output: TF tensor in shape (B,3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER*4) including box centers, heading bin class scores and residuals, and size cluster scores and residuals ''' num_point = object_point_cloud.get_shape()[1].value net = tf.expand_dims(object_point_cloud, 2) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv-reg1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv-reg2', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv-reg3', bn_decay=bn_decay) net = tf_util.conv2d(net, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv-reg4', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool2') net = tf.squeeze(net, axis=[1, 2]) net = tf.concat([net, one_hot_vec], axis=1) net = tf_util.fully_connected(net, 512, scope='fc1', bn=True, is_training=is_training, bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, scope='fc2', bn=True, is_training=is_training, bn_decay=bn_decay) # The first 3 numbers: box center coordinates (cx,cy,cz), # the next NUM_HEADING_BIN*2: heading bin class scores and bin residuals # next NUM_SIZE_CLUSTER*4: box cluster scores and residuals output = tf_util.fully_connected(net, 3 + NUM_HEADING_BIN * 2 + NUM_SIZE_CLUSTER * 4, activation_fn=None, scope='fc3') return output, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ ConvNet baseline, input is BxNx9 gray image """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) k = 20 weight_decay = 0 # 添加的 adj = tf_util.pairwise_distance(point_cloud[:, :, 6:]) # in:B*N*3 out:B*N*N nn_idx = tf_util.knn(adj, k=k) # (batch, num_points, k) # out:B*N*K edge_feature = tf_util.get_edge_feature(input_image, nn_idx=nn_idx, k=k) # out:B*N*K*18 out1 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv1', bn_decay=bn_decay, is_dist=True) out2 = tf_util.conv2d(out1, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv2', bn_decay=bn_decay, is_dist=True) net_1 = tf.reduce_max(out2, axis=-2, keep_dims=True) adj = tf_util.pairwise_distance(net_1) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(net_1, nn_idx=nn_idx, k=k) out3 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv3', bn_decay=bn_decay, is_dist=True) out4 = tf_util.conv2d(out3, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv4', bn_decay=bn_decay, is_dist=True) net_2 = tf.reduce_max(out4, axis=-2, keep_dims=True) adj = tf_util.pairwise_distance(net_2) nn_idx = tf_util.knn(adj, k=k) edge_feature = tf_util.get_edge_feature(net_2, nn_idx=nn_idx, k=k) out5 = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, weight_decay=weight_decay, scope='adj_conv5', bn_decay=bn_decay, is_dist=True) # out6 = tf_util.conv2d(out5, 64, [1,1], # padding='VALID', stride=[1,1], # bn=True, is_training=is_training, weight_decay=weight_decay, # scope='adj_conv6', bn_decay=bn_decay, is_dist=True) net_3 = tf.reduce_max(out5, axis=-2, keep_dims=True) out7 = tf_util.conv2d(tf.concat([net_1, net_2, net_3], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='adj_conv7', bn_decay=bn_decay, is_dist=True) out_max = tf_util.max_pool2d(out7, [num_point, 1], padding='VALID', scope='maxpool') expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[expand, net_1, net_2, net_3]) # CONV net = tf_util.conv2d(concat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='seg/conv1', is_dist=True) net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='seg/conv2', is_dist=True) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, 13, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='seg/conv3', is_dist=True) net = tf.squeeze(net, [2]) return net
def get_instance_seg_v1_net(point_cloud, one_hot_vec, is_training, bn_decay, end_points): ''' 3D instance segmentation PointNet v1 network. Input: point_cloud: TF tensor in shape (B,N,4) frustum point clouds with XYZ and intensity in point channels XYZs are in frustum coordinate one_hot_vec: TF tensor in shape (B,3) length-3 vectors indicating predicted object type is_training: TF boolean scalar bn_decay: TF float scalar end_points: dict Output: logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object end_points: dict ''' batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value net = tf.expand_dims(point_cloud, 2) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) point_feat = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(point_feat, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') global_feat = tf.concat( [global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3) global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1]) concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand]) net = tf_util.conv2d(concat_feat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv6', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv7', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv8', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv9', bn_decay=bn_decay) net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5) logits = tf_util.conv2d(net, 2, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conv10') logits = tf.squeeze(logits, [2]) # BxNxC return logits, end_points
def input_transform_net(point_cloud, is_training, bn_decay=None, K=3): #输入(XYZ)变换网络,输入是BxNx3灰度图像 #返回:大小为3xK的变换矩阵 """ Input (XYZ) Transform Net, input is BxNx3 gray image Return: Transformation matrix of size 3xK """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv3', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: assert (K == 3) weights = tf.get_variable('weights', [256, 3 * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [3 * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant([1, 0, 0, 0, 1, 0, 0, 0, 1], dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, 3, K]) return transform
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Point functions (MLP implemented as conv2d) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def input_transform_net(edge_feature, is_training, bn_decay=None, K=3): """ Input (XYZ) Transform Net, input is BxNx3 gray image Return: Transformation matrix of size 3xK """ batch_size = edge_feature.get_shape()[0].value num_point = edge_feature.get_shape()[1].value # input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv2', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keep_dims=True) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv3', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: # assert(K==3) with tf.device('/cpu:0'): weights = tf.get_variable('weights', [256, K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant(np.eye(K).flatten(), dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, K, K]) return transform
def get_model_dg(point_cloud, is_training, bn_decay=None, K=4): """ Input (XYZ) Transform Net, input is BxNx3 gray image Return: Transformation matrix of size 3xK """ k = 20 adj_matrix = tf_util.pairwise_distance(point_cloud) nn_idx = tf_util.knn(adj_matrix, k=k) edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=k) batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value #input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv2', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keep_dims=True) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv3', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: # assert(K==3) weights = tf.get_variable('weights', [256, 4], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [4], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant([0, 0, 0, 1], dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, 4]) return transform
def get_model(point_cloud, is_training, bn_decay=None, K=4): """ Input (XYZ) Transform Net, input is BxNx3 gray image Return: Transformation matrix of size 3xK """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value #num_point = tf.shape(point_cloud)[1] input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv3', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') #net = tf.reduce_max(net, axis=[2]) net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: # assert(K==3) weights = tf.get_variable('weights', [256, K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [4], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant([0, 0, 0, 1], dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, 4]) transform = tf.nn.l2_normalize(transform, dim=1) return transform
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) #print(transform.shape) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) #find the maxmun and minmun input_image_add = tf.reduce_sum(input_image, 2, keep_dims=True) order = np.zeros([1,1024]) order[1,1] = tf.argmax(input_image_add, axis=1) order[1,-1] = tf.argmin(input_image_add, axis=1) print() net = tf_util.conv2d(input_image, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) #print(net.shape) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) #print(net.shape) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) #print(net.shape) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Autoencoder for point clouds. Input: point_cloud: TF tensor BxNx3 is_training: boolean bn_decay: float between 0 and 1 Output: net: TF tensor BxNx3, reconstructed point clouds end_points: dict """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value assert (num_point == 2048) point_dim = point_cloud.get_shape()[2].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Encoder net = tf_util.conv2d(input_image, 64, [1, point_dim], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) point_feat = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(point_feat, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(global_feat, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc00', bn_decay=bn_decay) embedding = tf.reshape(net, [batch_size, -1]) end_points['embedding'] = embedding # FC Decoder net = tf_util.fully_connected(embedding, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.fully_connected(net, 1024 * 3, activation_fn=None, scope='fc3') pc_fc = tf.reshape(net, (batch_size, -1, 3)) # UPCONV Decoder net = tf.reshape(embedding, [batch_size, 1, 1, -1]) net = tf_util.conv2d_transpose(net, 512, kernel_size=[2, 2], stride=[1, 1], padding='VALID', scope='upconv1', bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.conv2d_transpose(net, 256, kernel_size=[3, 3], stride=[1, 1], padding='VALID', scope='upconv2', bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.conv2d_transpose(net, 256, kernel_size=[4, 4], stride=[2, 2], padding='VALID', scope='upconv3', bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.conv2d_transpose(net, 128, kernel_size=[5, 5], stride=[3, 3], padding='VALID', scope='upconv4', bn=True, bn_decay=bn_decay, is_training=is_training) net = tf_util.conv2d_transpose(net, 3, kernel_size=[1, 1], stride=[1, 1], padding='VALID', scope='upconv5', activation_fn=None) end_points['xyzmap'] = net pc_upconv = tf.reshape(net, [batch_size, -1, 3]) # Set union net = tf.concat(values=[pc_fc, pc_upconv], axis=1) return net, end_points
def get_model_point_net(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Point functions (MLP implemented as conv2d) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: # assert(K==3) weights = tf.get_variable('weights', [256, 4], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [4], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant([0, 0, 0, 1], dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, 4]) # net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp1') # net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return transform #, end_points
def get_model(point_cloud, input_label, is_training, cat_num, part_num, \ batch_size, num_point, weight_decay, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ end_points = {} with tf.variable_scope('transform_net1') as sc: K = 3 transform = get_transform(point_cloud, is_training, bn_decay, K = 3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) out1 = tf_util.conv2d(input_image, 64, [1,K], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) out2 = tf_util.conv2d(out1, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) out3 = tf_util.conv2d(out2, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: K = 128 transform = get_transform_K(out3, is_training, bn_decay, K) end_points['transform'] = transform squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128]) net_transformed = tf.matmul(squeezed_out3, transform) net_transformed = tf.expand_dims(net_transformed, [2]) out4 = tf_util.conv2d(net_transformed, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) out5 = tf_util.conv2d(out4, 2048, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) out_max = tf_util.max_pool2d(out5, [num_point,1], padding='VALID', scope='maxpool') # classification network net = tf.reshape(out_max, [batch_size, -1]) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='cla/dp1') net = tf_util.fully_connected(net, cat_num, activation_fn=None, scope='cla/fc3') # segmentation network one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num]) out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand]) expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5]) net2 = tf_util.conv2d(concat, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp1') net2 = tf_util.conv2d(net2, 256, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp2') net2 = tf_util.conv2d(net2, 128, [1,1], padding='VALID', stride=[1,1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay) net2 = tf_util.conv2d(net2, part_num, [1,1], padding='VALID', stride=[1,1], activation_fn=None, bn=False, scope='seg/conv4', weight_decay=weight_decay) net2 = tf.reshape(net2, [batch_size, num_point, part_num]) return net, net2, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ Autoencoder for point clouds. Input: point_cloud: TF tensor BxNxC is_training: boolean bn_decay: float between 0 and 1 Output: net: TF tensor BxNxC, reconstructed point clouds end_points: dict """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value point_dim = point_cloud.get_shape()[2].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Encoder net = tf_util.conv2d(input_image, 64, [1, point_dim], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) point_feat = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(point_feat, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(global_feat, [batch_size, -1]) end_points['embedding'] = net # FC Decoder net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 1024, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.fully_connected(net, num_point * 3, activation_fn=None, scope='fc3') net = tf.reshape(net, (batch_size, num_point, 3)) return net, end_points
def get_model(point_cloud, is_training, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ #import pdb #pdb.set_trace() batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) # CONV net = tf_util.conv2d(input_image, 64, [1, num], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) #import pdb #pdb.set_trace() points_feat1 = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # MAX pc_feat1 = tf_util.max_pool2d(points_feat1, [num_point, 1], padding='VALID', scope='maxpool1') # FC pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1]) pc_feat1 = tf_util.fully_connected(pc_feat1, 256, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) pc_feat1 = tf_util.fully_connected(pc_feat1, 128, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) print(pc_feat1) # CONCAT pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]), [1, num_point, 1, 1]) points_feat1_concat = tf.concat(axis=3, values=[points_feat1, pc_feat1_expand]) # CONV net = tf_util.conv2d(points_feat1_concat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv6') net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv7') net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, classes, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conv8') net = tf.squeeze(net, [2]) return net
def get_model(point_cloud, one_hot_vec, is_training, bn_decay=None): """ Classification PointNet, input is BxNx4, onehotvec is Bx3, output BxNx2 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(input_image, 64, [1,6], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) point_feat = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(point_feat, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') print global_feat global_feat = tf.concat([global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3) print 'Global Feat: ', global_feat global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1]) print point_feat, global_feat_expand concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand]) print concat_feat net = tf_util.conv2d(concat_feat, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv6', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv7', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv8', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv9', bn_decay=bn_decay) net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5) logits = tf_util.conv2d(net, 2, [1,1], padding='VALID', stride=[1,1], activation_fn=None, scope='conv10') logits = tf.squeeze(logits, [2]) # BxNxC print logits print '-----------' #net = tf.concat(axis=3, values=[net, tf.expand_dims(tf.slice(point_cloud, [0,0,0], [-1,-1,3]), 2)]) mask = tf.slice(logits,[0,0,0],[-1,-1,1]) < tf.slice(logits,[0,0,1],[-1,-1,1]) mask = tf.to_float(mask) # BxNx1 mask_count = tf.tile(tf.reduce_sum(mask,axis=1,keep_dims=True), [1,1,3]) # Bx1x3 print mask point_cloud_xyz = tf.slice(point_cloud, [0,0,0], [-1,-1,3]) # BxNx3 # ---- Subtract points mean ---- mask_xyz_mean = tf.reduce_sum(tf.tile(mask, [1,1,3])*point_cloud_xyz, axis=1, keep_dims=True) # Bx1x3 mask_xyz_mean = mask_xyz_mean/tf.maximum(mask_count,1) # Bx1x3 point_cloud_xyz_stage1 = point_cloud_xyz - tf.tile(mask_xyz_mean, [1,num_point,1]) print 'Point cloud xyz stage1: ', point_cloud_xyz_stage1 # ---- Regress 1st stage center ---- net = tf.expand_dims(point_cloud_xyz_stage1, 2) print net net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg1-stage1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg2-stage1', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg3-stage1', bn_decay=bn_decay) mask_expand = tf.tile(tf.expand_dims(mask,-1), [1,1,1,256]) masked_net = net*mask_expand print masked_net net = tf_util.max_pool2d(masked_net, [num_point,1], padding='VALID', scope='maxpool-stage1') net = tf.squeeze(net, axis=[1,2]) print net net = tf.concat([net, one_hot_vec], axis=1) net = tf_util.fully_connected(net, 256, scope='fc1-stage1', bn=True, is_training=is_training, bn_decay=bn_decay) net = tf_util.fully_connected(net, 128, scope='fc2-stage1', bn=True, is_training=is_training, bn_decay=bn_decay) stage1_center = tf_util.fully_connected(net, 3, activation_fn=None, scope='fc3-stage1') stage1_center = stage1_center + tf.squeeze(mask_xyz_mean, axis=1) # Bx3 end_points['stage1_center'] = stage1_center # ---- Subtract stage1 center ---- point_cloud_xyz_submean = point_cloud_xyz - tf.expand_dims(stage1_center, 1) print 'Point cloud xyz submean: ', point_cloud_xyz_submean net = tf.expand_dims(point_cloud_xyz_submean, 2) print net net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg2', bn_decay=bn_decay) net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg3', bn_decay=bn_decay) net = tf_util.conv2d(net, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv-reg4', bn_decay=bn_decay) mask_expand = tf.tile(tf.expand_dims(mask,-1), [1,1,1,512]) masked_net = net*mask_expand print masked_net net = tf_util.max_pool2d(masked_net, [num_point,1], padding='VALID', scope='maxpool2') net = tf.squeeze(net, axis=[1,2]) print net net = tf.concat([net, one_hot_vec], axis=1) net = tf_util.fully_connected(net, 512, scope='fc1', bn=True, is_training=is_training, bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, scope='fc2', bn=True, is_training=is_training, bn_decay=bn_decay) # First 3 are cx,cy,cz, next NUM_HEADING_BIN*2 are for heading # next NUM_SIZE_CLUSTER*4 are for dimension output = tf_util.fully_connected(net, 3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER*4, activation_fn=None, scope='fc3') print output center = tf.slice(output, [0,0], [-1,3]) center = center + stage1_center # Bx3 end_points['center'] = center heading_scores = tf.slice(output, [0,3], [-1,NUM_HEADING_BIN]) heading_residuals_normalized = tf.slice(output, [0,3+NUM_HEADING_BIN], [-1,NUM_HEADING_BIN]) end_points['heading_scores'] = heading_scores # BxNUM_HEADING_BIN end_points['heading_residuals_normalized'] = heading_residuals_normalized # BxNUM_HEADING_BIN (should be -1 to 1) end_points['heading_residuals'] = heading_residuals_normalized * (np.pi/NUM_HEADING_BIN) # BxNUM_HEADING_BIN size_scores = tf.slice(output, [0,3+NUM_HEADING_BIN*2], [-1,NUM_SIZE_CLUSTER]) # BxNUM_SIZE_CLUSTER size_residuals_normalized = tf.slice(output, [0,3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER], [-1,NUM_SIZE_CLUSTER*3]) size_residuals_normalized = tf.reshape(size_residuals_normalized, [batch_size, NUM_SIZE_CLUSTER, 3]) # BxNUM_SIZE_CLUSTERx3 end_points['size_scores'] = size_scores end_points['size_residuals_normalized'] = size_residuals_normalized end_points['size_residuals'] = size_residuals_normalized * tf.expand_dims(tf.constant(mean_size_arr, dtype=tf.float32), 0) return logits, end_points