def ae_decoder(batch_size, num_point, net, is_training, bn_decay=None, reuse=False): net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay, reuse=reuse) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay, reuse=reuse) net = tf_util.fully_connected(net, num_point * 3, activation_fn=None, scope='fc3', reuse=reuse) net = tf.reshape(net, (batch_size, num_point, 3)) return net
def create_encoder(self, inputs): input_image = tf.expand_dims(inputs, -1) # Conv net = conv2d(input_image, 64, [1, 9], padding='VALID', stride=[1, 1], bn=True, is_training=self.is_training, scope='conv1', bn_decay=self.bn_decay) net = conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=self.is_training, scope='conv2', bn_decay=self.bn_decay) net = conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=self.is_training, scope='conv3', bn_decay=self.bn_decay) net = conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=self.is_training, scope='conv4', bn_decay=self.bn_decay) points_feat1 = conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=self.is_training, scope='conv5', bn_decay=self.bn_decay) # MaxPooling pc_feat1 = max_pool2d(points_feat1, [NUM_POINT, 1], padding='VALID', scope='maxpool') # Fully Connected Layers pc_feat1 = tf.reshape(pc_feat1, [BATCH_SIZE, -1]) pc_feat1 = fully_connected(pc_feat1, 256, bn=True, is_training=self.is_training, scope='fc1', bn_decay=self.bn_decay) pc_feat1 = fully_connected(pc_feat1, 128, bn=True, is_training=self.is_training, scope='fc2', bn_decay=self.bn_decay) # Concat pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [BATCH_SIZE, 1, 1, -1]), [1, NUM_POINT, 1, 1]) points_feat1_concat = tf.concat(axis=3, values=[points_feat1, pc_feat1_expand]) return points_feat1_concat
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNetwork :param point_cloud: point cloud BxNx3 :param is_training: training flag :param bn_decay: decay flag :return: output model Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # Point functions (MPL implemented as conv2d) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symetric function: Max Pooling net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def create_decoder(self, features): """fully connected layers for classification with dropout""" with tf.variable_scope('decoder_cls', reuse=tf.AUTO_REUSE): features = fully_connected(features, 512, bn=True, scope='fc1', is_training=self.is_training) features = dropout(features, keep_prob=0.7, scope='dp1', is_training=self.is_training) features = fully_connected(features, 256, bn=True, scope='fc2', is_training=self.is_training) features = dropout(features, keep_prob=0.7, scope='dp2', is_training=self.is_training) pred = fully_connected(features, NUM_CLASSES, activation_fn=None, scope='fc3', is_training=self.is_training) return pred
def get_transform(point_cloud, is_training, bn_decay=None, K=3): """ Transform Net, input is BxNx3 gray image Return: Transformation matrix of size 3xK """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv4', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 128, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 128, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: assert (K == 3) weights = tf.get_variable('weights', [128, 3 * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [3 * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) + tf.constant([1, 0, 0, 0, 1, 0, 0, 0, 1], dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) # transform = tf_util.fully_connected(net, 3*K, activation_fn=None, scope='tfc3') transform = tf.reshape(transform, [batch_size, 3, K]) return transform
def get_transform_K(inputs, is_training, bn_decay=None, K=3): """ Transform Net, input is BxNx1xK gray image Return: Transformation matrix of size KxK """ batch_size = inputs.get_shape()[0].value num_point = inputs.get_shape()[1].value net = tf_util.conv2d(inputs, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv2', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_feat') as sc: weights = tf.get_variable('weights', [256, K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) + tf.constant(np.eye(K).flatten(), dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) # transform = tf_util.fully_connected(net, 3*K, activation_fn=None, scope='tfc3') transform = tf.reshape(transform, [batch_size, K, K]) return transform
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1,3], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) net_transformed = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(net_transformed, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point,1], padding='VALID', scope='maxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def input_transform_net(point_cloud, is_training, bn_decay=None, K=3): """ Input (XYZ) Transform Net, input is BxNx3 gray image Return: Transformation matrix of size 3xK """ # print('the input shape for t-net:', point_cloud.get_shape()) batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value # point_cloud -> Tensor of (batch size, number of points, 3d coordinates) input_image = tf.expand_dims(point_cloud, -1) # point_cloud -> (batch size, number of points, 3d coordinates, 1) # batch size * height * width * channel '''tf_util.conv2d(inputs, num_output_channels, kernel_size, scope, stride=[1, 1], padding='SAME', use_xavier=True, stddev=1e-3, weight_decay=0.0, activation_fn=tf.nn.relu, bn=False, bn_decay=None(default is set to 0.9), is_training=None)''' net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv3', bn_decay=bn_decay) # net = mlp_conv(input_image, [64, 128, 1024]) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') '''(default stride: (2, 2))''' # net = tf.reduce_max(net, axis=1, keep_dims=True, name='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: assert(K == 3) weights = tf.get_variable('weights', [256, 3*K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [3*K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant([1, 0, 0, 0, 1, 0, 0, 0, 1], dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, 3, K]) return transform
def input_transform_net_dgcnn(edge_feature, is_training, bn_decay=None, K=3): """ Input (XYZ) Transform Net, input is BxNx3 gray image Return: Transformation matrix of size 3xK """ batch_size = edge_feature.get_shape()[0].value num_point = edge_feature.get_shape()[1].value # input_image = tf.expand_dims(point_cloud, -1) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv2', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keep_dims=True) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv3', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.variable_scope('transform_XYZ') as sc: # assert(K==3) with tf.device('/cpu:0'): weights = tf.get_variable('weights', [256, K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant(np.eye(K).flatten(), dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, K, K]) return transform
def get_geom_model(pcl, is_training): ''' Build the graph for the shape processing branch ''' # first get shape feature pointnet_feat = get_pointnet_model(pcl, is_training, bn_decay=bn_decay) # process pointnet output pt_vec = tf_util.fully_connected(pointnet_feat, 1024, weight_decay=weight_decay, bn=True, \ is_training=is_training, scope='geom_fc1', bn_decay=bn_decay) pt_vec = tf_util.fully_connected(pt_vec, 512, weight_decay=weight_decay, bn=True, \ is_training=is_training, scope='geom_fc2', bn_decay=bn_decay) pt_vec = tf_util.fully_connected(pt_vec, 128, weight_decay=weight_decay, bn=True, \ is_training=is_training, scope='geom_fc3', bn_decay=bn_decay) pt_vec = tf_util.fully_connected(pt_vec, 32, weight_decay=weight_decay, bn=True, \ is_training=is_training, scope='geom_fc4', bn_decay=bn_decay) shape_feat = tf_util.fully_connected(pt_vec, pcl_feat_size, weight_decay=weight_decay, bn=True, \ is_training=is_training, scope='geom_fc5', bn_decay=bn_decay) return shape_feat
def get_model(point_cloud, is_training, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) # CONV net = tf_util.conv2d(input_image, 64, [1,9], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) points_feat1 = tf_util.conv2d(net, 1024, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # MAX pc_feat1 = tf_util.max_pool2d(points_feat1, [num_point,1], padding='VALID', scope='maxpool1') # FC pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1]) pc_feat1 = tf_util.fully_connected(pc_feat1, 256, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) pc_feat1 = tf_util.fully_connected(pc_feat1, 128, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) print(pc_feat1) # CONCAT pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]), [1, num_point, 1, 1]) print(points_feat1) print(pc_feat1_expand) points_feat1_concat = tf.concat(axis=3, values=[points_feat1, pc_feat1_expand]) print(points_feat1_concat) # CONV net = tf_util.conv2d(points_feat1_concat, 512, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv6') net = tf_util.conv2d(net, 256, [1,1], padding='VALID', stride=[1,1], bn=True, is_training=is_training, scope='conv7') net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, 13, [1,1], padding='VALID', stride=[1,1], activation_fn=None, scope='conv8') net = tf.squeeze(net, [2]) return net
def create_decoder(self, features): """fully connected layers for classification with dropout""" with tf.variable_scope('decoder_cls', reuse=tf.AUTO_REUSE): # self.linear1 = nn.Linear(args.emb_dims*2, 512, bias=False) features = tf_util.fully_connected(features, 512, bn=True, bias=False, activation_fn=tf.nn.leaky_relu, scope='linear1', is_training=self.is_training) features = tf_util.dropout(features, keep_prob=0.5, scope='dp1', is_training=self.is_training) # self.linear2 = nn.Linear(512, 256) features = tf_util.fully_connected(features, 256, bn=True, bias=True, activation_fn=tf.nn.leaky_relu, scope='linear2', is_training=self.is_training) features = tf_util.dropout(features, keep_prob=0.5, scope='dp2', is_training=self.is_training) # self.linear3 = nn.Linear(256, output_channels) pred = tf_util.fully_connected(features, NUM_CLASSES, bn=False, bias=True, activation_fn=None, scope='linear3', is_training=self.is_training) return pred
def get_model(point_cloud, is_training, num_class, bn_decay=None): """ Semantic segmentation PointNet, input is BxNxF, output Bxnum_class """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} input_image = tf.expand_dims(point_cloud, -1) # CONV layers net = tf_util.conv2d(input_image, 64, [1, 9], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) points_feat1 = tf_util.conv2d(net, 2048, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # MAX Pooling pc_feat1 = tf_util.max_pool2d(points_feat1, [num_point, 1], padding='VALID', scope='maxpool1') # FC layers pc_feat1 = tf.reshape(pc_feat1, [batch_size, -1]) pc_feat1 = tf_util.fully_connected(pc_feat1, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) pc_feat1 = tf_util.fully_connected(pc_feat1, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) # CONCAT pc_feat1_expand = tf.tile(tf.reshape(pc_feat1, [batch_size, 1, 1, -1]), [1, num_point, 1, 1]) points_feat1_concat = tf.concat(axis=3, values=[points_feat1, pc_feat1_expand]) # CONV Layers net = tf_util.conv2d(points_feat1_concat, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv6') net = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv7') net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv8') net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv9') net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.conv2d(net, num_class, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conv10') net = tf.squeeze(net, [2]) return net, end_points
def feature_transform_net(inputs, is_training, bn_decay=None, K=64): """ Feature Transform Net, input is BxNx1xK Return: Transformation matrix of size KxK """ batch_size = inputs.get_shape()[0] # .value num_point = inputs.get_shape()[1] # .value net = tf_util.conv2d(inputs, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='tconv3', bn_decay=bn_decay) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='tfc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='tfc2', bn_decay=bn_decay) with tf.compat.v1.variable_scope('transform_feat') as sc: weights = tf.compat.v1.get_variable( 'weights', [256, K * K], initializer=tf.compat.v1.constant_initializer(0.0), dtype=tf.float32) biases = tf.compat.v1.get_variable( 'biases', [K * K], initializer=tf.compat.v1.constant_initializer(0.0), dtype=tf.float32) biases.assign_add(tf.constant(np.eye(K).flatten(), dtype=tf.float32)) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, K, K]) return transform
def get_model_other(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3 and BxNx1, output Bx4 """ batch_size = point_cloud.get_shape()[0] # .value num_point = point_cloud.get_shape()[1] # .value end_points = {} # transform net for input x,y,z with tf.compat.v1.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud[:, :, 1:4], is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud[:, :, 1:4], transform) input_image = tf.expand_dims(point_cloud_transformed, -1) # First MLP layers net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) # transform net for the features with tf.compat.v1.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) # point_feat = tf.expand_dims(net_transformed, [2]) # add the additional features to the second MLP layers # point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0:1], [2]) concat_other = tf.concat(axis=2, values=[ point_cloud[:, :, 0:1], net_transformed, point_cloud[:, :, 4:para.dim] ]) concat_other = tf.expand_dims(concat_other, [2]) # second MLP layers net = tf_util.conv2d( concat_other, 64, [1, 1], # 64 is the output #neuron padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 4, activation_fn=None, scope='fc3') return net, end_points
def get_model(inputs, is_training, bn_decay=None, num_class=40, FLAGS=None): """ Classification PointNet, input is BxNx3, output Bx40 """ point_cloud = inputs[:, :, 0:3] if FLAGS.normal: D = 6 points = inputs[:, :, 3:] else: D = 3 points = None # --------------------------------------- STN ------------------------------------- if FLAGS.STN: with tf.variable_scope('transform_net') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud = tf.matmul(point_cloud, transform) # ---------------------------------- Node Sampling -------------------------------- with tf.variable_scope('group_sampling') as sc: KNN = FLAGS.KNN point_cloud_sampled, nn_points, _, _ = sample_and_group( npoint=FLAGS.node_num, radius=0.2, nsample=KNN, xyz=point_cloud, points=points, knn=True, use_xyz=True) point_cloud_sampled = tf.expand_dims(point_cloud_sampled, axis=-1) net1 = tf_util.conv2d(point_cloud_sampled, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1_1', bn_decay=bn_decay) net1 = tf.tile(net1, multiples=[1, 1, KNN, 1]) net1 = tf.expand_dims(net1, axis=-2) nn_points = tf.expand_dims(nn_points, axis=-1) net = tf_util.conv3d(nn_points, 64, [1, 1, D], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv1_2', bn_decay=bn_decay) concat = tf.concat(values=[net, net1], axis=-1) net = tf_util.conv3d(concat, 128, [1, 1, 1], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv3d(net, 128, [1, 1, 1], padding='VALID', stride=[1, 1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) # ---------------------- local pooling: merge local feature ------------------------- if FLAGS.local_pool == 'average': pool_k = tf_util.avg_pool3d(net, kernel_size=[1, KNN, 1], stride=[1, 2, 2], padding='VALID', scope='pool_k') else: pool_k = tf_util.max_pool3d(net, kernel_size=[1, KNN, 1], stride=[1, 2, 2], padding='VALID', scope='pool_k') net = tf.squeeze(pool_k, axis=2) # ---------------------------------- VLAD layer -------------------------------------- net, index = VLAD(net, FLAGS, is_training, bn_decay, layer_name='VLAD') # -------------------------------- classification ------------------------------------ with tf.name_scope('fc_layer'): net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, num_class, activation_fn=None, scope='fc3') return net, index
def get_model_other(point_cloud, is_training, bn_decay=None): """ PointNet2 with multi-scale grouping Semantic segmentation network that uses feature propogation layers """ batch_size = point_cloud.get_shape()[0] # .value end_points = {} l0_xyz = point_cloud[:, :, 1:4] l0_points = tf.concat( axis=2, values=[point_cloud[:, :, 0:1], point_cloud[:, :, 4:para.dim]]) # Set abstraction layers # input B 1024 1 3 => 64+128+128 = 320 max pooling in small group n = 16 32 128 l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1') # , use_nchw=True # input B 512 320 => 128+256+256 = 640 max pooling in small group n = 32 64 128 l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2') # input B 128 640 => 1024, max pooling in all pointcloud = 128 # MLP layer to gather 3 scale features _, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # input B 1 1024 # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3') return net, end_points
def pointnet(self, point_cloud, is_training, bn=True, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bxn where n is num classes """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value input_image = tf.expand_dims(point_cloud, -1) # Point functions (MLP implemented as conv2d) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv2', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='conv5', bn_decay=bn_decay) # Symmetric function: max pooling net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') # MLP on global point cloud vector net = tf.layers.flatten(net) net = tf_util.fully_connected(net, 512, bn=bn, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=bn, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, self.n_classes, activation_fn=None, scope='fc3') return net
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value end_points = {} l0_xyz = point_cloud l0_points = None end_points['l0_xyz'] = l0_xyz # Set abstraction layers # Note: When using NCHW for layer 2, we see increased GPU memory usage (in TF1.4). # So we only use NCHW for layer 1 until this issue can be resolved. l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz, l0_points, npoint=512, radius=0.2, nsample=32, mlp=[64, 64, 128], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz, l1_points, npoint=128, radius=0.4, nsample=64, mlp=[128, 128, 256], mlp2=None, group_all=False, is_training=is_training, bn_decay=bn_decay, scope='layer2') l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.layers.flatten(l3_points) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def pct_model(point_cloud, is_training, bn_decay=None): """ :param point_cloud: :param is_training: :param bn_decay: :return: """ # point_cloud -> [batch_size, num_point, 3] batch_size = point_cloud.get_shape()[0].value point_dim = point_cloud.get_shape()[2].value """Input Embedding module""" # [batch_size, num_point, 64] x = tf_util.conv1d(point_cloud, 64, kernel_size=1, padding='VALID', stride=1, bn=True, is_biases=False, is_training=is_training, scope='conv0', bn_decay=bn_decay) x = tf_util.conv1d(x, 64, kernel_size=1, padding='VALID', stride=1, bn=True, is_biases=False, is_training=is_training, scope='conv1', bn_decay=bn_decay) """ Sample and Group """ new_xyz, new_feature, _, _ = pointnet_util.sample_and_group( npoint=512, radius=0.15, nsample=32, xyz=point_cloud, points=x, knn=True, use_xyz=True) # print(new_xyz.shape) # print("new_feature.shape", new_feature.shape) feature_0 = local_op(new_feature, out_dim=128, scope="SG1", bn_decay=bn_decay, is_training=is_training) new_xyz, new_feature, _, _ = pointnet_util.sample_and_group( npoint=256, radius=0.2, nsample=32, xyz=new_xyz, points=feature_0, knn=True, use_xyz=True) # NHC feature_1 = local_op(new_feature, out_dim=256, scope="SG2", bn_decay=bn_decay, is_training=is_training) # # NHC x = pt_last(feature_1, scope="pct_layer", out_dim=256, bn_decay=bn_decay, is_training=is_training) # concat in C (NHC) axis x = tf.concat([x, feature_1], axis=-1) x = tf_util.conv1d(x, 1024, kernel_size=1, padding='VALID', stride=1, bn=True, is_biases=False, is_training=is_training, scope='conv2', bn_decay=bn_decay, activation_fn=None) x = tf.nn.leaky_relu(x, alpha=0.2) x = tf.reduce_max(x, axis=1) """ ++++++++++++++++++++++++++++++++++++++++ Decoder ++++++++++++++++++++++++++++++++++++++++ """ x = tf_util.fully_connected(x, 512, bn=True, is_training=is_training, is_biases=False, scope='fc1', bn_decay=bn_decay) x = tf_util.dropout(x, keep_prob=0.5, is_training=is_training, scope='dp1') x = tf_util.fully_connected(x, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) x = tf_util.dropout(x, keep_prob=0.5, is_training=is_training, scope='dp2') x = tf_util.fully_connected(x, 40, activation_fn=None, scope='fc3') return x
def get_model(point_cloud, input_label, is_training, cat_num, part_num, \ batch_size, num_point, weight_decay, bn_decay=None): """ ConvNet baseline, input is BxNx3 gray image """ end_points = {} with tf.variable_scope('transform_net1') as sc: K = 3 transform = get_transform(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) out1 = tf_util.conv2d(input_image, 64, [1, K], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) out2 = tf_util.conv2d(out1, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) out3 = tf_util.conv2d(out2, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: K = 128 transform = get_transform_K(out3, is_training, bn_decay, K) end_points['transform'] = transform squeezed_out3 = tf.reshape(out3, [batch_size, num_point, 128]) net_transformed = tf.matmul(squeezed_out3, transform) net_transformed = tf.expand_dims(net_transformed, [2]) out4 = tf_util.conv2d(net_transformed, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) out5 = tf_util.conv2d(out4, 2048, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) out_max = tf_util.max_pool2d(out5, [num_point, 1], padding='VALID', scope='maxpool') # classification network net = tf.reshape(out_max, [batch_size, -1]) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='cla/fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, scope='cla/dp1') net = tf_util.fully_connected(net, cat_num, activation_fn=None, scope='cla/fc3') # segmentation network one_hot_label_expand = tf.reshape(input_label, [batch_size, 1, 1, cat_num]) out_max = tf.concat(axis=3, values=[out_max, one_hot_label_expand]) expand = tf.tile(out_max, [1, num_point, 1, 1]) concat = tf.concat(axis=3, values=[expand, out1, out2, out3, out4, out5]) net2 = tf_util.conv2d(concat, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv1', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp1') net2 = tf_util.conv2d(net2, 256, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv2', weight_decay=weight_decay) net2 = tf_util.dropout(net2, keep_prob=0.8, is_training=is_training, scope='seg/dp2') net2 = tf_util.conv2d(net2, 128, [1, 1], padding='VALID', stride=[1, 1], bn_decay=bn_decay, bn=True, is_training=is_training, scope='seg/conv3', weight_decay=weight_decay) net2 = tf_util.conv2d(net2, part_num, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, bn=False, scope='seg/conv4', weight_decay=weight_decay) net2 = tf.reshape(net2, [batch_size, num_point, part_num]) return net, net2, end_points
def get_model_pcn(pcn_enc, x, num_parts, is_training, bn_decay=None, reuse=False): with tf.variable_scope('trans', reuse=reuse): bias_initializer = np.array([[1., 0, 1, 0, 1, 0] for _ in xrange(num_parts)]) bias_initializer = bias_initializer.astype('float32').flatten() net = tf_util.fully_connected(pcn_enc, 256, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay, reuse=reuse) net = tf_util.fully_connected(net, 128, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay, reuse=reuse) trans = tf_util.fully_connected( net, num_parts * 6, activation_fn=None, scope='fc3', weights_initializer=tf.zeros_initializer(), biases_initializer=tf.constant_initializer(bias_initializer), reuse=reuse) # Perform transformation with tf.variable_scope('pcn', reuse=reuse): zeros_dims = tf.stack([tf.shape(x)[0], 1]) zeros_col = tf.fill(zeros_dims, 0.0) ''' sx 0 0 tx 0 sy 0 ty 0 0 sz tz ''' trans_mat = tf.concat( (tf.expand_dims(trans[:, 0], axis=1), zeros_col, zeros_col, tf.expand_dims(trans[:, 1], axis=1), zeros_col, tf.expand_dims(trans[:, 2], axis=1), zeros_col, tf.expand_dims(trans[:, 3], axis=1), zeros_col, zeros_col, tf.expand_dims(trans[:, 4], axis=1), tf.expand_dims(trans[:, 5], axis=1), tf.expand_dims(trans[:, 6], axis=1), zeros_col, zeros_col, tf.expand_dims(trans[:, 7], axis=1), zeros_col, tf.expand_dims(trans[:, 8], axis=1), zeros_col, tf.expand_dims(trans[:, 9], axis=1), zeros_col, zeros_col, tf.expand_dims(trans[:, 10], axis=1), tf.expand_dims(trans[:, 11], axis=1)), axis=1) for p in xrange(2, num_parts): start_ind = 6 * p trans_mat = tf.concat( (trans_mat, tf.expand_dims(trans[:, start_ind], axis=1), zeros_col, zeros_col, tf.expand_dims(trans[:, start_ind + 1], axis=1), zeros_col, tf.expand_dims(trans[:, start_ind + 2], axis=1), zeros_col, tf.expand_dims(trans[:, start_ind + 3], axis=1), zeros_col, zeros_col, tf.expand_dims(trans[:, start_ind + 4], axis=1), tf.expand_dims(trans[:, start_ind + 5], axis=1)), axis=1) trans_mat = tf.reshape(trans_mat, (-1, num_parts, 3, 4)) # adding 1 (w coordinate) to every point (x,y,z,1) w = tf.ones([tf.shape(x)[0], tf.shape(x)[1], tf.shape(x)[2], 1]) x = tf.concat((x, w), axis=-1) x_t = tf.transpose(x, [0, 1, 3, 2]) y_hat_t = tf.matmul(trans_mat, x_t) y_hat = tf.transpose(y_hat_t, [0, 1, 3, 2]) return y_hat
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output Bx40 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} l0_xyz = point_cloud l0_points = None # Set abstraction layers l1_xyz, l1_points = pointnet_sa_module_msg( l0_xyz, l0_points, 512, [0.1, 0.2, 0.4], [16, 32, 128], [[32, 32, 64], [64, 64, 128], [64, 96, 128]], is_training, bn_decay, scope='layer1', use_nchw=True) l2_xyz, l2_points = pointnet_sa_module_msg( l1_xyz, l1_points, 128, [0.2, 0.4, 0.8], [32, 64, 128], [[64, 64, 128], [128, 128, 256], [128, 128, 256]], is_training, bn_decay, scope='layer2') l3_xyz, l3_points, _ = pointnet_sa_module(l2_xyz, l2_points, npoint=None, radius=None, nsample=None, mlp=[256, 512, 1024], mlp2=None, group_all=True, is_training=is_training, bn_decay=bn_decay, scope='layer3') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.4, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3') return net, end_points
def get_dynamics_model(shape_feat, lin_vel, ang_vel, cell_type, num_cells, hidden_size, dropout_keep_prob, time_steps, is_training): ''' Build the graph for the state prediction module ''' batch_size = shape_feat.get_shape()[0].value # inputs are a 22-vec [lin_vel, ang_vel, shape_feat] tile_arg = tf.stack([tf.constant(1), time_steps, tf.constant(1)]) step_shape = tf.tile(tf.expand_dims(shape_feat, 1), tile_arg) inputs = tf.concat([lin_vel, ang_vel, step_shape], axis=2) # ouputs are size 23, 4 size-3 vectors representing change in state: dv, dw, dp, d\theta, one topply classify logit num_params = 13 W_hy = tf.get_variable('W_hy', shape=(hidden_size, num_params), initializer=tf.contrib.layers.xavier_initializer(), dtype=tf.float32) l2_normalization = tf.multiply(tf.nn.l2_loss(W_hy), weight_decay, name='weight_loss') tf.add_to_collection('losses', l2_normalization) b_hy = tf.get_variable('b_hy', shape=(1, num_params), initializer=tf.contrib.layers.xavier_initializer(), dtype=tf.float32) if cell_type == 'fc': # need to do it differently # num_cells used as number of FC layers, each with hidden_size nodes # inputs is B, num_steps, 12 input_feat_size = inputs.get_shape()[2].value inputs = tf.reshape(inputs, [batch_size * time_steps, input_feat_size]) cur_input = inputs for j in range(num_cells): cell_name = 'cell_fc' + str(j) # NOTE: batch norm causes some issues that really hinders training here - don't use it cur_input = tf_util.fully_connected(cur_input, hidden_size, weight_decay=weight_decay, bn=False, \ is_training=is_training, scope=cell_name, activation_fn=tf.nn.tanh, bn_decay=bn_decay) # final output y = tf.matmul(cur_input, W_hy) + b_hy y = tf.reshape(y, [batch_size, time_steps, num_params]) init_state = tf.constant(0) return y, init_state, init_state # no state to return # then feed to RNN with velocites if cell_type == 'rnn': rnn_cell = [ tf.nn.rnn_cell.DropoutWrapper( tf.nn.rnn_cell.BasicRNNCell(hidden_size), output_keep_prob=dropout_keep_prob) for i in range(0, num_cells) ] if cell_type == 'gru': rnn_cell = [ tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.GRUCell( hidden_size, kernel_initializer=tf.contrib.layers.xavier_initializer()), output_keep_prob=dropout_keep_prob) for i in range(0, num_cells) ] if cell_type == 'lstm': rnn_cell = [ tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.LSTMCell( hidden_size, initializer=tf.contrib.layers.xavier_initializer()), output_keep_prob=dropout_keep_prob) for i in range(0, num_cells) ] if num_cells > 1: rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_cell) else: rnn_cell = rnn_cell[0] init_state = rnn_cell.zero_state(batch_size, dtype=tf.float32) # feed through RNN # outputs are [batch, time_steps, hidden_size] outputs, state = tf.nn.dynamic_rnn(rnn_cell, inputs, initial_state=init_state, dtype=tf.float32) y = tf.matmul(tf.reshape(outputs, [batch_size * time_steps, hidden_size]), W_hy) + b_hy y = tf.reshape(y, [batch_size, time_steps, num_params]) return y, state, init_state
def get_model(point_cloud, is_training, mask=None, bn_decay=None, label_type="normal"): """Classification PointNet, input is BxNx3, output Bx40 arguments: point_clound: numpy array is_training: boolean bn_decay: boolean or None output: tensorflow model """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) # Point functions (MLP implemented as conv2d) with tf.variable_scope("conv1") as sc: net_conv1 = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1_1', bn_decay=bn_decay) net_conv3 = tf.pad(input_image, [[0, 0], [1, 1], [0, 0], [0, 0]]) net_conv3 = tf_util.conv2d(net_conv3, 64, [3, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1_3', bn_decay=bn_decay) net_conv5 = tf.pad(input_image, [[0, 0], [2, 2], [0, 0], [0, 0]]) net_conv5 = tf_util.conv2d(net_conv5, 64, [5, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1_5', bn_decay=bn_decay) net = tf.concat([net_conv1, net_conv3, net_conv5], 3, name="concat_1") with tf.variable_scope("conv2") as sc: net_conv1 = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2_1', bn_decay=bn_decay) net_conv3 = tf.pad(net, [[0, 0], [1, 1], [0, 0], [0, 0]]) net_conv3 = tf_util.conv2d(net_conv3, 128, [3, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2_3', bn_decay=bn_decay) net_conv5 = tf.pad(net, [[0, 0], [2, 2], [0, 0], [0, 0]]) net_conv5 = tf_util.conv2d(net_conv5, 128, [5, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2_5', bn_decay=bn_decay) net = tf.concat([net_conv1, net_conv3, net_conv5], 3, name="concat_2") with tf.variable_scope("conv3") as sc: net_conv1 = tf_util.conv2d(net, 256, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3_1', bn_decay=bn_decay) net_conv3 = tf.pad(net, [[0, 0], [1, 1], [0, 0], [0, 0]]) net_conv3 = tf_util.conv2d(net_conv3, 256, [3, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3_3', bn_decay=bn_decay) net_conv5 = tf.pad(net, [[0, 0], [2, 2], [0, 0], [0, 0]]) net_conv5 = tf_util.conv2d(net_conv5, 256, [5, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3_5', bn_decay=bn_decay) net = tf.concat([net_conv1, net_conv3, net_conv5], 3, name="concat_3") # masking net = tf.multiply(net, tf.cast(mask, tf.float32)) net = tf_util.max_pool2d(net, kernel_size=[num_point, 1], scope='max_pooling', padding='VALID') net = tf_util.conv2d(net, 512, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.fully_connected(net, 128, bn=True, is_training=is_training, scope='fc3', bn_decay=bn_decay) # net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training, # scope='dp1') if label_type == "normal": output = tf_util.fully_connected(net, 1, scope='fc4', activation_fn=None) elif label_type == "onehot": output = tf_util.fully_connected(net, 10, scope="fc4", activation_fn=None) return output
def get_model_other(point_cloud, is_training, bn_decay=None): """ DesnsePoint with 2 PPools + 3 PConvs + 1 global pooling narrowness k = 24; group number g = 2 """ batch_size = point_cloud.get_shape()[0] # .value end_points = {} # concatenate all features together SF,x,y,z, distance minSF point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0], axis=-1) l0_xyz = point_cloud[:, :, 1:4] l0_points = tf.concat( axis=2, values=[point_cloud_SF, point_cloud[:, :, 4:para.dim]]) # first stage: 1 PPool, 3 EnhancedPConv all_xyz, all_points = densepoint_module(l0_xyz, l0_points, is_training, bn_decay, npoint=512, radius=0.25, nsample=64, mlp=para.k_add * 4, scope='PPool1', ppool=True) # second stage: 2 PPool, 3 EnhancedPConv all_xyz, all_points = densepoint_module(all_xyz, all_points, is_training, bn_decay, npoint=256, radius=0.27, nsample=64, mlp=para.k_add * 4 - 3, scope='PPool2', ppool=True) for i in range(4): # B 128 1 93 -> 24 all_xyz, all_points = densepoint_module(all_xyz, all_points, is_training, bn_decay, npoint=256, radius=0.32, nsample=16, mlp=para.k_add * 4, group_num=para.group_num, scope=f'PConv1_{i + 1}') # # second stage: 2 PPool, 3 EnhancedPConv # all_xyz, all_points = densepoint_module(all_xyz, all_points, is_training, bn_decay, # npoint=128, radius=0.32, nsample=64, mlp=para.k_add * 4 - 3, # scope='PPool3', ppool=True) # # for i in range(4): # B 128 1 93 -> 24 # all_xyz, all_points = densepoint_module(all_xyz, all_points, is_training, bn_decay, # npoint=128, radius=0.39, nsample=16, mlp=para.k_add * 4, # group_num=para.group_num, # scope=f'PConv2_{i + 1}') l3_points = densepoint_module(all_xyz, all_points, is_training, bn_decay, mlp=512, scope='GloPool') # Fully connected layers net = tf.reshape(l3_points, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3') return net, end_points
def global_spatial_transformer(point_cloud, is_training, K=3, bn=True, bn_decay=None, is_dist=True): """ Input (XYZ) Transform Net, input is BxNx3 gray image Return: Transformation matrix of size KxK """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value net = tf_util.conv2d(point_cloud, 64, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='tconv1', bn_decay=bn_decay, is_dist=is_dist) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='tconv2', bn_decay=bn_decay, is_dist=is_dist) net = tf.reduce_max(net, axis=-2, keep_dims=True) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=bn, is_training=is_training, scope='tconv3', bn_decay=bn_decay, is_dist=is_dist) net = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='tmaxpool') net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=bn, is_training=is_training, scope='tfc1', bn_decay=bn_decay, is_dist=is_dist) net = tf_util.fully_connected(net, 256, bn=bn, is_training=is_training, scope='tfc2', bn_decay=bn_decay, is_dist=is_dist) with tf.variable_scope('transform_XYZ') as sc: weights = tf.get_variable('weights', [256, K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases = tf.get_variable('biases', [K * K], initializer=tf.constant_initializer(0.0), dtype=tf.float32) biases += tf.constant(np.eye(K).flatten(), dtype=tf.float32) transform = tf.matmul(net, weights) transform = tf.nn.bias_add(transform, biases) transform = tf.reshape(transform, [batch_size, K, K]) return transform
def get_model(point_cloud, is_training, bn_decay=None): """ Classification PointNet, input is BxNx3, output BxNx50 """ batch_size = point_cloud.get_shape()[0].value num_point = point_cloud.get_shape()[1].value end_points = {} with tf.variable_scope('transform_net1') as sc: transform = input_transform_net(point_cloud, is_training, bn_decay, K=3) point_cloud_transformed = tf.matmul(point_cloud, transform) input_image = tf.expand_dims(point_cloud_transformed, -1) net = tf_util.conv2d(input_image, 64, [1, 3], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv1', bn_decay=bn_decay) net = tf_util.conv2d(net, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv2', bn_decay=bn_decay) with tf.variable_scope('transform_net2') as sc: transform = feature_transform_net(net, is_training, bn_decay, K=64) end_points['transform'] = transform net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform) point_feat = tf.expand_dims(net_transformed, [2]) net = tf_util.conv2d(point_feat, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv3', bn_decay=bn_decay) net = tf_util.conv2d(net, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv4', bn_decay=bn_decay) net = tf_util.conv2d(net, 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='conv5', bn_decay=bn_decay) global_feat = tf_util.max_pool2d(net, [num_point, 1], padding='VALID', scope='maxpool') global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1]) concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand]) # (32,1024,1,1088) Fsem = tf_util.conv2d(concat_feat, 128, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='Fsem') ptssemseg_logits = tf_util.conv2d(Fsem, 50, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='ptssemseg_logits') ptssemseg_logits = tf.squeeze(ptssemseg_logits, [2]) # ptssemseg = tf.nn.softmax(ptssemseg_logits, name="ptssemseg") # Similarity matrix fts = tf.reshape(ptssemseg_logits, [batch_size, -1]) H = HGNN.construct_H_with_KNN(fts) G = HGNN.generate_G_from_H(H) # G = tf.convert_to_tensor(G) net = tf_util.fully_connected(fts, 128, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf.matmul(G, net) net = tf_util.fully_connected(net, 40, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf.matmul(G, net) return net, end_points
def get_model_other(point_cloud, is_training, bn_decay=None): """ B: batch size; N: number of points, C: channels; k: number of nearest neighbors point_cloud: B*N*C """ end_points = {} minSF = tf.reshape(tf.math.argmin(point_cloud[:, :, 0], axis=1), (-1, 1)) batch_size = point_cloud.get_shape()[0] # .value # # 1. graph for first EdgeConv B N C=6 adj_matrix = tf_util.pairwise_distance( point_cloud[:, :, :para.dim]) # B N C=6 => B*N*N # adj_matrix = tf_util.pairwise_distance(point_cloud[:, :, 1:para.dim]) # B N C=6 => B*N*N nn_idx = tf_util.knn(adj_matrix, k=20) # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn1'] = allSF_dist point_cloud = tf.expand_dims(point_cloud[:, :, :para.dim], axis=-2) # point_cloud = tf.expand_dims(point_cloud[:, :, 1:para.dim], axis=-2) edge_feature = tf_util.get_edge_feature(point_cloud, nn_idx=nn_idx, k=20) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn1', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keepdims=True) net1 = net # # 2. graph for second EdgeConv B N C=64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=20) # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn2'] = allSF_dist # net: B*N*1*6+64=71 net = tf.concat([point_cloud, net1], axis=-1) # edge_feature: B*N*k*142 edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn2', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keepdims=True) net2 = net # 3. graph for third EdgeConv B N C=64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=20) # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn3'] = allSF_dist # net: B*N*1*6+64+64=134 net = tf.concat([point_cloud, net1, net2], axis=-1) # edge_feature: B*N*k*268 edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn3', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keepdims=True) net3 = net # 4. graph for fourth EdgeConv B N C=64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=20) # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn4'] = allSF_dist # net: B*N*1*6+64+64+64=198 net = tf.concat([point_cloud, net1, net2, net3], axis=-1) # edge_feature: B*N*k*396 edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=20) net = tf_util.conv2d(edge_feature, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn4', bn_decay=bn_decay) net = tf.reduce_max(net, axis=-2, keepdims=True) net4 = net # input: B*N*1*6+64+64+128+128 = 326 => net: B*N*1*1024 net = tf_util.conv2d(tf.concat([point_cloud, net1, net2, net3, net4], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='agg', bn_decay=bn_decay) # net: B*1*1*1024 # SF_features = tf.gather(net, indices=minSF, axis=1, batch_dims=1) net = tf.reduce_max(net, axis=1, keepdims=True) # SF_all = tf.concat([SF_features,net], axis=-1) # net: B*1024 net = tf.squeeze(net) # net: B*2048 # net = tf.squeeze(SF_all) # MLP on global point cloud vector net = tf.reshape(net, [batch_size, -1]) print(net.get_shape()) end_points['global_feature'] = net # Fully connected end_points: classifier net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) end_points['fc1'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) end_points['fc2'] = net net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3') end_points['fc3'] = net return net, end_points
def get_model_other(point_cloud, is_training, bn_decay=None): """ Classification DGCNN, input is BxNxC, output BxCls B batch size N number of points per pointcloud C input channel: eg. x,y,z,SF,distance, minSF... Cls output class number """ batch_size = point_cloud.get_shape()[0] # .value end_points = {} # get MinSF index minSF = tf.reshape(tf.math.argmin(point_cloud[:, :, 0], axis=1), (-1, 1)) # # 1. graph for transform net with only x,y,z adj_matrix = tf_util.pairwise_distance(point_cloud[:, :, 1:4]) # B N C=3 => B N N nn_idx = tf_util.knn(adj_matrix, k=para.k) edge_feature = tf_util.get_edge_feature(point_cloud[:, :, 1:4], nn_idx=nn_idx, k=para.k) with tf.compat.v1.variable_scope('transform_net1') as sc: transform = input_transform_net_dgcnn(edge_feature, is_training, bn_decay, K=3) point_cloud_transform = tf.matmul(point_cloud[:, :, 1:4], transform) # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) # B N 1 end_points['knn1'] = allSF_dist point_cloud_SF = tf.expand_dims(point_cloud[:, :, 0], axis=-1) # # 2. graph for first EdgeConv with transform(x,y,z), SF, distance, minSF point_cloud_all = tf.concat(axis=2, values=[point_cloud_SF, point_cloud_transform, point_cloud[:, :, 4:para.dim]]) adj_matrix = tf_util.pairwise_distance(point_cloud_all) # B N C=6 nn_idx = tf_util.knn(adj_matrix, k=para.k) edge_feature = tf_util.get_edge_feature(point_cloud_all, nn_idx=nn_idx, k=para.k) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn1', bn_decay=bn_decay) net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True) net1 = net # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn2'] = allSF_dist # # 3. graph for second EdgeConv with C = 64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=para.k) edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn2', bn_decay=bn_decay) net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True) net2 = net # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn3'] = allSF_dist # # 4. graph for third EdgeConv with C = 64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=para.k) edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k) net = tf_util.conv2d(edge_feature, 64, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn3', bn_decay=bn_decay) net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True) net3 = net # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn4'] = allSF_dist # # 5. graph for fourth EdgeConv with C = 64 adj_matrix = tf_util.pairwise_distance(net) nn_idx = tf_util.knn(adj_matrix, k=para.k) edge_feature = tf_util.get_edge_feature(net, nn_idx=nn_idx, k=para.k) net = tf_util.conv2d(edge_feature, 128, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='dgcnn4', bn_decay=bn_decay) net = tf.reduce_max(input_tensor=net, axis=-2, keepdims=True) net4 = net # get the distance to minSF of 1024 points allSF_dist = tf.gather(adj_matrix, indices=minSF, axis=2, batch_dims=1) end_points['knn5'] = allSF_dist # # 6. MLP for all concatenate features 64+64+64+128 net = tf_util.conv2d(tf.concat([net1, net2, net3, net4], axis=-1), 1024, [1, 1], padding='VALID', stride=[1, 1], bn=True, is_training=is_training, scope='agg', bn_decay=bn_decay) net = tf.reduce_max(input_tensor=net, axis=1, keepdims=True) # maxpooling B N C=1024 => B 1 1024 # # 7. MLP on global point cloud vector B 1 1024 net = tf.reshape(net, [batch_size, -1]) net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1') net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2') net = tf_util.fully_connected(net, para.outputClassN, activation_fn=None, scope='fc3') return net, end_points