def __init__(self, points, features, is_training, setting): xconv_params = setting.xconv_params fc_params = setting.fc_params with_X_transformation = setting.with_X_transformation sorting_method = setting.sorting_method N = tf.shape(points)[0] if setting.sampling == 'fps': from sampling import tf_sampling self.layer_pts = [points] if features is None: self.layer_fts = [features] else: # expand feature dimentions features = tf.reshape(features, (N, -1, setting.data_dim - 3), name='features_reshape') C_fts = xconv_params[0]['C'] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) self.layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] # neighborhood size D = layer_param['D'] # dilate rate P = layer_param['P'] # representative point number C = layer_param['C'] # output channel number links = layer_param[ 'links'] # e.g. [-1,-2] will tell the current layer to receive inputs from the previous two layers if setting.sampling != 'random' and links: print( 'Error: flexible links are supported only when random sampling is used!' ) exit() # get k-nearest points pts = self.layer_pts[-1] # get the end one in list (last layer) fts = self.layer_fts[-1] # if P == -1 or p equals to last layer's p , than no sampling if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1]['P']): qrs = self.layer_pts[-1] else: # for segmentation task if setting.sampling == 'fps': # fps_indices: [[0_0, 0_1, ...., 0_P-1], ...., [N-1_0, N-1_1, ...., N-1_P-1]] (NxP) fps_indices = tf_sampling.farthest_point_sample(P, pts) # replicating a index(0...N) for each output poit(P) -> (N, P, 1) # batch_indices: [[[0],[0],...,[0]], ..., [[N-1],[N-1],...,[N-1]]] batch_indices = tf.tile( tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1)) # tf.expand_dims(fps_indices, -1): [[[0_0], [0_1], ..., [0_P-1]], ... , [[N-1_0], [N-1_1], ... , [N-1_P-1]]] # indices: [[[0, 0_0], [0, 0_1], ..., [0, 0_P-1]], ..., [[N-1, N-1_0], [N-1, N-1_1], ... , [N-1, N-1_P-1]]] indices = tf.concat( [batch_indices, tf.expand_dims(fps_indices, -1)], axis=-1) qrs = tf.gather_nd(pts, indices, name=tag + 'qrs') # (N, P, 3) elif setting.sampling == 'ids': indices = pf.inverse_density_sampling(pts, K, P) qrs = tf.gather_nd(pts, indices) # for classification tasks elif setting.sampling == 'random': qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3) else: print('Unknown sampling method!') exit() self.layer_pts.append(qrs) if layer_idx == 0: # lift each point coordinates into C_pts_fts dimensional space C_pts_fts = C // 2 if fts is None else C // 4 depth_multiplier = 4 else: C_prev = xconv_params[layer_idx - 1]['C'] C_pts_fts = C_prev // 4 depth_multiplier = math.ceil(C / C_prev) # with global feature at last layer with_global = (setting.with_global and layer_idx == len(xconv_params) - 1) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method, with_global) fts_list = [] # receive inputs from previous layers for link in links: fts_from_link = self.layer_fts[link] if fts_from_link is not None: fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1), name=tag + 'fts_slice_' + str(-link)) fts_list.append(fts_slice) if fts_list: fts_list.append(fts_xconv) self.layer_fts.append( tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat')) else: self.layer_fts.append(fts_xconv) if hasattr(setting, 'xdconv_params'): for layer_idx, layer_param in enumerate(setting.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] pts_layer_idx = layer_param['pts_layer_idx'] qrs_layer_idx = layer_param['qrs_layer_idx'] pts = self.layer_pts[pts_layer_idx + 1] fts = self.layer_fts[ pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1] qrs = self.layer_pts[qrs_layer_idx + 1] fts_qrs = self.layer_fts[qrs_layer_idx + 1] P = xconv_params[qrs_layer_idx]['P'] C = xconv_params[qrs_layer_idx]['C'] C_prev = xconv_params[pts_layer_idx]['C'] C_pts_fts = C_prev // 4 depth_multiplier = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) self.layer_pts.append(qrs) self.layer_fts.append(fts_fuse) self.fc_layers = [self.layer_fts[-1]] for layer_idx, layer_param in enumerate(fc_params): C = layer_param['C'] dropout_rate = layer_param['dropout_rate'] fc = pf.dense(self.fc_layers[-1], C, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) self.fc_layers.append(fc_drop)
def __init__(self, points, features, is_training, setting): xconv_params = setting.xconv_params fc_params = setting.fc_params with_X_transformation = setting.with_X_transformation sorting_method = setting.sorting_method N = tf.shape(points)[0] if setting.sampling == 'fps': import sys #sys.path.append("/home/hasan/data/paper2/PointCNN/") from sampling import tf_sampling self.layer_pts = [points] if features is None: self.layer_fts = [features] else: features = tf.reshape(features, (N, -1, setting.data_dim - 3), name='features_reshape') C_fts = xconv_params[0]['C'] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) self.layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] P = layer_param['P'] C = layer_param['C'] links = layer_param['links'] if setting.sampling != 'random' and links: print( 'Error: flexible links are supported only when random sampling is used!' ) exit() # get k-nearest points pts = self.layer_pts[-1] fts = self.layer_fts[-1] if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1]['P']): qrs = self.layer_pts[-1] else: if setting.sampling == 'fps': fps_indices = tf_sampling.farthest_point_sample(P, pts) batch_indices = tf.tile( tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1)) indices = tf.concat( [batch_indices, tf.expand_dims(fps_indices, -1)], axis=-1) qrs = tf.gather_nd(pts, indices, name=tag + 'qrs') # (N, P, 3) elif setting.sampling == 'ids': indices = pf.inverse_density_sampling(pts, K, P) qrs = tf.gather_nd(pts, indices) elif setting.sampling == 'random': qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3) else: print('Unknown sampling method!') exit() self.layer_pts.append(qrs) if layer_idx == 0: C_pts_fts = C // 2 if fts is None else C // 4 depth_multiplier = 4 else: C_prev = xconv_params[layer_idx - 1]['C'] C_pts_fts = C_prev // 4 depth_multiplier = math.ceil(C / C_prev) with_global = (setting.with_global and layer_idx == len(xconv_params) - 1) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method, with_global) fts_list = [] for link in links: fts_from_link = self.layer_fts[link] if fts_from_link is not None: fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1), name=tag + 'fts_slice_' + str(-link)) fts_list.append(fts_slice) if fts_list: fts_list.append(fts_xconv) self.layer_fts.append( tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat')) else: self.layer_fts.append(fts_xconv) if hasattr(setting, 'xdconv_params'): for layer_idx, layer_param in enumerate(setting.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] pts_layer_idx = layer_param['pts_layer_idx'] qrs_layer_idx = layer_param['qrs_layer_idx'] pts = self.layer_pts[pts_layer_idx + 1] fts = self.layer_fts[ pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1] qrs = self.layer_pts[qrs_layer_idx + 1] fts_qrs = self.layer_fts[qrs_layer_idx + 1] P = xconv_params[qrs_layer_idx]['P'] C = xconv_params[qrs_layer_idx]['C'] C_prev = xconv_params[pts_layer_idx]['C'] C_pts_fts = C_prev // 4 depth_multiplier = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) self.layer_pts.append(qrs) self.layer_fts.append(fts_fuse) self.fc_layers = [self.layer_fts[-1]] for layer_idx, layer_param in enumerate(fc_params): C = layer_param['C'] dropout_rate = layer_param['dropout_rate'] fc = pf.dense(self.fc_layers[-1], C, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) self.fc_layers.append(fc_drop)
def __init__(self, points, features, num_class, is_training, setting, task): xconv_params = setting.xconv_params fc_params = setting.fc_params with_X_transformation = setting.with_X_transformation sorting_method = setting.sorting_method N = tf.shape(points)[0] if setting.sampling == 'fps': from sampling import tf_sampling self.layer_pts = [points] if features is None: self.layer_fts = [features] else: C_fts = xconv_params[0]['C'] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) self.layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] P = layer_param['P'] C = layer_param['C'] links = layer_param['links'] if setting.sampling != 'random' and links: print('Error: flexible links are supported only when random sampling is used!') exit() # get k-nearest points pts = self.layer_pts[-1] fts = self.layer_fts[-1] if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1]['P']): qrs = self.layer_pts[-1] else: if setting.sampling == 'fps': indices = tf_sampling.farthest_point_sample(P, pts) qrs = tf_sampling.gather_point(pts, indices) # (N,P,3) elif setting.sampling == 'ids': indices = pf.inverse_density_sampling(pts, K, P) qrs = tf.gather_nd(pts, indices) elif setting.sampling == 'random': qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3) else: print('Unknown sampling method!') exit() self.layer_pts.append(qrs) if layer_idx == 0: C_pts_fts = C // 2 if fts is None else C // 4 depth_multiplier = 4 else: C_prev = xconv_params[layer_idx - 1]['C'] C_pts_fts = C_prev // 4 depth_multiplier = math.ceil(C / C_prev) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method, layer_idx != len(xconv_params) - 1) fts_list = [] for link in links: fts_from_link = self.layer_fts[link] if fts_from_link is not None: fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1), name=tag + 'fts_slice_' + str(-link)) C_forward = math.ceil(fts_slice.get_shape().as_list()[-1] / (-link)) fts_forward = pf.dense(fts_slice, C_forward, tag + 'fts_forward_' + str(-link), is_training) fts_list.append(fts_forward) if fts_list: fts_list.append(fts_xconv) self.layer_fts.append(tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat')) else: self.layer_fts.append(fts_xconv) if task == 'segmentation': for layer_idx, layer_param in enumerate(setting.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] pts_layer_idx = layer_param['pts_layer_idx'] qrs_layer_idx = layer_param['qrs_layer_idx'] pts = self.layer_pts[pts_layer_idx + 1] fts = self.layer_fts[pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1] qrs = self.layer_pts[qrs_layer_idx + 1] fts_qrs = self.layer_fts[qrs_layer_idx + 1] P = xconv_params[qrs_layer_idx]['P'] C = xconv_params[qrs_layer_idx]['C'] C_prev = xconv_params[pts_layer_idx]['C'] C_pts_fts = C_prev // 4 depth_multiplier = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) self.layer_pts.append(qrs) self.layer_fts.append(fts_fuse) self.fc_layers = [self.layer_fts[-1]] for layer_idx, layer_param in enumerate(fc_params): C = layer_param['C'] dropout_rate = layer_param['dropout_rate'] fc = pf.dense(self.fc_layers[-1], C, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) self.fc_layers.append(fc_drop) if task == 'classification': fc_mean = tf.reduce_mean(self.fc_layers[-1], axis=1, keep_dims=True, name='fc_mean') self.fc_layers[-1] = tf.cond(is_training, lambda: self.fc_layers[-1], lambda: fc_mean) self.logits = pf.dense(self.fc_layers[-1], num_class, 'logits', is_training, with_bn=False, activation=None) self.probs = tf.nn.softmax(self.logits, name='probs')
def Invariance_Transformation_Net(point_cloud, features, is_training, invarians_trans_param): xconv_params = invarians_trans_param.xconv_params fc_params = invarians_trans_param.fc_params with_X_transformation = invarians_trans_param.with_X_transformation sorting_method = invarians_trans_param.sorting_method #N=point_cloud.get_shape()[1].value N = point_cloud.get_shape()[0].value if invarians_trans_param.sampling == 'fps': import tf_sampling layer_pts = [point_cloud] if features is None: layer_fts = [features] else: features = tf.reshape(features, (N, -1, invarians_trans_param.data_dim - 3), name='features_reshape') C_fts = xconv_params[0]['C'] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str( layer_idx + 1) + '_' #####xconv_1_ #####xconv_2_ #####xconv_3_ #####xconv_4_ K = layer_param['K'] D = layer_param['D'] P = layer_param['P'] C = layer_param['C'] links = layer_param['links'] ## type(layer_param) is dict #get k-nearest points pts = layer_pts[-1] fts = layer_fts[-1] if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1]['P']): qrs = layer_pts[-1] else: if invarians_trans_param.sampling == 'fps': fps_indices = tf_sampling.farthest_point_sample(P, pts) batch_indices = tf.tile(tf.reshape(tf.range(N), (-1, 1, 1)), (1, P, 1)) indices = tf.concat( [batch_indices, tf.expand_dims(fps_indices, -1)], axis=-1) qrs = tf.gather_nd(pts, indices, name=tag + 'qrs') ### (N,P,3) print(tf.shape(qrs)) elif invarians_trans_param.sampling == 'ids': indices = pf.inverse_density_sampling(pts, K, P) qrs = tf.gather_nd(pts, indices) elif invarians_trans_param == 'random': qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') ### (N,P,3) else: print('unknown sampling method') exit() layer_pts.append(qrs) if layer_idx == 0: C_pts_fts = C // 2 if fts is None else C // 4 depth_multipiler = 4 else: C_prev = xconv_params[layer_idx - 1]['C'] C_pts_fts = C_prev // 4 depth_multipiler = math.ceil(C / C_prev) with_global = (invarians_trans_param.with_global and layer_idx == len(xconv_params) - 1) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multipiler, sorting_method, with_global) fts_list = [] for link in links: fts_from_link = layer_fts[link] if fts_from_link is not None: fts_slice = tf.slice(fts_from_link, (0, 0, 0), (-1, P, -1), name=tag + 'fts_slice' + str(-link)) fts_list.append(fts_slice) if fts_list: fts_list.append(fts_xconv) layer_fts.append( tf.concat(fts_list, axis=-1, name=tag + 'fts_list_concat')) else: layer_fts.append(fts_xconv) if hasattr(invarians_trans_param, 'xdconv_params'): for layer_idx, layer_param in enumerate( invarians_trans_param.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K = layer_param['K'] D = layer_param['D'] pts_layer_idx = layer_param['pts_layer_idx'] qrs_layer_idx = layer_param['qrs_layer_idx'] pts = layer_pts[pts_layer_idx + 1] fts = layer_fts[pts_layer_idx + 1] if layer_idx == 0 else layer_fts[-1] qrs = layer_pts[qrs_layer_idx + 1] fts_qrs = layer_fts[qrs_layer_idx + 1] P = xconv_params[qrs_layer_idx]['P'] C = xconv_params[qrs_layer_idx]['C'] C_prev = xconv_params[pts_layer_idx]['C'] C_pts_fts = C_prev // 4 depth_multipiler = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multipiler, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) layer_pts.append(qrs) layer_fts.append(fts_fuse) fc_layers = layer_fts[-1] for layer_idx, layer_param in enumerate(fc_params): C = layer_param['C'] dropout_rate = layer_param['dropout_rate'] fc = pf.dense(fc_layers[-1], C, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, dropout_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) fc_layers.append(fc_drop) return fc_layers
def __init__(self, points, features, num_class, is_training, setting, task): xconv_params = setting.xconv_params fc_params = setting.fc_params with_X_transformation = setting.with_X_transformation sorting_method = setting.sorting_method N = tf.shape(points)[0] if setting.sampling == 'fps': from sampling import tf_sampling self.layer_pts = [points] if features is None: self.layer_fts = [features] else: C_fts = xconv_params[0][-1] // 2 features_hd = pf.dense(features, C_fts, 'features_hd', is_training) self.layer_fts = [features_hd] for layer_idx, layer_param in enumerate(xconv_params): tag = 'xconv_' + str(layer_idx + 1) + '_' K, D, P, C = layer_param # get k-nearest points pts = self.layer_pts[-1] fts = self.layer_fts[-1] if P == -1 or (layer_idx > 0 and P == xconv_params[layer_idx - 1][2]): qrs = self.layer_pts[-1] else: if setting.sampling == 'fps': qrs = tf_sampling.gather_point( pts, tf_sampling.farthest_point_sample(P, pts)) # (N,P,3) elif setting.sampling == 'ids': qrs = pf.inverse_density_sampling(pts, K, P) elif setting.sampling == 'random': qrs = tf.slice(pts, (0, 0, 0), (-1, P, -1), name=tag + 'qrs') # (N, P, 3) else: print('Unknown sampling method!') exit() self.layer_pts.append(qrs) if layer_idx == 0: C_pts_fts = C // 2 if fts is None else C // 4 depth_multiplier = 4 else: C_prev = xconv_params[layer_idx - 1][-1] C_pts_fts = C_prev // 4 depth_multiplier = math.ceil(C / C_prev) fts_xconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) self.layer_fts.append(fts_xconv) if task == 'segmentation': for layer_idx, layer_param in enumerate(setting.xdconv_params): tag = 'xdconv_' + str(layer_idx + 1) + '_' K, D, pts_layer_idx, qrs_layer_idx = layer_param pts = self.layer_pts[pts_layer_idx + 1] fts = self.layer_fts[ pts_layer_idx + 1] if layer_idx == 0 else self.layer_fts[-1] qrs = self.layer_pts[qrs_layer_idx + 1] fts_qrs = self.layer_fts[qrs_layer_idx + 1] _, _, P, C = xconv_params[qrs_layer_idx] _, _, _, C_prev = xconv_params[pts_layer_idx] C_pts_fts = C_prev // 4 depth_multiplier = 1 fts_xdconv = xconv(pts, fts, qrs, tag, N, K, D, P, C, C_pts_fts, is_training, with_X_transformation, depth_multiplier, sorting_method) fts_concat = tf.concat([fts_xdconv, fts_qrs], axis=-1, name=tag + 'fts_concat') fts_fuse = pf.dense(fts_concat, C, tag + 'fts_fuse', is_training) self.layer_pts.append(qrs) self.layer_fts.append(fts_fuse) self.fc_layers = [self.layer_fts[-1]] for layer_idx, layer_param in enumerate(fc_params): channel_num, drop_rate = layer_param fc = pf.dense(self.fc_layers[-1], channel_num, 'fc{:d}'.format(layer_idx), is_training) fc_drop = tf.layers.dropout(fc, drop_rate, training=is_training, name='fc{:d}_drop'.format(layer_idx)) self.fc_layers.append(fc_drop) logits = pf.dense(self.fc_layers[-1], num_class, 'logits', is_training, with_bn=False, activation=None) if task == 'classification': logits_mean = tf.reduce_mean(logits, axis=1, keep_dims=True, name='logits_mean') self.logits = tf.cond(is_training, lambda: logits, lambda: logits_mean) elif task == 'segmentation': self.logits = logits else: print('Unknown task!') exit() self.probs = tf.nn.softmax(self.logits, name='probs')