def get_loss(pred, label, end_points): """ pred: BxNx3, label: BxNx3, """ dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( pred, label) pc_loss = tf.reduce_mean(dists_forward + dists_backward) end_points['pcloss'] = pc_loss tf.summary.scalar('pcloss', pc_loss) d1, _, d2, _ = tf_nndistance.nn_distance(end_points['pc1_xyz'], label) pc1_loss = tf.reduce_mean(d1) + tf.reduce_mean(d2) tf.summary.scalar('pc1loss', pc1_loss) loss = pc_loss + 0.1 * pc1_loss return loss * 100, end_points
def get_loss(pred, label, end_points): """ pred: BxNx3, label: BxNx3, """ dists_forward,_,dists_backward,_ = tf_nndistance.nn_distance(pred, label) loss = tf.reduce_mean(dists_forward+dists_backward) end_points['pcloss'] = loss return loss*100, end_points
def chamfer_loss(pc1, pc2): """ pred: BxNx3, label: BxNx3, """ dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance(pc1, pc2) # loss = dists_forward+dists_backward loss = tf.reduce_mean(dists_forward + dists_backward, axis=1) return loss
def get_cd_loss2(pred, gt, radius, forward_weight=1.0, threshold=100): """ pred: BxNxC, label: BxN, forward_weight: relative weight for forward_distance """ with tf.name_scope("cd_loss"): dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( gt, pred) if threshold is not None: forward_threshold = tf.reduce_mean( dists_forward, keepdims=True, axis=1) * threshold backward_threshold = tf.reduce_mean( dists_backward, keepdims=True, axis=1) * threshold # only care about distance within threshold (ignore strong outliers) dists_forward = tf.where(dists_forward < forward_threshold, dists_forward, tf.zeros_like(dists_forward)) dists_backward = tf.where(dists_backward < backward_threshold, dists_backward, tf.zeros_like(dists_backward)) # dists_forward is for each element in gt, the closest distance to this element dists_forward = tf.reduce_mean(dists_forward, axis=1) dists_backward = tf.reduce_mean(dists_backward, axis=1) CD_dist = forward_weight * dists_forward + dists_backward # CD_dist_norm = CD_dist/radius cd_loss = tf.reduce_mean(CD_dist) return cd_loss #, None
def get_loss(pred, label): """ pred: BxNx3, label: BxNx3, """ dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( pred, label) loss = tf.reduce_mean(dists_forward + dists_backward, axis=1) return loss * 100
def mesh_loss(pred, placeholders, block_id): gt_pt = placeholders['labels'][:, :3] # gt points gt_nm = placeholders['labels'][:, 3:] # gt normals # edge in graph nod1 = tf.gather(pred, placeholders['edges'][block_id - 1][:, 0]) nod2 = tf.gather(pred, placeholders['edges'][block_id - 1][:, 1]) edge = tf.subtract(nod1, nod2) # edge length loss edge_length = tf.reduce_sum(tf.square(edge), 1) edge_loss = tf.reduce_mean(edge_length) * 300 # chamer distance dist1, idx1, dist2, idx2 = tf_nndistance.nn_distance(gt_pt, pred) point_loss = (tf.reduce_mean(dist1) + 0.55 * tf.reduce_mean(dist2)) * 3000 # normal cosine loss normal = tf.gather(gt_nm, tf.squeeze(idx2, 0)) normal = tf.gather(normal, placeholders['edges'][block_id - 1][:, 0]) cosine = tf.abs(tf.reduce_sum(tf.multiply(unit(normal), unit(edge)), 1)) # cosine = tf.where(tf.greater(cosine,0.866), tf.zeros_like(cosine), cosine) # truncated normal_loss = tf.reduce_mean(cosine) * 0.5 total_loss = point_loss + edge_loss + normal_loss return total_loss
def get_labels_seg(pcl_gt, pcl_pred, metric): ''' Point wise correspondences between two point sets args: pcl_gt: (batch_size, n_pts, 3), gt pcl pcl_pred: (batch_size, n_pts, 3), predicted pcl metric: str, 'chamfer' or 'emd' metric to be considered for returning corresponding points returns: pts_match_fwd: gt to pred point-wise correspondence each point in gt is mapped to nearest point in pred pts_match_bwd: pred to gt point-wise correspondence each point in pred is mapped to nearest point in gt pts_match: one-to-one mapping between pred and gt, acc. to emd ''' if metric == 'chamfer': _, pts_match_fwd, _, pts_match_bwd = tf_nndistance.nn_distance(\ pcl_gt, pcl_pred) return pts_match_fwd, pts_match_bwd elif metric == 'emd': pts_match, _ = auction_match(pcl_gt, pcl_pred) return pts_match else: print 'Undefined metric' return None
def get_metrics(gt_pcl, pred_pcl, args): ''' Obtain chamfer and emd distances between GT and predicted pcl args: gt_pcl: float, (BS,N_PTS,3); GT point cloud pred_pcl: float, (BS,N_PTS,3); predicted point cloud returns: dists_forward: float, (BS); forward chamfer distance dists_backward: float, (BS); backward chamfer distance chamfer_distance: float, (BS); chamfer distance emd: float, (BS); earth mover's distance ''' dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( gt_pcl, pred_pcl) dists_forward = tf.reduce_mean( dists_forward, axis=1) # (BATCH_SIZE,args.N_PTS) --> (BATCH_SIZE) dists_backward = tf.reduce_mean(dists_backward, axis=1) chamfer_distance = dists_backward + dists_forward X, _ = tf.meshgrid(range(args.batch_size), range(args.OUTPUT_PCL_SIZE), indexing='ij') ind, _ = auction_match(pred_pcl, gt_pcl) # Ind corresponds to points in pcl_gt ind = tf.stack((X, ind), -1) emd = tf.reduce_mean( tf.reduce_sum((tf.gather_nd(gt_pcl, ind) - pred_pcl)**2, axis=-1), axis=1 ) # (BATCH_SIZE,args.N_PTS,3) --> (BATCH_SIZE,args.N_PTS) --> (BATCH_SIZE) return dists_forward, dists_backward, chamfer_distance, emd
def __init__(self, seq_length, num_points=128): self.ground_truth = tf.placeholder(tf.float32, [1, seq_length, num_points, 3]) self.prediction = tf.placeholder(tf.float32, [1, seq_length, num_points, 3]) gt_frames = tf.split(value=self.ground_truth, num_or_size_splits=seq_length, axis=1) gt_frames = [tf.squeeze(input=frame, axis=[1]) for frame in gt_frames] pd_frames = tf.split(value=self.prediction, num_or_size_splits=seq_length, axis=1) pd_frames = [tf.squeeze(input=frame, axis=[1]) for frame in pd_frames] cds, emds = [], [] for i in range(seq_length): match = tf_approxmatch.approx_match(gt_frames[i], pd_frames[i]) emd_distance = tf.reduce_mean( tf_approxmatch.match_cost(gt_frames[i], pd_frames[i], match)) emds.append(emd_distance) dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( pd_frames[i], gt_frames[i]) cd_distance = tf.reduce_mean(dists_forward + dists_backward) cds.append(cd_distance) self.cds = tf.stack(cds, 0) self.emds = tf.stack(emds, 0)
def get_loss(pred, label): """ pred: BxNx3, label: BxNx3, """ dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( pred, label) loss_per_sample = dists_forward + dists_backward loss = tf.reduce_mean(loss_per_sample) return loss, loss_per_sample
def get_chamfer_metrics(gt_pcl, pred_pcl): dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( gt_pcl, pred_pcl) dists_forward = tf.reduce_mean( dists_forward, axis=1) # (BATCH_SIZE,NUM_POINTS) --> (BATCH_SIZE) dists_backward = tf.reduce_mean(dists_backward, axis=1) chamfer_distance = dists_backward + dists_forward return dists_forward, dists_backward, chamfer_distance
def get_cd_loss(pred, gt, radius): """ pred: BxNxC, label: BxN, """ dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance(gt, pred) #dists_forward is for each element in gt, the cloest distance to this element CD_dist = 0.8 * dists_forward + 0.2 * dists_backward CD_dist = tf.reduce_mean(CD_dist, axis=1) CD_dist_norm = CD_dist / radius cd_loss = tf.reduce_mean(CD_dist_norm) return cd_loss, None
def create_loss(output, truth, loss_type='emd'): if loss_type == 'emd': match = tf_approxmatch.approx_match(output, truth) build_loss = tf.reduce_mean( tf_approxmatch.match_cost(output, truth, match)) else: cost_p1_p2, _, cost_p2_p1, _ = tf_nndistance.nn_distance(output, truth) build_loss = tf.reduce_mean(cost_p1_p2) + tf.reduce_mean(cost_p2_p1) # tf.summary.scalar('build loss', build_loss) # tf.add_to_collection('losses', build_loss) return build_loss
def get_loss(pred, label, end_points): """ pred: BxNx3, label: BxNx3, """ dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( pred, label) likelihoods = end_points['likelihoods'] bits = tf.reduce_sum(tf.log(likelihoods), axis=(1, 2, 3)) / -np.log(2) loss = tf.reduce_mean(dists_forward + dists_backward) main_loss = 0.5 * tf.reduce_mean(squared_error) + tf.reduce_mean(bits) end_points['pcloss'] = loss return loss * 100, end_points
def get_loss(pred, label, end_points): """ pred: BxNx3, label: BxNx3, """ dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( pred, label) pc_loss = tf.reduce_mean(dists_forward + dists_backward) end_points['pcloss'] = pc_loss match = tf_approxmatch.approx_match(label, pred) loss = tf.reduce_mean(tf_approxmatch.match_cost(label, pred, match)) tf.summary.scalar('loss', loss) return loss, end_points
def get_loss(img, pc_in, is_training=True, reuse_mode=False): logits = model.img_encoder(img, conv_net, fully_con, is_training=is_training, reuse_mode=reuse_mode) pc_out = model.pc_decoder(logits, fully_dec, is_training=is_training, reuse_mode=reuse_mode) cost_p1_p2, _, cost_p2_p1, _ = nn_distance(pc_in, pc_out) cd_loss = tf.reduce_mean(cost_p1_p2) + tf.reduce_mean(cost_p2_p1) return cd_loss, pc_out
def get_loss(pred, label, end_points): """ pred: BxNx3, label: BxNx3, """ # Reconstruction loss dists_forward,_,dists_backward,_ = tf_nndistance.nn_distance(pred, label) loss = tf.reduce_mean(dists_forward+dists_backward) end_points['pcloss'] = loss # KL Divergence loss kl_div_loss = 1 + end_points['z_std'] - tf.square(end_points['z_mean']) - tf.exp(end_points['z_std']) kl_div_loss = -0.5 * tf.reduce_sum(kl_div_loss, 1) kl_div_loss = tf.reduce_mean(kl_div_loss) * 0.001 end_points['kl_div_loss'] = kl_div_loss return loss*100 + kl_div_loss, end_points
def _reconstruction_loss(self, recon, input): if self.loss == 'chamfer': cost_p1_p2, _, cost_p2_p1, _ = nn_distance(recon, input) loss = tf.reduce_mean(cost_p1_p2) + tf.reduce_mean(cost_p2_p1) elif self.loss == 'emd': match = approx_match(recon, input) loss = match_cost(recon, input, match) loss = tf.reduce_mean(loss) loss = tf.div(loss, self.point_cloud_shape[0]) # return point-wise loss elif self.loss == 'hausdorff': distances = directed_hausdorff(input, recon) # partial-noisy ->fake_clean loss = tf.reduce_mean(distances) return loss
def _reconstruction_loss(self, recon, input): #latent_code = encoder(input, self.is_training) #recon = decoder(latent_code, self.is_training) if self.loss == 'chamfer': cost_p1_p2, _, cost_p2_p1, _ = nn_distance(recon, input) loss = tf.reduce_mean(cost_p1_p2) + tf.reduce_mean(cost_p2_p1) elif self.loss == 'emd': match = approx_match(recon, input) loss = match_cost(recon, input, match) loss = tf.reduce_mean(loss) loss = tf.div(loss, self.point_cloud_shape[0]) # return point-wise loss return loss
def get_chamfer_metrics(gt_pcl, pred_pcl): ''' Obtain chamfer distance between GT and predicted point clouds args: gt_pcl: float, (BS,N_PTS,3); GT point cloud pred_pcl: float, (BS,N_PTS,3); predicted point cloud returns: dists_forward: float, (); forward chamfer distance dists_backward: float, (); backward chamfer distance chamfer_distance: float, (); chamfer distance ''' dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance(gt_pcl, pred_pcl) dists_forward = tf.reduce_mean(dists_forward, axis=1) # (BATCH_SIZE,NUM_POINTS) --> (BATCH_SIZE) dists_backward = tf.reduce_mean(dists_backward, axis=1) chamfer_distance = dists_backward + dists_forward return dists_forward, dists_backward, chamfer_distance
def get_chamfer_dist(gt_pcl, pred_pcl): ''' Calculate chamfer distance between two point clouds Args: gt_pcl: (BS,N_pts,3); GT point cloud pred_pcl: (BS,N_pts,3); predicted point cloud Returns: dists_forward: (); averaged forward chamfer distance dists_backward: (); averaged backward chamfer distance chamfer_distance: (); averaged chamfer distance ''' # FWD: GT-->Pred, Bwd: Pred-->GT dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance(gt_pcl, pred_pcl) dists_forward = tf.reduce_mean(dists_forward) # (BATCH_SIZE,NUM_POINTS) --> (BATCH_SIZE) dists_backward = tf.reduce_mean(dists_backward) chamfer_distance = dists_backward + dists_forward return dists_forward, dists_backward, chamfer_distance
def hausdorff_loss(pred, gt, radius=1.0, forward_weight=1.0, threshold=None): """ pred: BxNxC, label: BxN, forward_weight: relative weight for forward_distance """ dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance(gt, pred) # only care about distance within threshold (ignore strong outliers) if threshold is not None: dists_forward = tf.where(dists_forward < threshold, dists_forward, tf.zeros_like(dists_forward)) dists_backward = tf.where(dists_backward < threshold, dists_backward, tf.zeros_like(dists_backward)) # dists_forward is for each element in gt, the closest distance to this element dists_forward = tf.reduce_max(dists_forward, axis=1) dists_backward = tf.reduce_max(dists_backward, axis=1) CD_dist = forward_weight * dists_forward + dists_backward CD_dist_norm = CD_dist / radius cd_loss = tf.reduce_max(CD_dist_norm) return cd_loss
def get_rec_metrics(gt_pcl, pred_pcl, batch_size=10, num_points=1024): ''' Calculate chamfer and emd metrics args: gt_pcl: float, (BS,N_PTS,3); ground truth point cloud pred_pcl: float, (BS,N_PTS,3); predicted point cloud ''' dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance(gt_pcl, pred_pcl) dists_forward = tf.reduce_mean(tf.sqrt(dists_forward), axis=1) # (B, ) dists_backward = tf.reduce_mean(tf.sqrt(dists_backward), axis=1) # (B, ) chamfer_distance = dists_backward + dists_forward X,_ = tf.meshgrid(range(batch_size), range(num_points), indexing='ij') ind, _ = auction_match(pred_pcl, gt_pcl) # Ind corresponds to points in pcl_gt print X.get_shape() print ind.get_shape() ind = tf.stack((X, ind), -1) print gt_pcl.get_shape() print ind.get_shape() print emd = tf.reduce_mean(tf.sqrt(tf.reduce_sum((tf.gather_nd(gt_pcl, ind) - \ pred_pcl)**2, axis=-1)), axis=1) # (BATCH_SIZE,NUM_POINTS,3) --> (BATCH_SIZE,NUM_POINTS) --> (BATCH_SIZE) return dists_forward, dists_backward, chamfer_distance, emd
for i in range(len(pre_max_val)): #get the most important critical points if NUM_ADD < number of critical points #the importance is demtermined by counting how many elements in the global featrue vector is #contributed by one specific point idx, counts = np.unique(np.argmax(pre_max_val[i], axis=0), return_counts=True) idx_idx = np.argsort(counts) if len(counts) > NUM_ADD: points = data[i][idx[idx_idx[-NUM_ADD:]]] else: points = data[i][idx] tmp_num = NUM_ADD - len(counts) while (tmp_num > len(counts)): points = np.concatenate([points, data[i][idx]]) tmp_num -= len(counts) points = np.concatenate([points, data[i][-tmp_num:]]) critical_points.append(points) critical_points = np.stack(critical_points) return critical_points if __name__ == '__main__': with tf.Graph().as_default(): inputs = tf.zeros((32, 1024, 3)) inputs2 = tf.zeros((32, 122, 3)) outputs = get_model(inputs, tf.constant(True)) print(outputs) dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( inputs2, inputs)
dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance(gt, pred) #dists_forward is for each element in gt, the cloest distance to this element CD_dist = 0.8 * dists_forward + 0.2 * dists_backward CD_dist = tf.reduce_mean(CD_dist, axis=1) CD_dist_norm = CD_dist / radius cd_loss = tf.reduce_mean(CD_dist_norm) return cd_loss, None def get_uniform_loss_knn(pred): var, _ = knn_point(6, pred, pred) mean = tf.reduce_mean(var, axis=2) _, variance = tf.nn.moments(mean, axes=[1]) variance1 = tf.reduce_sum(variance) _, var = tf.nn.moments(var, axes=[2]) var = tf.reduce_sum(var) variance2 = tf.reduce_sum(var) return variance1 + variance2 if __name__ == '__main__': gt = tf.constant([[[1, 0, 0], [2, 0, 0], [3, 0, 0], [4, 0, 0]]], tf.float32) pred = tf.constant([[[-10, 0, 0], [1, 0, 0], [2, 0, 0], [3, 0, 0]]], tf.float32) dists_forward, idx1, dists_backward, idx2 = tf_nndistance.nn_distance( gt, pred) with tf.Session() as sess: print(idx1.eval()) # for each element in gt, the idx of pred print(idx2.eval()) # for each element in pred,
def chamfer_distance(pcd1, pcd2): dist1, _, dist2, _ = tf_nndistance.nn_distance(pcd1, pcd2) return dist1, dist2
def attack(): is_training = False with tf.Graph().as_default(): with tf.device('/gpu:' + str(GPU_INDEX)): pointclouds_pl, labels_pl = MODEL.placeholder_inputs( BATCH_SIZE, NUM_POINT) is_training_pl = tf.placeholder(tf.bool, shape=()) pert = tf.get_variable( name='pert', shape=[BATCH_SIZE, NUM_ADD, 3], initializer=tf.truncated_normal_initializer(stddev=0.01)) initial_point_pl = tf.placeholder(shape=[BATCH_SIZE, NUM_ADD, 3], dtype=tf.float32) point_added = initial_point_pl + pert pointclouds_input = tf.concat([pointclouds_pl, point_added], axis=1) pred, end_points = MODEL.get_model(pointclouds_input, is_training_pl) #adv loss adv_loss = MODEL.get_adv_loss(pred, TARGET) dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( point_added, pointclouds_pl) if FLAGS.constraint == 'c': #Chamfer dists_forward = tf.reduce_mean(dists_forward, axis=1) dists_backward = tf.reduce_mean(dists_backward, axis=1) #not used elif FLAGS.constraint == 'h': #Hausdorff dists_forward = tf.reduce_max(dists_forward, axis=1) dists_backward = tf.reduce_max(dists_backward, axis=1) #not used else: raise Exception( "Invalid constraint type. Please try c for Chamfer and h for Hausdorff" ) dist_weight = tf.placeholder(shape=[BATCH_SIZE], dtype=tf.float32) lr_attack = tf.placeholder(dtype=tf.float32) attack_optimizer = tf.train.AdamOptimizer(lr_attack) attack_op = attack_optimizer.minimize( adv_loss + tf.reduce_mean(tf.multiply(dist_weight, dists_forward)), var_list=[pert]) vl = tf.global_variables() vl = [x for x in vl if 'pert' not in x.name] saver = tf.train.Saver(vl) # Create a session config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True #config.log_device_placement = True sess = tf.Session(config=config) sess.run(tf.global_variables_initializer()) ops = { 'pointclouds_pl': pointclouds_pl, 'labels_pl': labels_pl, 'is_training_pl': is_training_pl, 'pointclouds_input': pointclouds_input, 'initial_point_pl': initial_point_pl, 'dist_weight': dist_weight, 'pert': pert, 'point_added': point_added, 'pre_max': end_points['pre_max'], 'post_max': end_points['post_max'], 'pred': pred, 'adv_loss': adv_loss, #'dists_backward':dists_backward, 'dists_forward': dists_forward, 'total_loss': tf.reduce_mean(tf.multiply(dist_weight, dists_forward)) + adv_loss, 'lr_attack': lr_attack, 'attack_op': attack_op } saver.restore(sess, MODEL_PATH) print('model restored!') dist_list = [] for victim in [ 0, 1, 2, 3 ]: #the class index of selected 10 largest classed in ModelNet40 if victim == TARGET: continue attacked_data = attacked_data_all[ victim][:25] #attacked_data shape:25*1024*3 for j in range(25 // BATCH_SIZE): dist, img = attack_one_batch( sess, ops, attacked_data[j * BATCH_SIZE:(j + 1) * BATCH_SIZE]) dist_list.append(dist) np.save( os.path.join('.', DUMP_DIR, '{}_{}_{}_adv.npy'.format(victim, TARGET, j)), img)
def gen_batch(): np.random.seed() point_cloud = np.random.rand(batch_size, cfg.VOXEL_POINT_COUNT, 3) mask = np.random.choice(a=[False, True], size=(batch_size, cfg.VOXEL_POINT_COUNT, 1), p=[0.8, 0.2]) for i in range(batch_size): if np.sum(mask[i]) == 0: mask[i, np.random.randint(cfg.VOXEL_POINT_COUNT), 0] = True return point_cloud, mask BASE_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.join(BASE_DIR, 'nn_distance')) from tf_nndistance import nn_distance dists_forward, _, dists_backward, _ = nn_distance(result, point_cloud_pl) loss_pred = tf.reduce_mean(dists_forward + dists_backward) tf.summary.scalar('loss_pred', loss_pred) if False: tvars = tf.trainable_variables() lossL1 = tf.add_n( [tf.reduce_sum(tf.abs(v)) for v in tvars if 'bias' not in v.name]) * 0.001 tf.summary.scalar('lossL1', lossL1) loss = loss_pred + lossL1 else: loss = loss_pred train = tf.train.AdamOptimizer(learning_rate=0.00005).minimize(loss) saver = tf.train.Saver(max_to_keep=2)
def __init__(self, batch_size, seq_length, num_points=1024, num_samples=8, knn=False, alpha=1.0, beta=1.0, learning_rate=0.001, max_gradient_norm=5.0, is_training=False): self.global_step = tf.Variable(0, trainable=False) self.inputs = tf.placeholder(tf.float32, [batch_size, seq_length, num_points, 3]) frames = tf.split(value=self.inputs, num_or_size_splits=seq_length, axis=1) frames = [tf.squeeze(input=frame, axis=[1]) for frame in frames] cell1 = PointLSTMCell(radius=1.0 + 1e-6, nsample=3 * num_samples, out_channels=128, knn=knn, pooling='max') cell2 = PointLSTMCell(radius=2.0 + 1e-6, nsample=2 * num_samples, out_channels=256, knn=knn, pooling='max') cell3 = PointLSTMCell(radius=4.0 + 1e-6, nsample=1 * num_samples, out_channels=512, knn=knn, pooling='max') # context states1 = None states2 = None states3 = None for i in range(int(seq_length / 2)): # 512 xyz1, _, _, _ = sample_and_group(int(num_points / 2), radius=0.5 + 1e-6, nsample=num_samples, xyz=frames[i], points=None, knn=False, use_xyz=False) with tf.variable_scope('encoder_1', reuse=tf.AUTO_REUSE) as scope: states1 = cell1((xyz1, None), states1) s_xyz1, h_feat1, _ = states1 # 256 xyz2, feat2, _, _ = sample_and_group(int(num_points / 2 / 2), radius=1.0 + 1e-6, nsample=num_samples, xyz=s_xyz1, points=h_feat1, knn=False, use_xyz=False) feat2 = tf.reduce_max(feat2, axis=[2], keepdims=False, name='maxpool') with tf.variable_scope('encoder_2', reuse=tf.AUTO_REUSE) as scope: states2 = cell2((xyz2, feat2), states2) s_xyz2, h_feat2, _ = states2 # 128 xyz3, feat3, _, _ = sample_and_group(int(num_points / 2 / 2 / 2), radius=2.0 + 1e-6, nsample=num_samples, xyz=s_xyz2, points=h_feat2, knn=False, use_xyz=False) feat3 = tf.reduce_max(feat3, axis=[2], keepdims=False, name='maxpool') with tf.variable_scope('encoder_3', reuse=tf.AUTO_REUSE) as scope: states3 = cell3((xyz3, feat3), states3) # prediction predicted_motions = [] predicted_frames = [] input_frame = frames[int(seq_length / 2) - 1] for i in range(int(seq_length / 2), seq_length): # 512 xyz1, _, _, _ = sample_and_group(int(num_points / 2), radius=0.5 + 1e-6, nsample=num_samples, xyz=input_frame, points=None, knn=False, use_xyz=False) with tf.variable_scope('decoder_1', reuse=tf.AUTO_REUSE) as scope: states1 = cell1((xyz1, None), states1) s_xyz1, h_feat1, _ = states1 # 256 xyz2, feat2, _, _ = sample_and_group(int(num_points / 2 / 2), radius=1.0 + 1e-6, nsample=num_samples, xyz=s_xyz1, points=h_feat1, knn=False, use_xyz=False) feat2 = tf.reduce_max(feat2, axis=[2], keepdims=False, name='maxpool') with tf.variable_scope('decoder_2', reuse=tf.AUTO_REUSE) as scope: states2 = cell2((xyz2, feat2), states2) s_xyz2, h_feat2, _ = states2 # 128 xyz3, feat3, _, _ = sample_and_group(int(num_points / 2 / 2 / 2), radius=2.0 + 1e-6, nsample=num_samples, xyz=s_xyz2, points=h_feat2, knn=False, use_xyz=False) feat3 = tf.reduce_max(feat3, axis=[2], keepdims=False, name='maxpool') with tf.variable_scope('decoder_3', reuse=tf.AUTO_REUSE) as scope: states3 = cell3((xyz3, feat3), states3) s_xyz3, h_feat3, _ = states3 with tf.variable_scope('fp', reuse=tf.AUTO_REUSE) as scope: l2_feat = pointnet_fp_module(xyz2, xyz3, h_feat2, h_feat3, mlp=[256], last_mlp_activation=True, scope='fp2') l1_feat = pointnet_fp_module(xyz1, xyz2, h_feat1, l2_feat, mlp=[256], last_mlp_activation=True, scope='fp1') l0_feat = pointnet_fp_module(input_frame, xyz1, None, l1_feat, mlp=[256], last_mlp_activation=True, scope='fp0') with tf.variable_scope('fc', reuse=tf.AUTO_REUSE) as scope: predicted_motion = tf.layers.conv1d( inputs=l0_feat, filters=128, kernel_size=1, strides=1, padding='valid', data_format='channels_last', activation=tf.nn.relu, name='fc1') predicted_motion = tf.layers.conv1d( inputs=predicted_motion, filters=3, kernel_size=1, strides=1, padding='valid', data_format='channels_last', activation=None, name='fc2') predicted_motions.append(predicted_motion) input_frame += predicted_motion predicted_frames.append(input_frame) # loss if is_training: self.loss = self.emd = self.cd = 0 for i in range(int(seq_length / 2)): match = tf_approxmatch.approx_match( frames[i + int(seq_length / 2)], predicted_frames[i]) emd_distance = tf.reduce_mean( tf_approxmatch.match_cost(frames[i + int(seq_length / 2)], predicted_frames[i], match)) loss_emd = emd_distance self.emd += loss_emd dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( predicted_frames[i], frames[i + int(seq_length / 2)]) loss_cd = tf.reduce_mean(dists_forward + dists_backward) self.cd += loss_cd self.loss += (alpha * loss_cd + beta * loss_emd) self.cd /= int(seq_length / 2) self.emd /= (int(seq_length / 2) * num_points) self.loss /= int(seq_length / 2) params = tf.trainable_variables() gradients = tf.gradients(self.loss, params) clipped_gradients, norm = tf.clip_by_global_norm( gradients, max_gradient_norm) self.train_op = tf.train.AdamOptimizer( learning_rate).apply_gradients(zip(clipped_gradients, params), global_step=self.global_step) self.predicted_motions = tf.stack(values=predicted_motions, axis=1) self.predicted_frames = tf.stack(values=predicted_frames, axis=1) self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)
def build_graph(resourceid, reg_weight=0.001): with tf.device('/gpu:%d' % resourceid): tflearn.init_graph(seed=1029, num_cores=2, gpu_memory_fraction=0.9, soft_placement=True) img_inp = tf.placeholder(tf.float32, shape=(BATCH_SIZE, HEIGHT, WIDTH, 1), name='img_inp') pt_gt = tf.placeholder(tf.float32, shape=(BATCH_SIZE, POINTCLOUDSIZE, 3), name='pt_gt') label = tf.placeholder(tf.float32, shape=(BATCH_SIZE, 13), name='label') x = img_inp #192 256 x = tflearn.layers.conv.conv_2d(x, 16, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x = tflearn.layers.conv.conv_2d(x, 16, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x0 = x with tf.variable_scope('transform_net1') as sc: x_d = tflearn.layers.conv.max_pool_2d(x, 8, strides=8) transform = input_transform_net( tf.reshape(x_d, (BATCH_SIZE, 4096, 3)), is_training=tf.constant(True, dtype=tf.bool), bn_decay=0.5, K=3) x = tflearn.layers.conv.conv_2d(x, 32, (3, 3), strides=2, activation='relu', weight_decay=1e-5, regularizer='L2') #96 128 x = tflearn.layers.conv.conv_2d(x, 32, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x = tflearn.layers.conv.conv_2d(x, 32, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x1 = x x = tflearn.layers.conv.conv_2d(x, 64, (3, 3), strides=2, activation='relu', weight_decay=1e-5, regularizer='L2') #48 64 x = tflearn.layers.conv.conv_2d(x, 64, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x = tflearn.layers.conv.conv_2d(x, 64, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x2 = x x = tflearn.layers.conv.conv_2d(x, 128, (3, 3), strides=2, activation='relu', weight_decay=1e-5, regularizer='L2') #24 32 x = tflearn.layers.conv.conv_2d(x, 128, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x = tflearn.layers.conv.conv_2d(x, 128, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x3 = x x = tflearn.layers.conv.conv_2d(x, 256, (5, 5), strides=2, activation='relu', weight_decay=1e-5, regularizer='L2') #12 16 x = tflearn.layers.conv.conv_2d(x, 256, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x = tflearn.layers.conv.conv_2d(x, 256, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x4 = x x = tflearn.layers.conv.conv_2d(x, 512, (5, 5), strides=2, activation='relu', weight_decay=1e-5, regularizer='L2') #6 8 x = tflearn.layers.conv.conv_2d(x, 512, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x = tflearn.layers.conv.conv_2d(x, 512, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x = tflearn.layers.conv.conv_2d(x, 512, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x5 = x x = tflearn.layers.conv.conv_2d(x, 512, (5, 5), strides=2, activation='relu', weight_decay=1e-5, regularizer='L2') #3 4 x_additional = tflearn.layers.core.fully_connected(x, 2048, activation='relu', weight_decay=1e-3, regularizer='L2') x = tflearn.layers.conv.conv_2d_transpose(x, 256, [5, 5], [6, 8], strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #6 8 x5 = tflearn.layers.conv.conv_2d(x5, 256, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x5)) x = tflearn.layers.conv.conv_2d(x, 256, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x5 = x x = tflearn.layers.conv.conv_2d_transpose(x, 128, [5, 5], [12, 16], strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #12 16 x4 = tflearn.layers.conv.conv_2d(x4, 128, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x4)) x = tflearn.layers.conv.conv_2d(x, 128, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x4 = x x = tflearn.layers.conv.conv_2d_transpose(x, 64, [5, 5], [24, 32], strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #24 32 x3 = tflearn.layers.conv.conv_2d(x3, 64, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x3)) x = tflearn.layers.conv.conv_2d(x, 64, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x3 = x x = tflearn.layers.conv.conv_2d_transpose(x, 32, [5, 5], [48, 64], strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #48 64 x2 = tflearn.layers.conv.conv_2d(x2, 32, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x2)) x = tflearn.layers.conv.conv_2d(x, 32, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x2 = x x = tflearn.layers.conv.conv_2d_transpose(x, 16, [5, 5], [96, 128], strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #96 128 x1 = tflearn.layers.conv.conv_2d(x1, 16, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x1)) x = tflearn.layers.conv.conv_2d(x, 16, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x = tflearn.layers.conv.conv_2d(x, 32, (3, 3), strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #48 64 x2 = tflearn.layers.conv.conv_2d(x2, 32, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x2)) x = tflearn.layers.conv.conv_2d(x, 32, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x2 = x x = tflearn.layers.conv.conv_2d(x, 64, (3, 3), strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #24 32 x3 = tflearn.layers.conv.conv_2d(x3, 64, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x3)) x = tflearn.layers.conv.conv_2d(x, 64, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x3 = x x = tflearn.layers.conv.conv_2d(x, 128, (5, 5), strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #12 16 x4 = tflearn.layers.conv.conv_2d(x4, 128, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x4)) x = tflearn.layers.conv.conv_2d(x, 128, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x4 = x x = tflearn.layers.conv.conv_2d(x, 256, (5, 5), strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #6 8 x5 = tflearn.layers.conv.conv_2d(x5, 256, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x5)) x = tflearn.layers.conv.conv_2d(x, 256, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x5 = x x = tflearn.layers.conv.conv_2d(x, 512, (5, 5), strides=2, activation='relu', weight_decay=1e-5, regularizer='L2') #3 4 x_additional = tflearn.layers.core.fully_connected(x_additional, 2048, activation='linear', weight_decay=1e-4, regularizer='L2') x_additional = tf.nn.relu( tf.add( x_additional, tflearn.layers.core.fully_connected(x, 2048, activation='linear', weight_decay=1e-3, regularizer='L2'))) x = tflearn.layers.conv.conv_2d_transpose(x, 256, [5, 5], [6, 8], strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #6 8 x5 = tflearn.layers.conv.conv_2d(x5, 256, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x5)) x = tflearn.layers.conv.conv_2d(x, 256, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x5 = x x = tflearn.layers.conv.conv_2d_transpose(x, 128, [5, 5], [12, 16], strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #12 16 x4 = tflearn.layers.conv.conv_2d(x4, 128, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x4)) x = tflearn.layers.conv.conv_2d(x, 128, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x4 = x x = tflearn.layers.conv.conv_2d_transpose(x, 64, [5, 5], [24, 32], strides=2, activation='linear', weight_decay=1e-5, regularizer='L2') #24 32 x3 = tflearn.layers.conv.conv_2d(x3, 64, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.nn.relu(tf.add(x, x3)) x = tflearn.layers.conv.conv_2d(x, 64, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x = tflearn.layers.conv.conv_2d(x, 64, (3, 3), strides=1, activation='relu', weight_decay=1e-5, regularizer='L2') x_additional = tflearn.layers.core.fully_connected(x_additional, 1024, activation='relu', weight_decay=1e-3, regularizer='L2') x_additional = tflearn.layers.core.fully_connected(x_additional, 256 * 3, activation='linear', weight_decay=1e-3, regularizer='L2') x_additional = tf.reshape(x_additional, (BATCH_SIZE, 256, 3)) x = tflearn.layers.conv.conv_2d(x, 3, (3, 3), strides=1, activation='linear', weight_decay=1e-5, regularizer='L2') x = tf.reshape(x, (BATCH_SIZE, 32 * 24, 3)) x = tf.concat([x_additional, x], 1) x = tf.reshape(x, (BATCH_SIZE, OUTPUTPOINTS, 3)) x = tf.matmul(x, transform) K = transform.get_shape()[1].value mat_diff = tf.matmul(transform, tf.transpose(transform, perm=[0, 2, 1])) mat_diff -= tf.constant(np.eye(K), dtype=tf.float32) mat_diff_loss = tf.nn.l2_loss(mat_diff) dists_forward, _, dists_backward, _ = tf_nndistance.nn_distance( pt_gt, x) mindist = dists_forward dist0 = mindist[0, :] dists_forward = tf.reduce_mean(dists_forward) dists_backward = tf.reduce_mean(dists_backward) loss_nodecay = (dists_forward + dists_backward / 2.0) * 10000 loss = loss_nodecay + tf.add_n( tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) ) * 0.1 + mat_diff_loss * reg_weight batchno = tf.Variable(0, dtype=tf.int32) optimizer = tf.train.AdamOptimizer( 3e-5 * BATCH_SIZE / FETCH_BATCH_SIZE).minimize(loss, global_step=batchno) batchnoinc = batchno.assign(batchno + 1) return img_inp, x, pt_gt, loss, optimizer, batchno, batchnoinc, mindist, loss_nodecay, dists_forward, dists_backward, dist0