def eval_one_epoch(sess, ops, num_votes=1): is_training = False # Make sure batch data is of same size cur_batch_data = np.zeros((BATCH_SIZE, NUM_POINT, TEST_DATASET.num_channel())) cur_batch_label = np.zeros((BATCH_SIZE), dtype=np.int32) total_correct = 0 total_seen = 0 loss_sum = 0 batch_idx = 0 total_seen_class = [0 for _ in range(NUM_CLASSES)] total_correct_class = [0 for _ in range(NUM_CLASSES)] while TEST_DATASET.has_next_batch(): batch_data, batch_label = TEST_DATASET.next_batch(augment=False) bsize = batch_data.shape[0] print('Batch: %03d, batch size: %d' % (batch_idx, bsize)) # for the last batch in the epoch, the bsize:end are from last batch cur_batch_data[0:bsize, ...] = batch_data cur_batch_label[0:bsize] = batch_label batch_pred_sum = np.zeros((BATCH_SIZE, NUM_CLASSES)) # score for classes for vote_idx in range(num_votes): # Shuffle point order to achieve different farthest samplings shuffled_indices = np.arange(NUM_POINT) np.random.shuffle(shuffled_indices) if FLAGS.normal: rotated_data = provider.rotate_point_cloud_by_angle_with_normal(cur_batch_data[:, shuffled_indices, :], vote_idx / float(num_votes) * np.pi * 2) else: rotated_data = provider.rotate_point_cloud_by_angle(cur_batch_data[:, shuffled_indices, :], vote_idx / float(num_votes) * np.pi * 2) feed_dict = {ops['pointclouds_pl']: rotated_data, ops['labels_pl']: cur_batch_label, ops['is_training_pl']: is_training} loss_val, pred_val = sess.run([ops['loss'], ops['pred']], feed_dict=feed_dict) batch_pred_sum += pred_val pred_val = np.argmax(batch_pred_sum, 1) correct = np.sum(pred_val[0:bsize] == batch_label[0:bsize]) total_correct += correct total_seen += bsize loss_sum += loss_val batch_idx += 1 for i in range(bsize): l = batch_label[i] total_seen_class[l] += 1 total_correct_class[l] += (pred_val[i] == l) log_string('eval mean loss: %f' % (loss_sum / float(batch_idx))) log_string('eval accuracy: %f' % (total_correct / float(total_seen))) log_string('eval avg class acc: %f' % ( np.mean(np.array(total_correct_class) / np.array(total_seen_class, dtype=np.float)))) class_accuracies = np.array(total_correct_class) / np.array(total_seen_class, dtype=np.float) for i, name in enumerate(SHAPE_NAMES): log_string('%10s:\t%0.3f' % (name, class_accuracies[i]))
def eval_one_epoch(sess, ops, num_votes=1, topk=1): is_training = False # Make sure batch data is of same size cur_batch_data = np.zeros((BATCH_SIZE,NUM_POINT,TEST_DATASET.num_channel())) cur_batch_label = np.zeros((BATCH_SIZE), dtype=np.int32) total_correct = 0 total_seen = 0 loss_sum = 0 batch_idx = 0 shape_ious = [] total_seen_class = [0 for _ in range(NUM_CLASSES)] total_correct_class = [0 for _ in range(NUM_CLASSES)] while TEST_DATASET.has_next_batch(): batch_data, batch_label = TEST_DATASET.next_batch(augment=False) bsize = batch_data.shape[0] print('Batch: %03d, batch size: %d'%(batch_idx, bsize)) # for the last batch in the epoch, the bsize:end are from last batch cur_batch_data[0:bsize,...] = batch_data cur_batch_label[0:bsize] = batch_label batch_pred_sum = np.zeros((BATCH_SIZE, NUM_CLASSES)) # score for classes for vote_idx in range(num_votes): # Shuffle point order to achieve different farthest samplings shuffled_indices = np.arange(NUM_POINT) np.random.shuffle(shuffled_indices) if FLAGS.normal: rotated_data = provider.rotate_point_cloud_by_angle_with_normal(cur_batch_data[:, shuffled_indices, :], vote_idx/float(num_votes) * np.pi * 2) else: rotated_data = provider.rotate_point_cloud_by_angle(cur_batch_data[:, shuffled_indices, :], vote_idx/float(num_votes) * np.pi * 2) feed_dict = {ops['pointclouds_pl']: rotated_data, ops['labels_pl']: cur_batch_label, ops['is_training_pl']: is_training} loss_val, pred_val = sess.run([ops['loss'], ops['pred']], feed_dict=feed_dict) batch_pred_sum += pred_val pred_val = np.argmax(batch_pred_sum, 1) correct = np.sum(pred_val[0:bsize] == batch_label[0:bsize]) total_correct += correct total_seen += bsize loss_sum += loss_val batch_idx += 1 for i in range(bsize): l = batch_label[i] total_seen_class[l] += 1 total_correct_class[l] += (pred_val[i] == l) log_string('eval mean loss: %f' % (loss_sum / float(batch_idx))) log_string('eval accuracy: %f'% (total_correct / float(total_seen))) log_string('eval avg class acc: %f' % (np.mean(np.array(total_correct_class)/np.array(total_seen_class,dtype=np.float)))) class_accuracies = np.array(total_correct_class)/np.array(total_seen_class,dtype=np.float) for i, name in enumerate(SHAPE_NAMES): log_string('%10s:\t%0.3f' % (name, class_accuracies[i]))
def eval_one_epoch(sess, ops, num_votes=1, topk=1): is_training = False test_idxs = np.arange(0, len(TEST_DATASET)) num_batches = (len(TEST_DATASET) + BATCH_SIZE - 1) / BATCH_SIZE total_correct = 0 total_seen = 0 loss_sum = 0 shape_ious = [] total_seen_class = [0 for _ in range(NUM_CLASSES)] total_correct_class = [0 for _ in range(NUM_CLASSES)] for batch_idx in range(num_batches): print batch_idx, num_batches start_idx = batch_idx * BATCH_SIZE end_idx = min((batch_idx + 1) * BATCH_SIZE, len(TEST_DATASET)) bsize = end_idx - start_idx batch_data, batch_label = get_batch(TEST_DATASET, test_idxs, start_idx, end_idx) batch_pred_sum = np.zeros( (BATCH_SIZE, NUM_CLASSES)) # score for classes for vote_idx in range(num_votes): # Shuffle point order to achieve different farthest samplings shuffled_indices = np.arange(NUM_POINT) np.random.shuffle(shuffled_indices) if FLAGS.normal: rotated_data = provider.rotate_point_cloud_by_angle_with_normal( batch_data[:, shuffled_indices, :], vote_idx / float(num_votes) * np.pi * 2) else: rotated_data = provider.rotate_point_cloud_by_angle( batch_data[:, shuffled_indices, :], vote_idx / float(num_votes) * np.pi * 2) feed_dict = { ops['pointclouds_pl']: rotated_data, ops['labels_pl']: batch_label, ops['is_training_pl']: is_training } loss_val, pred_val = sess.run([ops['loss'], ops['pred']], feed_dict=feed_dict) batch_pred_sum += pred_val pred_val = np.argmax(batch_pred_sum, 1) correct = np.sum(pred_val[0:bsize] == batch_label[0:bsize]) total_correct += correct total_seen += bsize loss_sum += (loss_val * float(bsize / BATCH_SIZE)) for i in range(start_idx, end_idx): l = batch_label[i - start_idx] total_seen_class[l] += 1 total_correct_class[l] += (pred_val[i - start_idx] == l) log_string('eval mean loss: %f' % (loss_sum / float(num_batches))) log_string('eval accuracy: %f' % (total_correct / float(total_seen))) log_string('eval avg class acc: %f' % (np.mean( np.array(total_correct_class) / np.array(total_seen_class, dtype=np.float)))) class_accuracies = np.array(total_correct_class) / np.array( total_seen_class, dtype=np.float) for i, name in enumerate(SHAPE_NAMES): log_string('%10s:\t%0.3f' % (name, class_accuracies[i]))
def eval_one_epoch(sess, ops, test_writer): """ ops: dict mapping from string to tf ops """ global EPOCH_CNT global BEST_ACC global BEST_CLS_ACC is_training = False # Make sure batch data is of same size cur_batch_data = np.zeros((BATCH_SIZE,NUM_POINT,TEST_DATASET.num_channel())) cur_batch_label = np.zeros((BATCH_SIZE), dtype=np.int32) total_correct = 0 total_seen = 0 loss_sum = 0 batch_idx = 0 shape_ious = [] total_seen_class = [0 for _ in range(NUM_CLASSES)] total_correct_class = [0 for _ in range(NUM_CLASSES)] log_string(str(datetime.now())) log_string('---- EPOCH %03d EVALUATION ----'%(EPOCH_CNT)) while TEST_DATASET.has_next_batch(): batch_data, batch_label = TEST_DATASET.next_batch(augment=False) bsize = batch_data.shape[0] # print('Batch: %03d, batch size: %d'%(batch_idx, bsize)) # for the last batch in the epoch, the bsize:end are from last batch cur_batch_data[0:bsize,...] = batch_data cur_batch_label[0:bsize] = batch_label if ROTATE_FLAG: batch_pred_sum = np.zeros((BATCH_SIZE, NUM_CLASSES)) # score for classes for vote_idx in range(12): # Shuffle point order to achieve different farthest samplings shuffled_indices = np.arange(NUM_POINT) np.random.shuffle(shuffled_indices) if NORMAL_FLAG: rotated_data = provider.rotate_point_cloud_by_angle_with_normal(cur_batch_data[:, shuffled_indices, :], vote_idx/float(12) * np.pi * 2) rotated_data = provider.rotate_perturbation_point_cloud_with_normal(rotated_data) else: rotated_data = provider.rotate_point_cloud_by_angle(cur_batch_data[:, shuffled_indices, :], vote_idx/float(12) * np.pi * 2) rotated_data = provider.rotate_perturbation_point_cloud(rotated_data) jittered_data = provider.random_scale_point_cloud(rotated_data[:,:,0:3]) jittered_data = provider.jitter_point_cloud(jittered_data) rotated_data[:,:,0:3] = jittered_data # else: # rotated_data = provider.rotate_point_cloud_by_angle(cur_batch_data[:, shuffled_indices, :], # vote_idx/float(12) * np.pi * 2) feed_dict = {ops['pointclouds_pl']: rotated_data, ops['labels_pl']: cur_batch_label, ops['is_training_pl']: is_training} loss_val, pred_val = sess.run([ops['loss'], ops['pred']], feed_dict=feed_dict) batch_pred_sum += pred_val pred_val = np.argmax(batch_pred_sum, 1) else: feed_dict = {ops['pointclouds_pl']: cur_batch_data, ops['labels_pl']: cur_batch_label, ops['is_training_pl']: is_training} summary, step, loss_val, pred_val = sess.run([ops['merged'], ops['step'], ops['loss'], ops['pred']], feed_dict=feed_dict) test_writer.add_summary(summary, step) pred_val = np.argmax(pred_val, 1) correct = np.sum(pred_val[0:bsize] == batch_label[0:bsize]) total_correct += correct total_seen += bsize loss_sum += loss_val batch_idx += 1 for i in range(bsize): l = batch_label[i] total_seen_class[l] += 1 total_correct_class[l] += (pred_val[i] == l) current_acc = total_correct / float(total_seen) current_cls_acc = np.mean(np.array(total_correct_class)/np.array(total_seen_class,dtype=np.float)) log_string('eval mean loss: %f' % (loss_sum / float(batch_idx))) log_string('eval accuracy: %f'% (current_acc)) log_string('eval avg class acc: %f' % (current_cls_acc)) best_acc_flag, best_cls_acc_flag = False, False if current_acc > BEST_ACC: BEST_ACC = current_acc best_acc_flag = True if current_cls_acc > BEST_CLS_ACC: BEST_CLS_ACC = current_cls_acc best_cls_acc_flag = True log_string('eval best accuracy: %f'% (BEST_ACC)) log_string('eval best avg class acc: %f'% (BEST_CLS_ACC)) EPOCH_CNT += 1 TEST_DATASET.reset() return (best_acc_flag, best_cls_acc_flag)
def eval_one_epoch(config, sess, ops, topk=1, epoch=0): is_training = False # Make sure batch data is of same size cur_batch_data = np.zeros( (config.batch_size, config.num_points, TEST_DATASET.num_channel())) cur_batch_label = np.zeros((config.batch_size), dtype=np.int32) total_correct = 0 total_seen = 0 loss_sum = 0 batch_idx = 0 shape_ious = [] predictions = [] labels = [] while TEST_DATASET.has_next_batch(): batch_data, batch_label = TEST_DATASET.next_batch(augment=False) bsize = batch_data.shape[0] # for the last batch in the epoch, the bsize:end are from last batch cur_batch_data[0:bsize, ...] = batch_data cur_batch_label[0:bsize] = batch_label batch_pred_sum = np.zeros( (config.batch_size, config.num_classes)) # score for classes for vote_idx in range(config.num_votes): # Shuffle point order to achieve different farthest samplings shuffled_indices = np.arange(config.num_points) np.random.shuffle(shuffled_indices) if config.normal: rotated_data = provider.rotate_point_cloud_by_angle_with_normal( cur_batch_data[:, shuffled_indices, :], vote_idx / float(config.num_votes) * np.pi * 2) else: rotated_data = provider.rotate_point_cloud_by_angle( cur_batch_data[:, shuffled_indices, :], vote_idx / float(config.num_votes) * np.pi * 2) feed_dict = { ops['pointclouds_pl']: rotated_data, ops['labels_pl']: cur_batch_label, ops['is_training_pl']: is_training } loss_val, pred_val = sess.run([ops['loss'], ops['pred']], feed_dict=feed_dict) batch_pred_sum += pred_val pred_val = np.argmax(batch_pred_sum, 1) correct = np.sum(pred_val[0:bsize] == batch_label[0:bsize]) predictions += pred_val[0:bsize].tolist() labels += batch_label[0:bsize].tolist() total_correct += correct total_seen += bsize loss_sum += loss_val batch_idx += 1 loss = (loss_sum / float(batch_idx)) acc = (total_correct / float(total_seen)) log(config.log_file, "EVALUATING epoch {} - loss: {} acc: {} ".format(epoch, loss, acc)) if config.test: import Evaluation_tools as et eval_file = os.path.join(config.log_dir, '{}.txt'.format(config.name)) et.write_eval_file(config.data, eval_file, predictions, labels, config.name) et.make_matrix(config.data, eval_file, config.log_dir) else: LOSS_LOGGER.log(loss, epoch, "eval_loss") ACC_LOGGER.log(acc, epoch, "eval_accuracy") TEST_DATASET.reset() return total_correct / float(total_seen)
def eval_one_time(self, topk=1, data=0): is_training = False # Make sure batch data is of same size #cur_batch_data = np.zeros((BATCH_SIZE,NUM_POINT,TEST_DATASET.num_channel())) #cur_batch_label = np.zeros((BATCH_SIZE), dtype=np.int32) cur_batch_data = np.zeros((1, data.shape[0], data.shape[1])) cur_batch_label = np.zeros((1), dtype=np.int32) total_correct = 0 total_seen = 0 loss_sum = 0 batch_idx = 0 shape_ious = [] total_seen_class = [0 for _ in range(self.NUM_CLASSES)] total_correct_class = [0 for _ in range(self.NUM_CLASSES)] batch_data, batch_label = data, 4 #TEST_DATASET.next_batch(augment=False) bsize = batch_data.shape[0] #print('Batch: %03d, batch size: %d'%(batch_idx, bsize)) # for the last batch in the epoch, the bsize:end are from last batch cur_batch_data[0:bsize, ...] = batch_data cur_batch_label[0] = batch_label batch_pred_sum = np.zeros( (self.BATCH_SIZE, self.NUM_CLASSES)) # score for classes for vote_idx in range(self.NUM_VOTES): # Shuffle point order to achieve different farthest samplings shuffled_indices = np.arange(self.NUM_POINT) np.random.shuffle(shuffled_indices) if self.NORMAL: rotated_data = provider.rotate_point_cloud_by_angle_with_normal( cur_batch_data[:, shuffled_indices, :], vote_idx / float(self.NUM_VOTES) * np.pi * 2) else: rotated_data = provider.rotate_point_cloud_by_angle( cur_batch_data[:, shuffled_indices, :], vote_idx / float(self.NUM_VOTES) * np.pi * 2) feed_dict = { self.ops['pointclouds_pl']: rotated_data, self.ops['labels_pl']: cur_batch_label, self.ops['is_training_pl']: is_training } loss_val, pred_val = self.sess.run( [self.ops['loss'], self.ops['pred']], feed_dict=feed_dict) batch_pred_sum += pred_val min_pred_val = np.argmin(batch_pred_sum, 1) pred_val = np.argmax(batch_pred_sum, 1) """ batch_pred_sum[0][pred_val[0]] = -100000000000000000000 second_pred_val = np.argmax(batch_pred_sum, 1) batch_pred_sum[0][second_pred_val[0]] = -100000000000000000000 third_pred_val = np.argmax(batch_pred_sum, 1) """ #print heapq.nlargest(3, range(len(batch_pred_sum)), batch_pred_sum.take) print(Fore.RED + self.SHAPE_NAMES[pred_val[0]]) print(Style.RESET_ALL) #print self.print_letters(self.SHAPE_NAMES[second_pred_val[0]])# self.SHAPE_NAMES[third_pred_val[0]]#, self.SHAPE_NAMES[min_pred_val[0]] """
while TEST_DATASET.has_next_batch(): batch_data, batch_label = TEST_DATASET.next_batch(augment=False) bsize = batch_data.shape[0] print('Batch: %03d, batch size: %d'%(batch_idx, bsize)) # for the last batch in the epoch, the bsize:end are from last batch cur_batch_data[0:bsize,...] = batch_data cur_batch_label[0:bsize] = batch_label batch_pred_sum = np.zeros((BATCH_SIZE, NUM_CLASSES)) # score for classes for vote_idx in range(num_votes): # Shuffle point order to achieve different farthest samplings shuffled_indices = np.arange(NUM_POINT) np.random.shuffle(shuffled_indices) if FLAGS.normal: rotated_data = provider.rotate_point_cloud_by_angle_with_normal(cur_batch_data[:, shuffled_indices, :], vote_idx/float(num_votes) * np.pi * 2) else: rotated_data = provider.rotate_point_cloud_by_angle(cur_batch_data[:, shuffled_indices, :], vote_idx/float(num_votes) * np.pi * 2) is_training = False #feed_dict = {input: rotated_data,input_1: cur_batch_label,input_2: is_training} pred_val = sess.run(output, feed_dict={input:cur_batch_data[:, shuffled_indices, :],input_2:is_training}) print(pred_val) class_accuracies = np.array(total_correct_class)/np.array(total_seen_class,dtype=np.float) for i, name in enumerate(SHAPE_NAMES): log_string('%10s:\t%0.3f' % (name, class_accuracies[i]))
def eval_one_epoch(sess, ops, test_writer, testp_writer): global EPOCH_CNT is_training = False num_votes = 4 cur_batch_data = np.zeros((BATCH_SIZE, NUM_POINT, TEST_DATASET.num_channel())) cur_batch_label = np.zeros((BATCH_SIZE), dtype=np.int32) cur_batch_dir = np.zeros((BATCH_SIZE, NUM_POINT, 6)) total_correct = 0 total_seen = 0 loss_sum = 0 batch_idx = 0 total_seen_class = [0 for _ in range(NUM_CLASSES)] total_correct_class = [0 for _ in range(NUM_CLASSES)] log_string(str(datetime.now())) log_string('---- EPOCH %03d EVALUATION ----' % (EPOCH_CNT)) while TEST_DATASET.has_next_batch(): batch_data, batch_label, batch_dir = TEST_DATASET.next_batch(augment=False) bsize = batch_data.shape[0] cur_batch_data[0:bsize, ...] = batch_data cur_batch_label[0:bsize] = batch_label cur_batch_dir[0:bsize, ...] = batch_dir batch_pred_sum = np.zeros((BATCH_SIZE, NUM_CLASSES)) for vote_idx in range(num_votes): shuffled_indices = np.arange(NUM_POINT) np.random.shuffle(shuffled_indices) if FLAGS.normal: rotated_data, rotated_dir = provider.rotate_point_cloud_by_angle_with_normal(cur_batch_data[:, shuffled_indices, :], cur_batch_dir[:, shuffled_indices, :], vote_idx / float(num_votes) * np.pi * 2) else: rotated_data, rotated_dir = provider.rotate_point_cloud_by_angle(cur_batch_data[:, shuffled_indices, :], cur_batch_dir[:, shuffled_indices, :], vote_idx / float(num_votes) * np.pi * 2) feed_dict = {ops['pointclouds_pl']: rotated_data, ops['dir_pl']: rotated_dir, ops['labels_pl']: cur_batch_label, ops['is_training_pl']: is_training} summary, step, loss_val, pred_val = sess.run([ops['merged'], ops['step'], ops['loss'], ops['pred']], feed_dict=feed_dict) batch_pred_sum += pred_val test_writer.add_summary(summary, step) pred_val = np.argmax(batch_pred_sum, 1) correct = np.sum(pred_val[0:bsize] == batch_label[0:bsize]) total_correct += correct total_seen += bsize loss_sum += loss_val batch_idx += 1 for i in range(0, bsize): l = batch_label[i] total_seen_class[l] += 1 total_correct_class[l] += (pred_val[i] == l) dev_summary = tf.Summary() dev_summary.value.add(tag="loss", simple_value=loss_sum / float(batch_idx)) dev_summary.value.add(tag="accuracy", simple_value=total_correct / float(total_seen)) dev_summary.value.add(tag="avg class acc", simple_value=np.mean( np.array(total_correct_class) / np.array(total_seen_class, dtype=np.float))) testp_writer.add_summary(dev_summary, step) log_string('eval mean loss: %f' % (loss_sum / float(batch_idx))) log_string('eval accuracy: %f' % (total_correct / float(total_seen))) log_string('eval avg class acc: %f' % ( np.mean(np.array(total_correct_class) / np.array(total_seen_class, dtype=np.float)))) EPOCH_CNT += 1 TEST_DATASET.reset() return total_correct / float(total_seen)