def nearsub(args, train_features, train_labels, test_features, test_labels): """Perform nearest subspace classification. Options: n_comp (int): number of components for PCA or SVD """ scores_pca = [] scores_svd = [] num_classes = train_labels.numpy().max() + 1 # should be correct most of the time features_sort, _ = utils.sort_dataset(train_features.numpy(), train_labels.numpy(), num_classes=num_classes, stack=False) for j in range(num_classes): pca = PCA(n_components=args.n_comp).fit(features_sort[j]) pca_subspace = pca.components_.T mean = np.mean(features_sort[j], axis=0) pca_j = (np.eye(params["fd"]) - pca_subspace @ pca_subspace.T) \ @ (test_features.numpy() - mean).T score_pca_j = np.linalg.norm(pca_j, ord=2, axis=0) svd = TruncatedSVD(n_components=args.n_comp).fit(features_sort[j]) svd_subspace = svd.components_.T svd_j = (np.eye(params["fd"]) - svd_subspace @ svd_subspace.T) \ @ (test_features.numpy()).T score_svd_j = np.linalg.norm(svd_j, ord=2, axis=0) scores_pca.append(score_pca_j) scores_svd.append(score_svd_j) test_predict_pca = np.argmin(scores_pca, axis=0) test_predict_svd = np.argmin(scores_svd, axis=0) acc_pca = utils.compute_accuracy(test_predict_pca, test_labels.numpy()) acc_svd = utils.compute_accuracy(test_predict_svd, test_labels.numpy()) print('PCA: {}'.format(acc_pca)) print('SVD: {}'.format(acc_svd)) return acc_pca
def fit(self, X, y, X_valid, y_valid): for epoch in range(self.num_of_epochs): print("epoch number: " + str(epoch + 1)) permuted_indices = np.random.permutation(X.shape[0]) for i in range(0, X.shape[0], self.batch_size): selected_data_points = np.take(permuted_indices, range(i, i + self.batch_size), mode='wrap') delta_w = self._d_cost(X[selected_data_points], y[selected_data_points], self.weights) self.weights -= delta_w * self.learning_rate training_accuracy = compute_accuracy(self.predict(X), np.argmax(y, 1)) validation_accuracy = compute_accuracy(self.predict(X_valid), np.argmax(y_valid, 1)) print("training accuracy: " + str(round(training_accuracy, 2))) print("validation accuracy: " + str(round(validation_accuracy, 2))) print("cost: " + str(self._cost(X, y, self.weights))) if self.validation_accuracy < validation_accuracy: self.validation_accuracy = validation_accuracy self.old_weights = self.weights else: self.weights = self.old_weights self.learning_rate = 0.5 * self.learning_rate
def fit(self, X, y, X_valid, y_valid): for epoch in range(self.num_of_epochs): print("epoch number: " + str(epoch + 1)) permuted_indices = np.random.permutation(X.shape[0]) for i in range(0, X.shape[0], self.batch_size): selected_data_points = np.take(permuted_indices, range(i, i+self.batch_size), mode='wrap') delta_w = self._d_cost(X[selected_data_points], y[selected_data_points], self.weights) for w, d in zip(self.weights, delta_w): w -= d*self.learning_rate for i in range(len(self.weights)): self.ema_weights[i] = self.ema_weights[i]*self.ema + self.weights[i]*(1-self.ema) training_accuracy = compute_accuracy(self.predict(X, self.weights), np.argmax(y, 1)) validation_accuracy = compute_accuracy(self.predict(X_valid, self.weights), np.argmax(y_valid, 1)) print("training accuracy: " + str(round(training_accuracy, 2))) print("validation accuracy: " + str(round(validation_accuracy, 2))) print("cost: " + str(self._cost(X, y, self.weights))) training_accuracy = compute_accuracy(self.predict(X, self.ema_weights), np.argmax(y, 1)) validation_accuracy = compute_accuracy(self.predict(X_valid, self.ema_weights), np.argmax(y_valid, 1)) print("training accuracy ema: " + str(round(training_accuracy, 2))) print("validation accuracy ema: " + str(round(validation_accuracy, 2))) self.save_average_and_std(X)
def train_model(): global best_acc, best_epoch batch_idx = 0 model.train() N = len(train_loader.dataset) train_loss, all_preds, all_targets = 0., [], [] val_preds, val_targets = [], [] for batch in train_loader: optimizer.zero_grad() loss, output = model(batch) # if params.task == '1': target = batch['labels'].numpy() valid_mask = batch['valid_mask'].numpy() test_mask = batch['test_mask'].numpy() validation_flag = (1 - valid_mask) * test_mask training_flag = test_mask * valid_mask elif params.task == '2': target = batch['ans'].numpy() valid_mask = batch['valid_mask'].numpy() test_mask = batch['test_mask'].numpy() validation_flag = (1 - valid_mask) * test_mask training_flag = test_mask * valid_mask loss.backward() optimizer.step() all_preds.append(output[training_flag == 1]) all_targets.append(target[training_flag == 1]) val_preds.append(output[validation_flag == 1]) val_targets.append(target[validation_flag == 1]) train_loss += float(loss.detach().cpu().numpy()) batch_idx += 1 all_pred = np.concatenate(all_preds, axis=0) all_target = np.concatenate(all_targets, axis=0) val_pred = np.concatenate(val_preds, axis=0) val_target = np.concatenate(val_targets, axis=0) #model.eval() if params.task == '1': train_auc = compute_auc(all_target, all_pred) val_auc = compute_auc(val_target, val_pred) train_accuracy = compute_accuracy(all_target, all_pred) val_accuracy = compute_accuracy(val_target, val_pred) print( 'Train Epoch {} Loss: {} train auc: {} train acc: {} val auc: {} val accuracy: {} n_validation : {}' .format(epoch, train_loss / batch_idx, train_auc, train_accuracy, val_auc, val_accuracy, val_target.shape)) if params.task == '2': train_accuracy = np.mean(all_target == all_pred) val_accuracy = np.mean(val_target == val_pred) print('Train Epoch {} Loss: {} train acc: {} val accuracy: {}'.format( epoch, train_loss / batch_idx, train_accuracy, val_accuracy)) if best_acc is None or val_accuracy > best_acc: best_acc = val_accuracy best_epoch = epoch print('Train Epoch {} best val accuracy: {} best epoch: {}'.format( epoch, best_acc, best_epoch))
def test(test_loader, model, args): print('Testing...') losses = AverageMeter() accuracy = AverageMeter() # Switch to evaluate mode model.eval() with torch.no_grad(): for n_episode, batch in enumerate(test_loader, 1): data, _ = [_.cuda(non_blocking=True) for _ in batch] p = args.n_support * args.n_way data_support, data_query = data[:p], data[p:] # Compute class prototypes (n_way, output_dim) class_prototypes = model(data_support).reshape(args.n_support, args.n_way, -1).mean(dim=0) # Generate labels (n_way, n_query) labels = torch.arange(args.n_way).repeat(args.n_query) labels = labels.type(torch.cuda.LongTensor) # Compute loss and metrics logits = euclidean_dist(model(data_query), class_prototypes) loss = F.cross_entropy(logits, labels) acc = compute_accuracy(logits, labels) # Record loss and accuracy losses.update(loss.item(), data_query.size(0)) accuracy.update(acc, data_query.size(0)) print('Test Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Test Accuracy {accuracy.val:.3f} ({accuracy.avg:.3f})\t'.format(loss=losses, accuracy=accuracy)) return losses.avg, accuracy.avg
def test(DCN, gen): import pdb accuracies_test = [[] for ii in gen.scales['test']] iterations_te = int(gen.num_examples_test / batch_size) for it in range(iterations_te): for i, scales in enumerate(gen.scales['test']): # depth tells how many times the dynamic model will be unrolled depth = 1 if args.dynamic: depth = scales _, length = gen.compute_length(scales, mode='test') DCN.merge.n, DCN.split.n = [length] * 2 input, tar = gen.get_batch(batch=it, scales=scales, mode='test') # forward DCN out = DCN(input, tar, length, depth, it=it, random_split=args.random_split, mode='test', dynamic=args.dynamic) Phis, Inputs_N, target, Perms, e, loss, pg_loss, var = out acc = utils.compute_accuracy(Perms[-1], target) accuracies_test[i].append(acc) print(sum(accuracies_test[i]) / float(it + 1)) accuracies_test = [sum(accs) / iterations_te for accs in accuracies_test] print('acc test:', accuracies_test) return accuracies_test
def test(DCN, gen): accuracies_test = [[] for ii in gen.scales['test']] iterations_te = int(gen.num_examples_test / batch_size) for it in range(iterations_te): for i, scales in enumerate(gen.scales['test']): # depth tells how many times the dynamic model will be unrolled depth = 1 if args.dynamic: depth = scales _, length = gen.compute_length(scales, mode='test') DCN.merge.n, DCN.split.n = [length] * 2 input, tar = gen.get_batch(batch=it, scales=scales, mode='test') # forward DCN out = DCN(input, tar, length, depth, it=it, random_split=args.random_split, mode='test', dynamic=args.dynamic) Phis, Inputs_N, target, Perms, e, loss, pg_loss, var = out acc = utils.compute_accuracy(Perms[-1], target) accuracies_test[i].append(acc) print(sum(accuracies_test[i]) / float(it + 1)) accuracies_test = [sum(accs) / iterations_te for accs in accuracies_test] print('acc test:', accuracies_test) return accuracies_test
def get_object_confusion(class1, class2, similarity_model, config): model_config = config['model'] benchmark_config = config['benchmark'] model_path = model_config['model_filename'] dataset_path = benchmark_config['dataset_path'] params = { 'dim': model_config['input_shape'], 'batch_size': benchmark_config['batch_size'], 'shuffle': False } test_dataset = ImageDataset(dataset_path, 'validation') test_dataset.prepare_specific(benchmark_config['test_cases'] // 2, class1, class2) test_generator = DataGenerator(test_dataset, **params) preds = np.array([]) gts = np.array([]) for i in tqdm(range(len(test_generator))): batch = test_generator[i] pred = similarity_model.predict_on_batch(batch[0]) preds = np.append(preds, pred.flatten()) gts = np.append(gts, batch[1]) if benchmark_config['vis_output'] and not i % benchmark_config[ 'test_cases'] // (5 * benchmark_config['batch_size']): show_output(batch[0][0], batch[0][1], pred, batch[1]) te_acc = compute_accuracy(preds, gts) print("Class 1: " + class1 + ", Class2: " + class2 + ", Distinguishability Score: " + str(te_acc)) return te_acc
def test(args): model = NTMOneShotLearningModel(args) data_loader = OmniglotDataLoader(args) saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(args.save_dir + '/' + args.model + '_' + args.label_type) with tf.Session() as sess: saver.restore(sess, ckpt.model_checkpoint_path) print( "Test Result\n1st\t2nd\t3rd\t4th\t5th\t6th\t7th\t8th\t9th\t10th\tloss" ) y_list = [] output_list = [] loss_list = [] for episode in range(args.test_batch_num): x_image, x_label, y = data_loader.fetch_batch( args, mode='test', augment=args.augment, sample_strategy=args.sample_strategy) feed_dict = { model.x_image: x_image, model.x_label: x_label, model.y: y } output, learning_loss = sess.run([model.o, model.loss], feed_dict=feed_dict) y_list.append(y) output_list.append(output) loss_list.append(learning_loss) accuracy = compute_accuracy(args, np.concatenate(y_list, axis=0), np.concatenate(output_list, axis=0)) for accu in accuracy: print('%.4f' % accu) print(np.mean(loss_list))
def main(): for file in [ 'data/breast-cancer-assignment5.txt', 'data/german-assignment5.txt' ]: data, labels, types = load_matrix_from_txt(file) splices = k_fold_split(10, data, labels) accuracies = [] for i in range(10): train_indexes = splices[i][0] test_indexes = splices[i][1] train_data = np.copy(data[train_indexes]) train_label = np.copy(labels[train_indexes]) test_data = np.copy(data[test_indexes]) test_label = np.copy(labels[test_indexes]) boost = AdaBoost() boost.train(train_data, train_label, types) class_result = boost.test(test_data) accuracy = compute_accuracy(class_result, test_label) accuracies.append(accuracy) print 'accuracy: %f' % accuracy print('file: {}, mean: {}, std: {}'.format(file, np.mean(accuracies), np.std(accuracies)))
def add_test_accuracy(self, pred, labels, perms, W, cities, oracle_costs, last=False, beam_size=2): accuracy = utils.compute_accuracy(pred, labels) costs, Paths = utils.beamsearch_hamcycle(pred.data, W.data, beam_size=beam_size) self.accuracy_test_aux.append(accuracy) self.cost_test_aux.append(np.array(costs.cpu().numpy()).mean()) self.cost_test_aux_oracle.append(np.array(oracle_costs).mean()) if last: accuracy_test = np.array(self.accuracy_test_aux).mean() self.accuracy_test.append(accuracy_test) self.accuracy_test_aux = [] cost_test = np.array(self.cost_test_aux).mean() self.cost_test.append(cost_test) self.cost_test_aux = [] cost_test_oracle = np.array(self.cost_test_aux_oracle).mean() self.cost_test_oracle.append(cost_test_oracle) self.cost_test_aux_oracle = [] self.plot_example(Paths, costs, oracle_costs, perms, cities)
def test_step(model, test_loader, loss_func, device, epoch, results): losses = AverageMeter("Loss", ':.4e') top1 = AverageMeter("Acc@1", ':6.2f') num_steps = len(test_loader) model.eval() with torch.no_grad(): for i, (data, labels) in enumerate(test_loader): data, labels = data.to(device), labels.to(device) predictions = model(data) loss = loss_func(predictions, labels) # Metrics acc1 = compute_accuracy(predictions, labels, topk=(1, ))[0] losses.update(loss.item(), data.size(0)) top1.update(acc1.item(), data.size(0)) print( "[Epoch {}; Step {}/{}] Val Loss: {:.6f}; Val Accuracy: {:.6f}"\ .format(epoch+1, i+1, num_steps, loss, acc1) ) sys.stdout.flush() results["val"]["losses"].append(losses.avg) results["val"]["top1_accs"].append(top1.avg) # Save losses and accuracies every epoch so we can plot loss and accuracy pickle.dump(results, open(results["fname"], "wb")) return losses.avg, top1.avg
def test(DCN, gen): accuracies_test = [[] for ii in gen.scales['test']] iterations_te = int(gen.num_examples_test / batch_size) for it in xrange(iterations_te): for i, scales in enumerate(gen.scales['test']): # depth tells how many times the dynamic model will be unrolled depth = 1 if args.dynamic: # at test time, split deeper depth = scales + 2 _, length = gen.compute_length(scales) DCN.merge.n, DCN.split.n = [length] * 2 input, tar = gen.get_batch(batch=it, scales=scales, mode='test') # forward DCN out = DCN(input, tar, length, depth, it=it, mergesort_split=args.mergesort_split, quicksort_merge=args.quicksort_merge, mode='test', dynamic=args.dynamic) Phis, Inputs_N, target, Perms, e, loss, pg_loss, var = out if not args.quicksort_merge: acc = utils.compute_accuracy(Perms[-1], target) else: acc = 1 - loss.data.cpu().numpy() accuracies_test[i].append(acc) accuracies_test = [sum(accs) / iterations_te for accs in accuracies_test] return accuracies_test
def build_model(self,): utils.prepare_data(data_file=self.data_file) self.lap_list, self.feature = utils.load_gcn_data(self.graph_file, self.num_support) self.num_feature = self.feature.shape[1] self.x = tf.placeholder(tf.float32, [None, self.d_input_step, self.d_input_size]) self.z = tf.placeholder(tf.float32, [None, self.g_input_step, self.g_input_size]) self.z_t = tf.placeholder(tf.float32, [None, self.g_input_step, self.g_input_size]) self.lap = tf.placeholder(tf.float32, [self.num_support, self.d_input_size, self.d_input_size]) self.fea = tf.placeholder(tf.float32, [self.d_input_size, self.num_feature]) self.x_ = self.generator(self.z, self.g_input_step, self.g_input_size, self.g_hidden_size, self.g_batch_size) self.D = self.discriminator(self.x, self.d_input_step, self.d_input_size, self.d_hidden_size, 1, self.g_batch_size) self.D_ = self.discriminator(self.x_, self.d_input_step, self.d_input_size, self.d_hidden_size, 1, self.g_batch_size, reuse=True) if self.wgan == 1: self.d_loss_real = tf.reduce_mean(self.D) self.d_loss_fake = tf.reduce_mean(self.D_) self.g_loss = self.d_loss_fake self.d_loss = self.d_loss_real - self.d_loss_fake else: self.d_loss_real = utils.compute_loss(self.D, tf.ones_like(self.D)) self.d_loss_fake = utils.compute_loss(self.D_, tf.zeros_like(self.D_)) self.g_loss = utils.compute_loss(self.D_, tf.ones_like(self.D_)) self.d_loss = self.d_loss_real + self.d_loss_fake self.accuracy = utils.compute_accuracy(self.z_t, self.z_)
def build(self): self.add_placeholders() self.keep_prob = tf.placeholder(tf.float32) self.pred = self.add_prediction_op() self.loss = self.add_loss_op(self.pred) self.train_op = self.add_training_op(self.loss) self.accuracy = utils.compute_accuracy(self.pred, self.y_)
def test_knn(): data, _ = utils.read_table("combined_data_normalized.csv", True) class_index = 4 predictors = [2, 3, 5, 9] results = knn.knn_classifier(data, class_index, predictors, 5, 5) accuracy = utils.compute_accuracy(results) print(accuracy) utils.confusion_matrix(results, "Crime Rate?", "KNN Classifier Prediction of Crime Rate")
def test_naive_bayes(): data, header = utils.read_table("combined_data_normalized.csv", True) class_index = 4 predictors = [2, 3, 5, 7] results = bayes.naive_bayes_classifier(data, header, 10, class_index, predictors, [2, 3, 5, 9]) accuracy = utils.compute_accuracy(results) print(accuracy) utils.confusion_matrix(results, "Crime Rate?", "Naive Bayes Classifier Prediction of Crime Rate")
def fit(self, X_train, y_train, X_valid, y_valid): best_validation_accuracy = 0 for epoch in range(self.num_of_epochs): print("epoch number: " + str(epoch + 1)) permuted_indices = np.random.permutation(X_train.shape[0]) for i in tqdm(range(0, X_train.shape[0], self.batch_size)): selected_data_points = np.take(permuted_indices, range(i, i + self.batch_size), mode='wrap') self.sess.run( self.optimizer, { self.X: X_train[selected_data_points], self.y: y_train[selected_data_points], self.prob: self.keep_prob, self.is_training: True }) training_predictions = self.get_predictions(X_train) training_accuracy = compute_accuracy(training_predictions, np.argmax(y_train, 1)) validation_preditions = self.get_predictions(X_valid) validation_accuracy = compute_accuracy(validation_preditions, np.argmax(y_valid, 1)) if validation_accuracy > best_validation_accuracy: best_validation_accuracy = validation_accuracy self.saver.save(sess=self.sess, save_path=self.save_dir + "best_model") print("training accuracy: " + str(round(training_accuracy, 2))) print("validation accuracy: " + str(round(validation_accuracy, 2))) summary = tf.Summary(value=[ tf.Summary.Value(tag="training accuracy", simple_value=training_accuracy) ]) self.writer.add_summary(summary, epoch) summary = tf.Summary(value=[ tf.Summary.Value(tag="validation accuracy", simple_value=validation_accuracy) ]) self.writer.add_summary(summary, epoch)
def test_decision_tree(): data, header = utils.read_table("combined_data_discretized.csv", True) class_index = 4 predictors = [2, 3, 5, 9] results = dtree.decision_tree_classifier(data, header, class_index, predictors, 30) accuracy = utils.compute_accuracy(results) print(accuracy) utils.confusion_matrix( results, "Crime Rate?", "Decision Tree Classifier Prediction of Crime Rate")
def test_random_forest(): data, header = utils.read_table("combined_data_discretized.csv", True) class_index = 4 predictors = [2, 3, 5, 9] results = rforest.random_forest_classifier(data, header, class_index, predictors, 100, 25, 3) accuracy = utils.compute_accuracy(results) print(accuracy) utils.confusion_matrix( results, "Crime Rate?", "Random Forest Classifier Prediction of Crime Rate")
def src_supervised_step(self, src_end_points, src_labels): # compute source classification loss src_classification_loss = self.class_criterion( src_end_points['logits'], src_labels) self.losses_dict['src_classification_loss'] = src_classification_loss # compute source train accuracy src_train_accuracy = compute_accuracy(src_end_points['logits'], src_labels, acc_metric=self.acc_metric) self.src_train_acc_queue.put(src_train_accuracy)
def eval_tgt(self, tgt_test_collection): tgt_test_acc = compute_accuracy(tgt_test_collection['logits'], tgt_test_collection['true_labels'], acc_metric=self.acc_metric, print_result=True) tgt_test_acc = round(tgt_test_acc, 3) self.acc_dict['tgt_test_acc'] = tgt_test_acc # self.acc_dict['tgt_best_test_acc'] = max(self.acc_dict['tgt_best_test_acc'], tgt_test_acc) if self.acc_dict['tgt_test_acc'] > self.acc_dict['tgt_best_test_acc']: self.acc_dict['tgt_best_test_acc'] = self.acc_dict['tgt_test_acc'] self.save_checkpoint() self.print_acc()
def validate(val_loader, model, att, args): print('Validating...') losses = AverageMeter() accuracy = AverageMeter() # Switch to evaluate mode model.eval() att.eval() with torch.no_grad(): for n_episode, batch in enumerate(val_loader, 1): data, _ = [_.cuda(non_blocking=True) for _ in batch] p = args.n_support * args.n_way_val data_support, data_query = data[:p], data[p:] # Compute class prototypes (n_way, output_dim) # Calculate weighted averages for class prototypes # (n_support, n_way_val feature_dimension) latent_vecs_val = model(data_support).reshape( args.n_support, args.n_way_val, -1) # (n_way_train, n_val, feature_dimension) latent_vecs_val = latent_vecs_val.transpose(0, 1) # _, scores_val = att(latent_vecs_val) scores_val = F.softmax(att(latent_vecs_val), dim=1) # scores_val = scores_val.unsqueeze(-1).expand_as(latent_vecs_val) scores_val = scores_val.expand_as(latent_vecs_val) # class_prototypes = torch.sum( # torch.matmul(scores_val, latent_vecs_val), 1) class_prototypes = torch.sum( torch.mul(scores_val, latent_vecs_val), 1) # class_prototypes = att(model(data_support)).reshape( # args.n_support, args.n_way_val, -1).mean(dim=0) # Generate labels (n_way, n_query) labels = torch.arange(args.n_way_val).repeat(args.n_query_val) labels = labels.type(torch.cuda.LongTensor) # Compute loss and metrics logits = euclidean_dist(model(data_query), class_prototypes) loss = F.cross_entropy(logits, labels) acc = compute_accuracy(logits, labels) # Record loss and accuracy losses.update(loss.item(), data_query.size(0)) accuracy.update(acc, data_query.size(0)) print('Validation Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Validation Accuracy {accuracy.val:.3f} ({accuracy.avg:.3f})\t'. format(loss=losses, accuracy=accuracy)) return losses.avg, accuracy.avg
def test(self): """ test func. runs the model on the test data. :return: list of predictions and acc """ y_list = [] y_hat_list = [] for ex_dict in ut.TEST_LIST: y_list.append(ex_dict[1]) y_hat_list.append(self.predict(ex_dict[0])) acc = ut.compute_accuracy(y_hat_list, y_list) return y_hat_list, acc
def knn(args, train_features, train_labels, test_features, test_labels): """Perform k-Nearest Neighbor classification using cosine similarity as metric. Options: k (int): top k features for kNN """ sim_mat = train_features @ test_features.T topk = sim_mat.topk(k=args.k, dim=0) topk_pred = train_labels[topk.indices] test_pred = topk_pred.mode(0).values.detach() acc = utils.compute_accuracy(test_pred.numpy(), test_labels.numpy()) print("kNN: {}".format(acc)) return acc
def test(self, flags, ite, log_prefix, log_dir='logs/', batImageGenTest=None): # switch on the network test mode self.network.eval() images_test, labels_test = batImageGenTest threshold = 50 if len(images_test) > threshold: n_slices_test = len(images_test) / threshold indices_test = [] for per_slice in range(int(n_slices_test - 1)): indices_test.append(len(images_test) * (per_slice + 1) / n_slices_test) test_image_splits = np.split(images_test, indices_or_sections=indices_test) # Verify the splits are correct test_image_splits_2_whole = np.concatenate(test_image_splits) assert np.all(images_test == test_image_splits_2_whole) # split the test data into splits and test them one by one test_image_preds = [] for test_image_split in test_image_splits: images_test = Variable(torch.from_numpy(np.array(test_image_split, dtype=np.float32))) outputs, end_points = self.network(images_test) predictions = end_points['Predictions'] predictions = predictions.data.numpy() test_image_preds.append(predictions) # concatenate the test predictions first predictions = np.concatenate(test_image_preds) else: images_test = Variable(torch.from_numpy(np.array(images_test, dtype=np.float32))) outputs, end_points = self.network(images_test) predictions = end_points['Predictions'] predictions = predictions.data.numpy() accuracy = compute_accuracy(predictions=predictions, labels=labels_test) print('----------accuracy test----------:', accuracy) if not os.path.exists(log_dir): os.makedirs(log_dir) log_path = os.path.join(log_dir, '{}.txt'.format(log_prefix)) write_log(str('ite:{}, accuracy:{}'.format(ite, accuracy)), log_path=log_path) # switch on the network train mode after test self.network.train() return accuracy
def train(self): """ train function. runs the model on the train data. :return: acc """ y_list = [] y_hat_list = [] for ex_dict in ut.EXAMPLES_LIST: y_list.append(ex_dict[1]) y_hat_list.append(self.predict(ex_dict[0])) acc = ut.compute_accuracy(y_hat_list, y_list) return acc
def train(self, epochs: int, val_frequency: int, print_frequency: int = 20, log_frequency: int = 5, start_epoch: int = 0): self.model.train() for epoch in range(start_epoch, epochs): self.model.train() data_load_start_time = time.time() for i, (inputs, target, filename) in enumerate(self.train_loader): batch = inputs.to(self.device) labels = target.to(self.device) data_load_end_time = time.time() logits = self.model.forward(batch) loss = self.criterion(logits, labels) loss.backward() self.optimizer.step() self.optimizer.zero_grad() with torch.no_grad(): preds = logits.argmax(-1) accuracy = compute_accuracy(labels, preds) data_load_time = data_load_end_time - data_load_start_time step_time = time.time() - data_load_end_time if ((self.step + 1) % log_frequency) == 0: self.log_metrics(epoch, accuracy, loss, data_load_time, step_time) if ((self.step + 1) % print_frequency) == 0: self.print_metrics(epoch, accuracy, loss, data_load_time, step_time) self.step += 1 data_load_start_time = time.time() if (epoch + 1) % self.checkpoint_frequency == 0 or (epoch + 1) == epochs: print(f"Saving model to {self.checkpoint_path}") torch.save({ "epoch": epoch, "model": self.model.state_dict() }, self.checkpoint_path) self.summary_writer.add_scalar("epoch", epoch, self.step) if ((epoch + 1) % val_frequency) == 0: self.validate() self.model.train()
def fit(self, X_train, y_train, X_valid, y_valid): for epoch in range(self.num_of_epochs): print("epoch number: " + str(epoch + 1)) permuted_indices = np.random.permutation(X_train.shape[0]) for i in range(0, X_train.shape[0], self.batch_size): selected_data_points = np.take(permuted_indices, range(i, i + self.batch_size), mode='wrap') _, image = self.sess.run( [self.optimizer, self.image], { self.X: X_train[selected_data_points], self.y: y_train[selected_data_points], self.prob: self.keep_prob, self.is_training: True }) training_accuracy = compute_accuracy(self.predict(X_train), np.argmax(y_train, 1)) validation_accuracy = compute_accuracy(self.predict(X_valid), np.argmax(y_valid, 1)) print("training accuracy: " + str(round(training_accuracy, 2))) print("validation accuracy: " + str(round(validation_accuracy, 2))) summary = tf.Summary(value=[ tf.Summary.Value(tag="training accuracy", simple_value=training_accuracy) ]) self.writer.add_summary(summary, epoch) summary = tf.Summary(value=[ tf.Summary.Value(tag="validation accuracy", simple_value=validation_accuracy) ]) self.writer.add_summary(summary, epoch) self.writer.add_summary(image, epoch)
def prepare_tgt_conf_dataset(self): src_test_collection = self.collect_samples('src_test') tgt_test_collection = self.collect_samples('tgt_test') tgt_pseudo_probabilities = self.tgt_pseudo_labeler.pseudo_label_tgt( src_test_collection, tgt_test_collection) tgt_pseudo_acc = compute_accuracy(tgt_pseudo_probabilities, tgt_test_collection['true_labels'], acc_metric=self.acc_metric, print_result=False) self.acc_dict['tgt_pseudo_acc'] = tgt_pseudo_acc self.eval_tgt(tgt_test_collection) tgt_pseudo_confidences, tgt_pseudo_labels = torch.max( tgt_pseudo_probabilities, dim=1) tgt_conf_mask = tgt_pseudo_confidences.ge(self.thresh) tgt_conf_indices = torch.tensor(range( self.tgt_size)).cuda()[tgt_conf_mask].tolist() tgt_conf_predictions = tgt_pseudo_labels[tgt_conf_mask].tolist() self.tgt_conf_pair = list(zip(tgt_conf_indices, tgt_conf_predictions)) self.data_loader['tgt_conf'] = ConfidentDataLoader( self.tgt_file, self.train_data_loader_kwargs, self.tgt_conf_pair, self.min_conf_samples, training=True) if self.data_loader['tgt_conf'].data_loader is None: self.data_iterator['tgt_conf'] = None else: self.data_iterator['tgt_conf'] = iter(self.data_loader['tgt_conf']) tgt_non_conf_mask = tgt_pseudo_confidences.lt(self.thresh) self.tgt_non_conf_indices = torch.tensor(range( self.tgt_size)).cuda()[tgt_non_conf_mask].tolist() self.data_loader['tgt_non_conf'] = NonConfidentDataLoader( self.tgt_file, self.train_data_loader_kwargs, self.tgt_non_conf_indices, training=True) if self.data_loader['tgt_non_conf'].data_loader is None: self.data_iterator['tgt_non_conf'] = None else: self.data_iterator['tgt_non_conf'] = iter( self.data_loader['tgt_non_conf'])
def validate(opt, model, data_transform): total_TP = 0.0 total_FP = 0.0 total_FN = 0.0 # switch to evaluate mode model.eval() img_dir = '{:s}/images/val'.format(opt.train['data_dir']) label_dir = opt.test['label_dir'] img_names = os.listdir(img_dir) for img_name in img_names: # load test image img_path = '{:s}/{:s}'.format(img_dir, img_name) img = Image.open(img_path) name = os.path.splitext(img_name)[0] label_path = '{:s}/{:s}_label_point.png'.format(label_dir, name) gt = io.imread(label_path) input, label = data_transform((img, Image.fromarray(gt))) input = input.unsqueeze(0) prob_map = get_probmaps(input, model, opt) prob_map = prob_map.cpu().numpy() pred = prob_map > opt.test['threshold'] # prediction pred_labeled, N = measure.label(pred, return_num=True) if N > 1: bg_area = ski_morph.remove_small_objects(pred_labeled, opt.post['max_area']) > 0 large_area = ski_morph.remove_small_objects( pred_labeled, opt.post['min_area']) > 0 pred = pred * (bg_area == 0) * (large_area > 0) TP, FP, FN = utils.compute_accuracy(pred, gt, radius=opt.r1) total_TP += TP total_FP += FP total_FN += FN recall = float(total_TP) / (total_TP + total_FN + 1e-8) precision = float(total_TP) / (total_TP + total_FP + 1e-8) F1 = 2 * precision * recall / (precision + recall + 1e-8) logger.info('\t=> Val Avg:\tRecall {:.4f}\tPrec {:.4f}\tF1 {:.4f}'.format( recall, precision, F1)) return recall, precision, F1
def train(DCN, logger, gen): Loss, Loss_reg = [], [] Discard_rates = [[[] for jj in range(ii)] for ii in gen.scales['train']] Accuracies_tr = [[] for ii in gen.scales['train']] Accuracies_te = [[] for ii in gen.scales['test']] iterations_tr = int(gen.num_examples_train / batch_size) for epoch in range(num_epochs): lr = DCN.upd_learning_rate(epoch) for it in range(iterations_tr): losses = 0.0 variances = 0.0 start = time.time() for i, scales in enumerate(gen.scales['train']): # depth tells how many times the dynamic model will be unrolled depth = 1 if args.dynamic: depth = scales _, length = gen.compute_length(scales, mode='train') DCN.merge.n, DCN.split.n = [length] * 2 input, tar = gen.get_batch(batch=it, scales=scales, mode='train') # forward DCN out = DCN(input, tar, length, depth, it=it, epoch=epoch, random_split=args.random_split, mode='train', dynamic=args.dynamic) Phis, Inputs_N, target, Perms, e, loss, pg_loss, var = out # backward DCN if args.dynamic: DCN.step_split(pg_loss, var, regularize=args.regularize_split) DCN.step_merge(loss) losses += loss variances += var # Save discard rates discard_rates = utils.compute_dicard_rate(Perms) for sc in range(len(Perms) - 1): Discard_rates[i][sc].append(discard_rates[sc]) Accuracies_tr[i].append(utils.compute_accuracy(Perms[-1], target)) losses /= len(gen.scales['train']) variances /= len(gen.scales['train']) # optimizer.step() Loss.append(losses.data.cpu().numpy()) Loss_reg.append(variances.data.cpu().numpy()) elapsed = time.time() - start if it % 64 == 0: print('TRAINING --> | Epoch {} | Batch {} / {} | Loss {} |' ' Accuracy Train {} | Elapsed {} | dynamic {} |' ' Learning Rate {}' .format(epoch, it, iterations_tr, losses.data.cpu().numpy(), Accuracies_tr[-1][-1], elapsed, args.dynamic, lr)) logger.plot_Phis_sparsity(Phis, fig=0) logger.plot_norm_points(Inputs_N, e, Perms, gen.scales['train'][-1], fig=1) logger.plot_losses(Loss, Loss_reg, fig=2) logger.plot_rates(Discard_rates, fig=3) logger.plot_accuracies(Accuracies_tr, scales=gen.scales['train'], mode='train', fig=4) # keep track of output examples # for perm in Perms: # print('permutation', perm.data[0, 1:].cpu().numpy()) # print('target', target[0].data.cpu().numpy()) if it % 1000 == 1000 - 1: print('Saving model parameters') DCN.save_split(args.path) DCN.save_merge(args.path) accuracies_test = test(DCN, gen) for i, scales in enumerate(gen.scales['test']): Accuracies_te[i].append(accuracies_test[i]) show_accs = ' | '.join(['Acc Len {} -> {}' .format(scales, Accuracies_te[i][-1]) for i, scales in enumerate(gen.scales['test'])]) print('TESTING --> epoch {} '.format(epoch) + show_accs) logger.plot_accuracies(Accuracies_te, scales=gen.scales['test'], mode='test', fig=2) logger.save_results(Loss, Accuracies_te, Discard_rates)