def q4confmat(full_dat, noisy_dat): ref_dict = full_dat.getDictionary() # ground truth labels annotations = [] for attrib in noisy_dat.attrib: attribString = ','.join(str(v) for v in attrib) if not attribString in ref_dict: print("ERROR: attribString not present!") continue annotations.append(ref_dict[attribString]) evaluator = Evaluator() c_matrix = evaluator.confusion_matrix(noisy_dat.labels, annotations) print(c_matrix) target_names = ["A", "C", "E", "G", "O", "Q"] plot_confusion_matrix(c_matrix, target_names, "Noisy vs Full") precision, macro_p = evaluator.precision(c_matrix) recall, macro_r = evaluator.recall(c_matrix) f1, macro_f1 = evaluator.f1_score(c_matrix) p = np.append(precision, macro_p) r = np.append(recall, macro_r) f1 = np.append(f1, macro_f1) performance_matrix = np.vstack((p, np.vstack((r, f1)))) print(performance_matrix) plot_other_stats(performance_matrix, "Train_noisy") return
def test(model_file, test_file, device=-1): context = utils.Saver.load_context(model_file) if context.seed is not None: utils.set_random_seed(context.seed, device) test_dataset = context.loader.load(test_file, train=False, bucketing=True) kwargs = dict(context) if context.model_config is not None: kwargs.update(context.model_config) model = _build_parser(**dict(kwargs)) chainer.serializers.load_npz(model_file, model) if device >= 0: chainer.cuda.get_device_from_id(device).use() model.to_gpu(device) pbar = training.listeners.ProgressBar(lambda n: tqdm(total=n)) pbar.init(len(test_dataset)) evaluator = Evaluator(model, context.loader.rel_map, test_file, logging.getLogger()) utils.chainer_train_off() for batch in test_dataset.batch(context.batch_size, colwise=True, shuffle=False): xs, ts = batch[:-1], batch[-1] ys = model.forward(*xs) evaluator.on_batch_end({'train': False, 'xs': xs, 'ys': ys, 'ts': ts}) pbar.update(len(ts)) evaluator.on_epoch_validate_end({})
def cross_validation(x, y, k): xpart, ypart = data_split(x, y, k) accuracy = np.zeros(k) classifiers = np.empty(k, dtype=object) for i in range(k): # split data correctly xval = xpart[i] yval = ypart[i] xtrain = np.delete(xpart, i, 0).reshape((k - 1) * xval.shape[0], xval.shape[1]) ytrain = np.delete(ypart, i, 0).reshape((k - 1) * xval.shape[0], 1) # train on training slice classifiers[i] = DecisionTreeClassifier() classifiers[i] = classifiers[i].train(xtrain, ytrain) #predict for test class predictions = classifiers[i].predict(xval) # validate using statistics eval = Evaluator() confusion = eval.confusion_matrix(predictions, yval) accuracy[i] = eval.accuracy(confusion) return accuracy, classifiers
def main(): from utils.hyp import parse_args args = parse_args() trainer = Trainer(args) evaluator = Evaluator(args) for epoch in range(trainer.start_epoch, args.epochs): trainer.training(epoch) if not args.no_val and epoch % args.validate == (args.validate - 1): map, mf1 = evaluator.validation(trainer.model, epoch) val_svar_pf = '[mode: val ' +\ 'mAP: %5.4g, ' % map +\ 'mF1: %5.4g]' % mf1 trainer.saver.save_log(val_svar_pf) if args.visdom: update_vis_plot(trainer.vis, epoch, [map, mf1], trainer.val_plot, 'append') if evaluator.is_best: trainer.best_pred = evaluator.new_pred if args.is_save: # save checkpoint every epoch trainer.saver.save_checkpoint({ 'epoch': epoch, 'state_dict': trainer.model.module.state_dict() \ if args.ng > 1 and args.use_multi_gpu else trainer.model.state_dict(), 'optimizer': trainer.optimizer.state_dict(), 'best_pred': evaluator.best_pred, }, evaluator.is_best)
def getAccuracy(self): evaluator = Evaluator() predictions = self.decisionTreeClassifier.predict( self.validationAttrib) c_matrix = evaluator.confusion_matrix(predictions, self.validationLabel) return evaluator.accuracy(c_matrix)
def test(model_file, test_file, device=-1): context = utils.Saver.load_context(model_file) if context.seed is not None: utils.set_random_seed(context.seed, device) test_dataset = context.loader.load(test_file, train=False, bucketing=True) model = _build_parser(**dict(context)) chainer.serializers.load_npz(model_file, model) if device >= 0: chainer.cuda.get_device_from_id(device).use() model.to_gpu(device) pbar = training.listeners.ProgressBar(lambda n: tqdm(total=n)) pbar.init(len(test_dataset)) evaluator = Evaluator(model, context.loader.rel_map, test_file, Log.getLogger()) utils.chainer_train_off() for batch in test_dataset.batch(context.batch_size, colwise=True, shuffle=False): xs, ts = batch[:-1], batch[-1] parsed = model.parse(*xs) evaluator.append([tokens[1:] for tokens in xs[-1]], parsed) pbar.update(len(ts)) evaluator.report(show_details=False)
def calculate_best_pruned_tree(self, original_tree, trees, x_val, y_val): eval = Evaluator() classifier = DecisionTreeClassifier() classifier.is_trained = True original_predictions = classifier.predict(x_val, original_tree) original_error = self.get_apperent_error_rate(original_predictions, y_val) stored_j = 0 initial_alpha = 0 previous_diff = 0 previous_alpha = 0 #go through each tree and compute the ratio of caculated error (right/total) for j in range(1, len(trees)): predictions = classifier.predict(x_val, trees[j]) error = self.get_apperent_error_rate(predictions, y_val) original_number_leaves = self.count_leaves(original_tree) number_of_leaves = self.count_leaves(trees[j]) alpha = (original_error - error) / (original_number_leaves - number_of_leaves) diff = initial_alpha - alpha if diff < previous_diff: stored_j = j previous_alpha = alpha return trees[stored_j], previous_alpha
def get_evaluator(self): evaldataset = CityscapesSelectDataset(im_size=self.args.img_size, n_samples=self.args.eval_samples) evaluator = Evaluator(evaldataset, samples=25, metrics=["miou"], crf=False) return evaluator
def prune(node, dataset): """ Recursively prunes a decision tree classifier Parameters ---------- root : Node the node being considered for pruning dataset: Dataset the dataset being used for pruning """ if not node.is_leaf(): two_leaves = node.right.is_leaf() and node.left.is_leaf() if not two_leaves: if (node.left.is_node()): prune(node.left, dataset) if (node.right.is_node()): prune(node.right, dataset) two_leaves = node.right.is_leaf() and node.left.is_leaf() if two_leaves: annotation = dataset.labels attributes = dataset.attributes # Try and prune current node # Calculate accuracy before pruning evaluator = Evaluator() predictions_before = tree.predict(attributes) confusion = evaluator.confusion_matrix(predictions_before, annotation) accuracy_before = evaluator.accuracy(confusion) # Store leaves and rule temporarily temp_label_left = node.left.label temp_label_right = node.right.label temp_rule = node.rule # Prune current node node.label = node.majority_label node.left.label = None node.right.label = None node.rule = None # Calculate accuracy after pruning predictions_after = tree.predict(attributes) confusion = evaluator.confusion_matrix(predictions_after, annotation) accuracy_after = evaluator.accuracy(confusion) # Restore node if accuracy dropped if (accuracy_after < accuracy_before): node.left.label = temp_label_left node.right.label = temp_label_right node.label = None node.rule = temp_rule
def old_test(): # data_read("data/toy.txt") prediction = ["A", "B"] annotation = ["A", "A"] class_labels = ["B", "A"] obj = Evaluator() matrix = obj.confusion_matrix(prediction, annotation, class_labels) print(str.format('{0:.15f}', obj.accuracy(matrix))) print(obj.precision(matrix)) print(obj.recall(matrix)) print(obj.f1_score(matrix))
def prune(tree: BinTree): vld_dataset = data_read("data/validation.txt") x_val, y_val = vld_dataset.shim_to_arrays() ev = Evaluator() for i in range(10): print(f"----prune attempt {i + 1}---") tree.prune(node=tree.root_node, og_vld_feats=x_val, og_vld_lbls=y_val, dataset=vld_dataset, ev=ev, is_aggressive=False)
def evaluate_model(model, num_classes, X_test, Y_test): predicted = model.predict(X_test) target_label = convert_prob_to_label(Y_test) predicted_label = convert_prob_to_label(predicted) target = np.ravel(target_label) predicted = np.ravel(predicted_label) evaluator = Evaluator(num_classes) accuracy = evaluator.calculate_accuracy(target, predicted) cm, precisions, recalls = evaluator.calculate_metrics(target, predicted) precision = np.mean(precisions) recall = np.mean(recalls) f1 = 2 * precision * recall / (precision + recall) return Result(cm, accuracy, precisions, precision, recalls, recall, f1)
def cross_validation(k, filename): """ Performs cross validation on a dataset Parameters ---------- k : int number of times dataset is split filename : string name of the file to load the dataset Returns ------- list of ints containing the accuracies of each split int global error estimate """ file_path = "./data/" + filename dataset = np.loadtxt(file_path, dtype=str, delimiter=',') np.random.shuffle(dataset) subsets = np.array_split(dataset, k) accuracies = [] for i in range(k): train = np.delete(subsets, i, axis=0) train = np.concatenate(train) train_att = train[:, :-1].astype(int) train_labels = train[:, -1] test = subsets[i] test_att = test[:, :-1].astype(int) test_labels = test[:, -1] tree = DecisionTreeClassifier() tree = tree.train(train_att, train_labels) prediction = tree.predict(test_att) evaluator = Evaluator() confusion = evaluator.confusion_matrix(prediction, test_labels) a = evaluator.accuracy(confusion) accuracies.append(a) global_error_estimate = np.mean(accuracies) np.set_printoptions(formatter={'float': '{: 0.4f}'.format}) return accuracies, global_error_estimate
def main(): print("Loading the training dataset...") x = np.array([[5, 7, 1], [4, 6, 2], [4, 6, 3], [1, 3, 1], [2, 1, 2], [5, 2, 6]]) y = np.array(["A", "A", "A", "C", "C", "C"]) print("Training the decision tree...") classifier = DecisionTreeClassifier() classifier = classifier.train(x, y) print("Loading the test set...") x_test = np.array([[1, 6, 3], [0, 5, 5], [1, 5, 0], [2, 4, 2]]) y_test = np.array(["A", "A", "C", "C"]) predictions = classifier.predict(x_test) print("Predictions: {}".format(predictions)) classes = ["A", "C"] print("Evaluating test predictions...") evaluator = Evaluator() confusion = evaluator.confusion_matrix(predictions, y_test) print("Confusion matrix:") print(confusion) accuracy = evaluator.accuracy(confusion) print() print("Accuracy: {}".format(accuracy)) (p, macro_p) = evaluator.precision(confusion) (r, macro_r) = evaluator.recall(confusion) (f, macro_f) = evaluator.f1_score(confusion) print() print("Class: Precision, Recall, F1") for (i, (p1, r1, f1)) in enumerate(zip(p, r, f)): print("{}: {:.2f}, {:.2f}, {:.2f}".format(classes[i], p1, r1, f1)) print() print("Macro-averaged Precision: {:.2f}".format(macro_p)) print("Macro-averaged Recall: {:.2f}".format(macro_r)) print("Macro-averaged F1: {:.2f}".format(macro_f))
def print_stats(predictions, y_test): eval = Evaluator() confusion = eval.confusion_matrix(predictions, y_test) accuracy = eval.accuracy(confusion) precision = eval.precision(confusion) recall = eval.recall(confusion) f1 = eval.f1_score(confusion) print("confusion", confusion) print("accuracy", accuracy) print("precision", precision) print("recall", recall) print("f1", f1) return
def train(self): # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") valid_evaluator = Evaluator() self.model = self.model.to(self.device) for epochIndex in range(self.epochNum): loss_train = self.one_pass_train(epochIndex) loss_overall_train, loss_emo_train = LossLogger( self.fo, 'train', loss_train, '--', epochIndex) loss_valid = valid_evaluator.evaluation(self.X_valid, self.Y_valid, self.model) loss_overall_valid, loss_emo_valid = LossLogger( self.fo, 'valid', loss_valid, '--', epochIndex) self.trainValidWritter(loss_train, loss_valid, loss_overall_train, loss_emo_train, loss_overall_valid, loss_emo_valid, epochIndex) if (self.betterSaver(loss_emo_valid) == True): break self.writter.close() self.fo.close()
def _train_eval(diff_vec_class, desc_vec_class, alp): metrics = "accuracy,recall,distance,nlg" suf = "{}_{}_{}".format(_get_class_name(diff_vec_class), _get_class_name(desc_vec_class), alp) out_file = os.path.join(os.path.dirname(valid_file), "nnupdater_result_{}.json".format(suf)) slice_test_main(train_file, valid_file, out_file, alpha=alp, slice_count=slice_count, diff_vec_class=diff_vec_class, desc_vec_class=desc_vec_class) evaluator = Evaluator(args={ "--metrics": metrics, "TEST_SET": valid_file, "RESULT_FILE": out_file }) r, lemma_r = evaluator.evaluate() return suf, r, lemma_r
def prune_tree_reduced_error(self, tree, x_val, y_val): """ Function to accept prunes which increase the tree's accuracy, otherwise ignore Args: tree (dict) - tree to be pruned x_val (2D array) - 2D array of attributes of validation set where each row is a differnt sample and each column is a differnt attribute y_val (1D array) - 1D array of correct labels for x_val validation data Output: tree (dict or str) - tree pruned such that any additional pruning would lower predictive accuracy on validation set. """ classifier = DecisionTreeClassifier() classifier.is_trained = True predictions = classifier.predict(x_val) eval = Evaluator() confusion = eval.confusion_matrix(predictions, y_val) root_accuracy = eval.accuracy(confusion) print("Results on Validation set") print("Original Accuracy: ", root_accuracy) is_pruned = True while (is_pruned and isinstance(tree, dict)): #make copy of tree then attempt to prune copy tree_copy = copy.deepcopy(tree) (is_pruned, tree_copy, tree) = self.prune(tree_copy, tree) if is_pruned: #compare accuracy of pruned tree to original new_predictions = classifier.predict(x_val, tree_copy) new_confusion = eval.confusion_matrix(new_predictions, y_val) new_accuracy = eval.accuracy(new_confusion) if new_accuracy >= root_accuracy: #if greater or equal accuracy make tree = copy root_accuracy = new_accuracy tree = copy.deepcopy(tree_copy) print("New Accuracy: ", root_accuracy) return tree
def test_DecisionTreeClassifier(dataset_filename: str = "toy.txt", should_load_file=False): # train extless_filename = dataset_filename[:-4] start = time.time() saved_tree_file = None if should_load_file: saved_tree_file = "tree_" + extless_filename + ".obj" cl = DecisionTreeClassifier(saved_tree_file=saved_tree_file) dataset = data_read("data/" + dataset_filename) unique_lbls = np.unique([e.label for e in dataset.entries]) x, y = dataset.shim_to_arrays() cl.train(x, y) cl.tree.save_tree("tree_" + extless_filename + ".obj") visualize_tree(cl.tree, save_filename=f"visualize_tree_{extless_filename}.txt", max_depth=8) duration = time.time() - start print("duration: ", duration) # predict test_dataset = data_read("data/test.txt") x_test, y_test = test_dataset.shim_to_arrays() preds = cl.predict(x_test) # preds = [random.choice('ACEGOQ') # for _ in range(len(y_test))] # testing random # evaluate ev = Evaluator() matrix = ev.confusion_matrix(preds, y_test, unique_lbls) print("real accuracy: ", accuracy_score(y_test, preds)) print("\nour calc accuracy: ", str.format('{0:.15f}', ev.accuracy(matrix))) print("\n precision:", precision_score(y_test, preds, average="macro")) print("\n our precision: ", ev.precision(matrix)) print("\nreal recall: ", recall_score(y_test, preds, average="macro")) print("\n our recall: ", ev.recall(matrix)) print("\n f1_score", f1_score(y_test, preds, average="macro")) print("\n f1_score: ", ev.f1_score(matrix)) print(matrix)
import sys import torch import torch.autograd as autograd from torch.autograd import Variable import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from torch.nn import init torch.manual_seed(1) SOS = 0 EOS = 1 e = Evaluator() torch.cuda.device(0) # let's go # precompute input tokens tokens = [torch.LongTensor([[x]]).cuda() for x in range(22)] token_lookup = { '@': 1, 'A': 2, 'B': 3, 'C': 4, 'D': 5, 'E': 6, 'F': 7, 'G': 8,
pred = pd.DataFrame(te_files_wav, columns=["fname"]) pred['label'] = pred_labels # # # =================================================================================================== EVAL # # =================================================================================================== EVAL # # =================================================================================================== EVAL print( '\nEvaluate ACC and print score============================================================================' ) # read ground truth gt_test = pd.read_csv(params_files.get('gt_test')) # init Evaluator object evaluator = Evaluator(gt_test, pred, list_labels, params_ctrl, params_files) print( '\n=============================ACCURACY===============================================================' ) print( '=============================ACCURACY===============================================================\n' ) evaluator.evaluate_acc() evaluator.evaluate_acc_classwise() evaluator.print_summary_eval() end = time.time() print( '\n=============================Job finalized==========================================================\n' )
def calc_stats(self, test_path, path_to_data, plt_title, prune, pruneAggressively): #load dataset, atttribs, labels d_subset = ClassifierDataset() d_subset.initFromFile(path_to_data) attribs = d_subset.attrib labels = d_subset.labels ds_test = ClassifierDataset() ds_test.initFromFile(test_path) test_attribs = ds_test.attrib test_labels = ds_test.labels #train and predict print("TRAINING") tree = DecisionTreeClassifier() tree.train(attribs, labels) print("FINISHED TRAINING") if prune == True: print("PRUNING") validationDataset = ClassifierDataset() validationDataset.initFromFile(val_path) Prune(tree, validationDataset.attrib, validationDataset.labels, pruneAggressively) print("FINISHED PRUNING") predictions = tree.predict(test_attribs) evaluator = Evaluator() c_matrix = evaluator.confusion_matrix(predictions, test_labels) print(c_matrix) a = ["A", "C", "E", "G", "O", "Q"] b = path_to_data[7:-4] if prune: if pruneAggressively: b = b + "_aggressively_pruned" else: b += "_pruned" else: b += "_not_pruned" plot_confusion_matrix(c_matrix, a, plt_title) print(" ") print("Accuracy: " + str(evaluator.accuracy(c_matrix))) print(" ") precision, macro_p = evaluator.precision(c_matrix) recall, macro_r = evaluator.recall(c_matrix) f1, macro_f1 = evaluator.f1_score(c_matrix) p = np.append(precision, macro_p) r = np.append(recall, macro_r) f1 = np.append(f1, macro_f1) performance_matrix = np.vstack((p, np.vstack((r, f1)))) print(performance_matrix) plot_other_stats(performance_matrix, plt_title) ''' print("Precision: " + str(precision)) print("Recall: " + str(recall)) print("F1 Score: " + str(f1))''' print(" ") print("Macro avg recall:" + str(macro_r)) print("Macro avg precision:" + str(macro_p)) print("Macro avg f1:" + str(macro_f1)) print(" ")
def __init__(self): self.evaluator = Evaluator(fold=KFOLD_NUM) return
# **************************************************************************** # # # # ::: :::::::: # # test.py :+: :+: :+: # # +:+ +:+ +:+ # # By: tbrizon <*****@*****.**> +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2019/11/05 16:53:07 by tbrizon #+# #+# # # Updated: 2019/11/05 17:52:45 by tbrizon ### ########.fr # # # # **************************************************************************** # from eval import Evaluator if __name__ == "__main__": words = ["Le", "Lorem", "Ipsum", "est", "simple"] coef = [1.0, 2.0, 1.0, 4.0, 0.5] a = Evaluator(words, coef) Evaluator.zip_evaluate(coef, words) Evaluator.enumerate_evaluate(coef, words)
def test(args): """Run model testing.""" model_args = args.model_args data_args = args.data_args logger_args = args.logger_args # import pdb; pdb.set_trace() # Get logger. logger = Logger(logger_args.log_path, logger_args.save_dir, logger_args.results_dir) # Get image paths corresponding to predictions for logging paths = None if model_args.config_path is not None: # Instantiate the EnsemblePredictor class for obtaining # model predictions. predictor = EnsemblePredictor(config_path=model_args.config_path, model_args=model_args, data_args=data_args, gpu_ids=args.gpu_ids, device=args.device, logger=logger) # Obtain ensemble predictions. # Caches both individual and ensemble predictions. # We always turn off caching to ensure that we write the Path column. predictions, groundtruth, paths = predictor.predict(cache=False, return_paths=True, all_gt_tasks=True) else: # Load the model at ckpt_path. ckpt_path = model_args.ckpt_path ckpt_save_dir = Path(ckpt_path).parent model_uncertainty = model_args.model_uncertainty # Get model args from checkpoint and add them to # command-line specified model args. model_args, transform_args\ = ModelSaver.get_args(cl_model_args=model_args, dataset=data_args.dataset, ckpt_save_dir=ckpt_save_dir, model_uncertainty=model_uncertainty) # TODO JBY: in test moco should never be true. model_args.moco = args.model_args.moco model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path, gpu_ids=args.gpu_ids, model_args=model_args, is_training=False) # Instantiate the Predictor class for obtaining model predictions. predictor = Predictor(model=model, device=args.device) # Get phase loader object. return_info_dict = True loader = get_loader(phase=data_args.phase, data_args=data_args, transform_args=transform_args, is_training=False, return_info_dict=return_info_dict, logger=logger) # Obtain model predictions. if return_info_dict: predictions, groundtruth, paths = predictor.predict(loader) else: predictions, groundtruth = predictor.predict(loader) # print(predictions[CHEXPERT_COMPETITION_TASKS]) if model_args.calibrate: #open the json file which has the saved parameters import json with open(CALIBRATION_FILE) as f: data = json.load(f) i = 0 #print(predictions) import math def sigmoid(x): return 1 / (1 + math.exp(-x)) for column in predictions: predictions[column] = predictions[column].apply \ (lambda x: sigmoid(x * data[i][0][0][0] \ + data[i][1][0])) i += 1 # print(predictions[CHEXPERT_COMPETITION_TASKS]) #run forward on all the predictions in each row of predictions # Log predictions and groundtruth to file in CSV format. logger.log_predictions_groundtruth(predictions, groundtruth, paths) if not args.inference_only: # Instantiate the evaluator class for evaluating models. evaluator = Evaluator(logger, operating_points_path=CHEXPERT_RAD_PATH) # Get model metrics and curves on the phase dataset. metrics, curves = evaluator.evaluate_tasks(groundtruth, predictions) # Log metrics to stdout and file. logger.log_stdout(f"Writing metrics to {logger.metrics_path}.") logger.log_metrics(metrics, save_csv=True) # TODO: make this work with ensemble # TODO: investigate if the eval_loader can just be the normal loader here if logger_args.save_cams: cams_dir = logger_args.save_dir / 'cams' print(f'Save cams to {cams_dir}') save_grad_cams(args, loader, model, cams_dir, only_competition=logger_args.only_competition_cams, only_top_task=False) logger.log("=== Testing Complete ===")
#QUESTION 2 print("Question 2") print("Training the tree with two different methods") print("Training the decision tree...") classifier = classifier.train(x, y) print("Loading the test set...") filename = "data/test.txt" x_test, y_test = classifier.load_data(filename) print("\nPredicting on test.txt data with 4 different trees") #Load the evaulator class eval = Evaluator() prune = Pruning() print("\nTree 2 unpruned") tree_3 = np.load('simple_tree.npy', allow_pickle=True).item() predictions = classifier.predict(x_test) confusion = eval.confusion_matrix(predictions, y_test) accuracy_3 = eval.accuracy(confusion) print("number of leaves:", prune.count_leaves(tree_3)) print("Tree 2 unpruned Accuracy: " + str(np.round(accuracy_3 * 100, 2))) print("\nTree 2 pruned") tree_4 = np.load('simple_tree_pruned.npy', allow_pickle=True).item() predictions = classifier.predict(x_test, tree_4) confusion = eval.confusion_matrix(predictions, y_test) accuracy_4 = eval.accuracy(confusion)
if __name__ == "__main__": print("Loading the datasets...") trainingData = dataReader.parseFile("data/train_full.txt") validationData = dataReader.parseFile("data/validation.txt") testData = dataReader.parseFile("data/test.txt") print("Training the decision tree...") classifier = DecisionTreeClassifier() classifier = classifier.train(trainingData[0], trainingData[1]) predictions = classifier.predict(testData[0]) print("Pre prunning predictions: {}".format(predictions)) print("Evaluating test predictions...") evaluator = Evaluator() confusion = evaluator.confusion_matrix(predictions, testData[1]) printMetric(confusion) print("Pruning the decision tree...") classifier.prune(validationData) predictions = classifier.predict(testData[0]) print("Post prunning predictions: {}".format(predictions)) print("Evaluating test predictions...") evaluator = Evaluator() confusion = evaluator.confusion_matrix(predictions, testData[1]) printMetric(confusion) classifier.plot_tree()
def train(args): """Run model training.""" print("Start Training ...") # Get nested namespaces. model_args = args.model_args logger_args = args.logger_args optim_args = args.optim_args data_args = args.data_args transform_args = args.transform_args # Get logger. print('Getting logger... log to path: {}'.format(logger_args.log_path)) logger = Logger(logger_args.log_path, logger_args.save_dir) # For conaug, point to the MOCO pretrained weights. if model_args.ckpt_path and model_args.ckpt_path != 'None': print("pretrained checkpoint specified : {}".format( model_args.ckpt_path)) # CL-specified args are used to load the model, rather than the # ones saved to args.json. model_args.pretrained = False ckpt_path = model_args.ckpt_path model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path, gpu_ids=args.gpu_ids, model_args=model_args, is_training=True) if not model_args.moco: optim_args.start_epoch = ckpt_info['epoch'] + 1 else: optim_args.start_epoch = 1 else: print( 'Starting without pretrained training checkpoint, random initialization.' ) # If no ckpt_path is provided, instantiate a new randomly # initialized model. model_fn = models.__dict__[model_args.model] if data_args.custom_tasks is not None: tasks = NamedTasks[data_args.custom_tasks] else: tasks = model_args.__dict__[TASKS] # TASKS = "tasks" print("Tasks: {}".format(tasks)) model = model_fn(tasks, model_args) model = nn.DataParallel(model, args.gpu_ids) # Put model on gpu or cpu and put into training mode. model = model.to(args.device) model.train() print("========= MODEL ==========") print(model) # Get train and valid loader objects. train_loader = get_loader(phase="train", data_args=data_args, transform_args=transform_args, is_training=True, return_info_dict=False, logger=logger) valid_loader = get_loader(phase="valid", data_args=data_args, transform_args=transform_args, is_training=False, return_info_dict=False, logger=logger) # Instantiate the predictor class for obtaining model predictions. predictor = Predictor(model, args.device) # Instantiate the evaluator class for evaluating models. evaluator = Evaluator(logger) # Get the set of tasks which will be used for saving models # and annealing learning rate. eval_tasks = EVAL_METRIC2TASKS[optim_args.metric_name] # Instantiate the saver class for saving model checkpoints. saver = ModelSaver(save_dir=logger_args.save_dir, iters_per_save=logger_args.iters_per_save, max_ckpts=logger_args.max_ckpts, metric_name=optim_args.metric_name, maximize_metric=optim_args.maximize_metric, keep_topk=logger_args.keep_topk) # TODO: JBY: handle threshold for fine tuning if model_args.fine_tuning == 'full': # Fine tune all layers. pass else: # Freeze other layers. models.PretrainedModel.set_require_grad_for_fine_tuning( model, model_args.fine_tuning.split(',')) # Instantiate the optimizer class for guiding model training. optimizer = Optimizer(parameters=model.parameters(), optim_args=optim_args, batch_size=data_args.batch_size, iters_per_print=logger_args.iters_per_print, iters_per_visual=logger_args.iters_per_visual, iters_per_eval=logger_args.iters_per_eval, dataset_len=len(train_loader.dataset), logger=logger) if model_args.ckpt_path and not model_args.moco: # Load the same optimizer as used in the original training. optimizer.load_optimizer(ckpt_path=model_args.ckpt_path, gpu_ids=args.gpu_ids) model_uncertainty = model_args.model_uncertainty loss_fn = evaluator.get_loss_fn( loss_fn_name=optim_args.loss_fn, model_uncertainty=model_args.model_uncertainty, mask_uncertain=True, device=args.device) # Run training while not optimizer.is_finished_training(): optimizer.start_epoch() # TODO: JBY, HACK WARNING # What is the hack? metrics = None for inputs, targets in train_loader: optimizer.start_iter() if optimizer.global_step and optimizer.global_step % optimizer.iters_per_eval == 0 or len( train_loader.dataset ) - optimizer.iter < optimizer.batch_size: # Only evaluate every iters_per_eval examples. predictions, groundtruth = predictor.predict(valid_loader) # print("predictions: {}".format(predictions)) metrics, curves = evaluator.evaluate_tasks( groundtruth, predictions) # Log metrics to stdout. logger.log_metrics(metrics) # Add logger for all the metrics for valid_loader logger.log_scalars(metrics, optimizer.global_step) # Get the metric used to save model checkpoints. average_metric = evaluator.evaluate_average_metric( metrics, eval_tasks, optim_args.metric_name) if optimizer.global_step % logger_args.iters_per_save == 0: # Only save every iters_per_save examples directly # after evaluation. print("Save global step: {}".format(optimizer.global_step)) saver.save(iteration=optimizer.global_step, epoch=optimizer.epoch, model=model, optimizer=optimizer, device=args.device, metric_val=average_metric) # Step learning rate scheduler. optimizer.step_scheduler(average_metric) with torch.set_grad_enabled(True): logits, embedding = model(inputs.to(args.device)) loss = loss_fn(logits, targets.to(args.device)) optimizer.log_iter(inputs, logits, targets, loss) optimizer.zero_grad() loss.backward() optimizer.step() optimizer.end_iter() optimizer.end_epoch(metrics) logger.log('=== Training Complete ===')
def main(): parser = argparse.ArgumentParser(description='CNNF-TTT testing') parser.add_argument('--dataset', choices=['cifar10', 'fashion'], default='cifar10', help='the dataset for training the model') parser.add_argument( '--test', choices=['average', 'last'], default='average', help='output averaged logits or logits from the last iteration') parser.add_argument('--csv-dir', default='results.csv', help='Directory for Saving the Evaluation results') parser.add_argument('--model-dir', default='models', help='Directory for Saved Models') args = parser.parse_args() use_cuda = torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") clean_dir = 'data/' # load in corrupted data if args.dataset == 'cifar10': dataloader = torch.utils.data.DataLoader(datasets.CIFAR10( clean_dir, train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ])), batch_size=64, shuffle=True, num_workers=4, pin_memory=True) eps = 0.063 eps_iter = 0.02 nb_iter = 7 elif args.dataset == 'fashion': dataloader = torch.utils.data.DataLoader(datasets.FashionMNIST( clean_dir, train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, ), (0.5, )), AddGaussianNoise(0., 0.5), ])), batch_size=100, shuffle=True) eps = 0.025 eps_iter = 0.071 nb_iter = 7 log_acc_path = args.csv_dir evalmethod = args.test model_dir = args.model_dir with open(log_acc_path, 'a') as f: f.write(',clean,pgd_first,pgd_last,spsa_first,spsa_last,transfer,') f.write('\n') # Model to evaluate if args.dataset == 'cifar10': model_name = 'CNNF_2_cifar.pt' model = WideResNet(40, 10, 2, 0.0, ind=5, cycles=2, res_param=0.1).to(device) elif args.dataset == 'fashion': model_name = 'CNNF_1_fmnist.pt' model = CNNF(10, ind=2, cycles=1, res_param=0.1).to(device) model_path = os.path.join(model_dir, model_name) model.load_state_dict(torch.load(model_path)) eval = Evaluator(device, model) corrupted_acc = eval.corrupted_accuracy(dataloader) optimizer = torch.optim.SGD(model.parameters(), 0.05, momentum=0.9, weight_decay=5e-4) spsa_acc_ete = eval.ttt_accuracy(dataloader, optimizer, per_image=False, batch_size=10) with open(log_acc_path, 'a') as f: f.write('%s,' % model_name) #f.write('%0.2f,' % (100. * clean_acc)) #f.write('%0.2f,' % (100. * pgd_acc_first)) # f.write('%0.2f,' % (100. * pgd_acc_ete)) #f.write('%0.2f,' % (100. * spsa_acc_first)) # f.write('%0.2f,' % (100. * spsa_acc_ete)) # f.write('%0.2f,' % (100. * transfer_acc)) f.write('\n')
def main(): args = parse_args() transformers_logger = logging.getLogger("transformers") transformers_logger.setLevel(logging.ERROR) if args.predict_file is None and args.do_eval: raise ValueError( "Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file " "or remove the --do_eval argument.") if args.output_dir and os.path.exists(args.output_dir) and \ os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir: raise ValueError( "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome." .format(args.output_dir)) if args.overwrite_output_dir and os.path.isdir(args.output_dir): shutil.rmtree(args.output_dir) os.mkdir(args.output_dir) # Setup CUDA, GPU & distributed training if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = torch.cuda.device_count() else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) torch.distributed.init_process_group(backend='nccl') args.n_gpu = 1 args.device = device # Setup logging logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN) with open(os.path.join(args.output_dir, 'args.txt'), 'w') as f: f.write(str(args)) for key, val in vars(args).items(): logger.info(f"{key} - {val}") try: write_meta_data(args.output_dir, args) except git.exc.InvalidGitRepositoryError: logger.info("didn't save metadata - No git repo!") logger.info( "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, amp training: %s", args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.amp) # Set seed set_seed(args) # Load pretrained model and tokenizer if args.local_rank not in [-1, 0]: # Barrier to make sure only the first process in distributed training download model & vocab torch.distributed.barrier() if args.config_name: config = AutoConfig.from_pretrained(args.config_name, cache_dir=args.cache_dir) elif args.model_name_or_path: config = AutoConfig.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir) else: config = CONFIG_MAPPING[args.model_type]() logger.warning( "You are instantiating a new config instance from scratch.") #config.hidden_dropout_prob = args.encoder_dropout_prob if args.tokenizer_name: tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir) elif args.model_name_or_path: tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir) else: raise ValueError( "You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it," "and load it from here, using --tokenizer_name") config_class = LongformerConfig base_model_prefix = "longformer" S2E.config_class = config_class S2E.base_model_prefix = base_model_prefix model = S2E.from_pretrained(args.model_name_or_path, config=config, cache_dir=args.cache_dir, args=args) model.to(args.device) if args.local_rank == 0: # End of barrier to make sure only the first process in distributed training download model & vocab torch.distributed.barrier() logger.info("Training/evaluation parameters %s", args) evaluator = Evaluator(args, tokenizer) # Training if args.do_train: train_dataset = get_dataset(args, tokenizer, evaluate=False) global_step, tr_loss = train(args, train_dataset, model, tokenizer, evaluator) logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) # Saving best-practices: if you use save_pretrained for the model and tokenizer, # you can reload them using from_pretrained() if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # Create output directory if needed if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: os.makedirs(args.output_dir) logger.info("Saving model checkpoint to %s", args.output_dir) # Save a trained model, configuration and tokenizer using `save_pretrained()`. # They can then be reloaded using `from_pretrained()` model_to_save = model.module if hasattr( model, 'module') else model # Take care of distributed/parallel training model_to_save.save_pretrained(args.output_dir) tokenizer.save_pretrained(args.output_dir) # Good practice: save your training arguments together with the trained model torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) # Evaluation results = {} if args.do_eval and args.local_rank in [-1, 0]: result = evaluator.evaluate(model, prefix="final_evaluation", official=True) results.update(result) return results return results