def eval_search_engine(res_fname, format, th=50): ir = read_res_file(res_fname, format) # evaluate IR rec = metrics.recall_of_1(ir, th) acc = metrics.accuracy(ir, th) acc1 = metrics.accuracy1(ir, th) acc2 = metrics.accuracy2(ir, th) mrr = metrics.mrr(ir, th) # MAP map_ir = metrics.map(ir) print "%10s" %"IR" print "MRR: %5.2f" % mrr print "MAP: %5.2f" % map_ir for i, (r, a, a1, a2) in enumerate(zip(rec, acc, acc1, acc2), 1): print "REC-1@%02d: %6.2f ACC@%02d: %6.2f AC1@%02d: %6.2f AC2@%02d: %4.0f" %(i, r, i, a, i, a1, i, a2) print print "REC-1 - percentage of questions with at least 1 correct answer in the top @X positions (useful for tasks were questions have at most one correct answer)" print "ACC - accuracy, i.e. number of correct answers retrieved at rank @X normalized by the rank and the total number of questions" print "AC1 - the number of correct answers at @X normalized by the number of maximum possible answers (perfect re-ranker)" print "AC2 - the absolute number of correct answers at @X"
def eval_reranker(res_fname="svm.test.res", pred_fname="svm.train.pred", format="trec", th=50, verbose=False, reranking_th=0.0, ignore_noanswer=False): ir, svm = read_res_pred_files(res_fname, pred_fname, format, verbose, reranking_th=reranking_th, ignore_noanswer=ignore_noanswer) # evaluate IR prec_se = metrics.recall_of_1(ir, th) acc_se = metrics.accuracy(ir, th) acc_se1 = metrics.accuracy1(ir, th) acc_se2 = metrics.accuracy2(ir, th) # evaluate SVM prec_svm = metrics.recall_of_1(svm, th) acc_svm = metrics.accuracy(svm, th) acc_svm1 = metrics.accuracy1(svm, th) acc_svm2 = metrics.accuracy2(svm, th) mrr_se = metrics.mrr(ir, th) mrr_svm = metrics.mrr(svm, th) map_se = metrics.map(ir) map_svm = metrics.map(svm) avg_acc1_svm = metrics.avg_acc1(svm, th) avg_acc1_ir = metrics.avg_acc1(ir, th) print "%13s %5s" %("IR", "SVM") print "MRR: %5.2f %5.2f" %(mrr_se, mrr_svm) print "MAP: %5.4f %5.4f" %(map_se, map_svm) print "AvgRec: %5.2f %5.2f" %(avg_acc1_ir, avg_acc1_svm) print "%16s %6s %14s %6s %14s %6s %12s %4s" % ("IR", "SVM", "IR", "SVM", "IR", "SVM", "IR", "SVM") for i, (p_se, p_svm, a_se, a_svm, a_se1, a_svm1, a_se2, a_svm2) in enumerate(zip(prec_se, prec_svm, acc_se, acc_svm, acc_se1, acc_svm1, acc_se2, acc_svm2), 1): print "REC-1@%02d: %6.2f %6.2f ACC@%02d: %6.2f %6.2f AC1@%02d: %6.2f %6.2f AC2@%02d: %4.0f %4.0f" %(i, p_se, p_svm, i, a_se, a_svm, i, a_se1, a_svm1, i, a_se2, a_svm2) print print "REC-1 - percentage of questions with at least 1 correct answer in the top @X positions (useful for tasks were questions have at most one correct answer)" print "ACC - accuracy, i.e. number of correct answers retrieved at rank @X normalized by the rank and the total number of questions" print "AC1 - the number of correct answers at @X normalized by the number of maximum possible answers (perfect re-ranker)" print "AC2 - the absolute number of correct answers at @X"
def train_teacher(dataset, nb_teachers, teacher_id): """ This function trains a teacher (teacher id) among an ensemble of nb_teachers models for the dataset specified. :param dataset: string corresponding to dataset (svhn, cifar10) :param nb_teachers: total number of teachers in the ensemble :param teacher_id: id of the teacher being trained :return: True if everything went well """ # If working directories do not exist, create them assert input.create_dir_if_needed(FLAGS.data_dir) assert input.create_dir_if_needed(FLAGS.train_dir) # Load the dataset if dataset == 'svhn': train_data,train_labels,test_data,test_labels = input.ld_svhn(extended=True) elif dataset == 'cifar10': train_data, train_labels, test_data, test_labels = input.ld_cifar10() elif dataset == 'mnist': train_data, train_labels, test_data, test_labels = input.ld_mnist() else: print("Check value of dataset flag") return False # Retrieve subset of data for this teacher data, labels = input.partition_dataset(train_data, train_labels, nb_teachers, teacher_id) print("Length of training data: " + str(len(labels))) # Define teacher checkpoint filename and full path if FLAGS.deeper: filename = str(nb_teachers) + '_teachers_' + str(teacher_id) + '_deep.ckpt' else: filename = str(nb_teachers) + '_teachers_' + str(teacher_id) + '.ckpt' ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + filename # Perform teacher training assert deep_cnn.train(data, labels, ckpt_path) # Append final step value to checkpoint for evaluation ckpt_path_final = ckpt_path + '-' + str(FLAGS.max_steps - 1) # Retrieve teacher probability estimates on the test data teacher_preds = deep_cnn.softmax_preds(test_data, ckpt_path_final) # Compute teacher accuracy precision = metrics.accuracy(teacher_preds, test_labels) print('Precision of teacher after training: ' + str(precision)) return True
def compute_all_metrics(execution_id, path_input, path_output, formula, append): from metrics import accuracy, precision, recall, f1, specificity """ Computes all metrics and persistes in a csv Args: execution_id (int): identifier of the execution path_input (string): path of the file that contains the classifications path_out (string): path of the file that will persist the metrics formula (string): mean_max | mean_mean append (boolean): true | false """ # loading results with open(path_input) as data_file: data = json.load(data_file) # computing metrics tp = tn = fp = fn = 0 for i in range(0, len(data)): if (data[i]['values'][formula]['positive'] >= data[i]['values'][formula]['negative']): if data[i]['values']['label'] == 'positive': tp += 1 else: fp += 1 elif (data[i]['values'][formula]['positive'] < data[i]['values'][formula]['negative']): if (data[i]['values']['label'] == 'negative'): tn += 1 else: fn += 1 else: raise Exception("Positive similarity equals to negative similarity to news " + data[i]['id']) accuracy = accuracy(tp, tn, fp, fn) recall = recall(tp, fn) precision = precision(tp, fp) f1 = f1(precision, recall); specificity = specificity(tn, fp); # persiting the results with open(path_output, 'a' if append else 'w') as csvfile: spamwriter = csv.writer(csvfile, delimiter=',') if (not append): spamwriter.writerow( ['execution_id', 'tp', 'tn', 'fp', 'fn', 'accuracy', 'precision', 'recall', 'f1', 'specificity']) spamwriter.writerow([execution_id, tp, tn, fp, fn, accuracy, precision, recall, f1, specificity])
def train_student(dataset, nb_teachers): """ This function trains a student using predictions made by an ensemble of teachers. The student and teacher models are trained using the same neural network architecture. :param dataset: string corresponding to mnist, cifar10, or svhn :param nb_teachers: number of teachers (in the ensemble) to learn from :return: True if student training went well """ assert input.create_dir_if_needed(FLAGS.train_dir) # Call helper function to prepare student data using teacher predictions stdnt_dataset = prepare_student_data(dataset, nb_teachers, save=True) # Unpack the student dataset stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels = stdnt_dataset # Prepare checkpoint filename and path if FLAGS.deeper: ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + str(nb_teachers) + '_student_deeper.ckpt' #NOLINT(long-line) else: ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + str(nb_teachers) + '_student.ckpt' # NOLINT(long-line) # Start student training assert deep_cnn.train(stdnt_data, stdnt_labels, ckpt_path) # Compute final checkpoint name for student (with max number of steps) ckpt_path_final = ckpt_path + '-' + str(FLAGS.max_steps - 1) # Compute student label predictions on remaining chunk of test set student_preds = deep_cnn.softmax_preds(stdnt_test_data, ckpt_path_final) # Compute teacher accuracy precision = metrics.accuracy(student_preds, stdnt_test_labels) print('Precision of student after training: ' + str(precision)) return True
def prepare_student_data(dataset, nb_teachers, save=False): """ Takes a dataset name and the size of the teacher ensemble and prepares training data for the student model, according to parameters indicated in flags above. :param dataset: string corresponding to mnist, cifar10, or svhn :param nb_teachers: number of teachers (in the ensemble) to learn from :param save: if set to True, will dump student training labels predicted by the ensemble of teachers (with Laplacian noise) as npy files. It also dumps the clean votes for each class (without noise) and the labels assigned by teachers :return: pairs of (data, labels) to be used for student training and testing """ assert input.create_dir_if_needed(FLAGS.train_dir) # Load the dataset if dataset == 'svhn': test_data, test_labels = input.ld_svhn(test_only=True) elif dataset == 'cifar10': test_data, test_labels = input.ld_cifar10(test_only=True) elif dataset == 'mnist': test_data, test_labels = input.ld_mnist(test_only=True) else: print("Check value of dataset flag") return False # Make sure there is data leftover to be used as a test set assert FLAGS.stdnt_share < len(test_data) # Prepare [unlabeled] student training data (subset of test set) stdnt_data = test_data[:FLAGS.stdnt_share] # Compute teacher predictions for student training data teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data) # Aggregate teacher predictions to get student training labels if not save: stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale) else: # Request clean votes and clean labels as well stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale, return_clean_votes=True) #NOLINT(long-line) # Prepare filepath for numpy dump of clean votes filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_clean_votes_lap_' + str(FLAGS.lap_scale) + '.npy' # NOLINT(long-line) # Prepare filepath for numpy dump of clean labels filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_teachers_labels_lap_' + str(FLAGS.lap_scale) + '.npy' # NOLINT(long-line) # Dump clean_votes array with gfile.Open(filepath, mode='w') as file_obj: np.save(file_obj, clean_votes) # Dump labels_for_dump array with gfile.Open(filepath_labels, mode='w') as file_obj: np.save(file_obj, labels_for_dump) # Print accuracy of aggregated labels ac_ag_labels = metrics.accuracy(stdnt_labels, test_labels[:FLAGS.stdnt_share]) print("Accuracy of the aggregated labels: " + str(ac_ag_labels)) # Store unused part of test set for use as a test set after student training stdnt_test_data = test_data[FLAGS.stdnt_share:] stdnt_test_labels = test_labels[FLAGS.stdnt_share:] if save: # Prepare filepath for numpy dump of labels produced by noisy aggregation filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_labels_lap_' + str(FLAGS.lap_scale) + '.npy' #NOLINT(long-line) # Dump student noisy labels array with gfile.Open(filepath, mode='w') as file_obj: np.save(file_obj, stdnt_labels) return stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels
x_values = [] x_values_fscore = [] # Para cada porcentaje de confianza for i in xrange(100): # Obtengo las predicciones con una confianza mayor a cierto umbral porcentaje = float(i)/100 aux = result[result['trust'] > porcentaje] # matrix = metrics.confusion_matrix(aux) matrix = metrics.hard_matrix(aux) # Si la precision es menor que cero, es porque no habian datos que superaran tal nivel de confianza precision = metrics.accuracy(matrix, clase) if precision >= 0: valores_accuracy.append(precision) valores_recall.append(metrics.recall(matrix, clase)) x_values.append(porcentaje) # Si el f_score es menor que cero, es porque no habian datos que superaran tal nivel de confianza f_score = metrics.f_score(matrix, clase) if f_score >= 0: valores_fscore.append(f_score) x_values_fscore.append(porcentaje) #graf(clase, x_values, valores_accuracy, 'Accuracy') graf(clase, x_values, valores_recall, 'Recall') #graf(clase, x_values_fscore, valores_fscore, 'F-Score') print 'a'
# длина периодов сезонных составляющих p1 = 24 p2 = 168 # тренировочный датасет train_set, данные с 14.01.2019 по 25.11.2019 train_start_date = datetime.datetime(2019, 1, 14, 0, 0, 0) train_end_date = datetime.datetime(2019, 11, 25, 23, 0, 0) train_set = fact[train_start_date:train_end_date] # тестовый датасет test_set, данные за прогнозный день 27.11.2019 pred_date = datetime.date(2019, 11, 27) pred_end_date = train_end_date + datetime.timedelta(days=2) pred_start_date = pred_end_date - datetime.timedelta(hours=23) test_set = fact[pred_start_date:pred_end_date] # план предприятия plan_set для сверки, данные за прогнозный день 27.11.2019 plan_set = plan[pred_start_date:pred_end_date] # выбор метода method = double_seasonal_multiplicativeHoltWinters # найденные постоянные сглаживания best_params = [[0.334097458, 0.000291639411, 0.278284232, 0.123083456]] # прогнозирование pred = method(best_params[0], train_set, p1, p2, 48)[24:] # оценка прогноза по метрикам nnf_val = nnfmetrics(pred, test_set, plan_set) mse_val = mse(test_set, pred) mape_val = mape(test_set, pred) acc_val = accuracy(test_set, pred) print('Оценка по NNFMETRICS = ', nnf_val) print('Оценка по MSE = ', mse_val) print('Оценка по MAPE = ', mape_val) print('Точность прогноза = ', acc_val) # отрисовка графика plot_results(pred_date.strftime('%d-%m-%Y'), test_set, pred, plan_set)
def metrics_for_eval(model, data_loader, device, loss_fn, topk=2, get_balanced_acc=False): """ This function returns accuracy, topk accuracy and loss for the evaluation partition :param model (nn.Model): the model you'd like to evaluate :param data_loader (DataLoader): the DataLoader containing the data partition :param checkpoint_path(string, optional): string with a checkpoint to load the model. If None, none checkpoint is loaded. Default is None. :param loss_fn (nn.Loss): the loss function used in the training :param get_balanced_acc (bool, optional); if you'd like to compute the balanced accuracy for the eval partition set it as True. The defaulf is False because it may impact in the training phase. :return: a instance of the classe metrics """ # setting the model to evaluation mode model.eval() print("\nEvaluating...") # Setting tqdm to show some information on the screen with tqdm(total=len(data_loader), ascii=True, ncols=100) as t: # Setting require_grad=False in order to dimiss the gradient computation in the graph with torch.no_grad(): loss_avg = AVGMetrics() acc_avg = AVGMetrics() topk_avg = AVGMetrics() if get_balanced_acc: # Setting the metrics object metrics = Metrics(['balanced_accuracy']) else: metrics = None for data in data_loader: # In data we may have imgs, labels and extra info. If extra info is [], it means we don't have it # for the this training case. Imgs came in data[0], labels in data[1] and extra info in data[2] try: images_batch, labels_batch, extra_info_batch, _ = data except ValueError: images_batch, labels_batch = data extra_info_batch = [] if (len(extra_info_batch)): # Moving the data to the deviced that we set above images_batch, labels_batch = images_batch.to( device), labels_batch.to(device) extra_info_batch = extra_info_batch.to(device) # Doing the forward pass using the extra info pred_batch = model(images_batch, extra_info_batch) else: # Moving the data to the deviced that we set above images_batch, labels_batch = images_batch.to( device), labels_batch.to(device) # Doing the forward pass without the extra info pred_batch = model(images_batch) # Computing the loss L = loss_fn(pred_batch, labels_batch) # Computing the accuracy acc, topk_acc = accuracy(pred_batch, labels_batch, topk=(1, topk)) loss_avg.update(L.item()) acc_avg.update(acc.item()) topk_avg.update(topk_acc.item()) if metrics is not None: # Moving the data to CPU and converting it to numpy in order to compute the metrics pred_batch_np = nnF.softmax(pred_batch, dim=1).cpu().numpy() labels_batch_np = labels_batch.cpu().numpy() # updating the scores metrics.update_scores(labels_batch_np, pred_batch_np) # Updating tqdm t.set_postfix(loss='{:05.3f}'.format(loss_avg())) t.update() if metrics is not None: # Getting the metrics metrics.compute_metrics() bal_acc = metrics.metrics_values['balanced_accuracy'] else: bal_acc = None return { "loss": loss_avg(), "accuracy": acc_avg(), "topk_acc": topk_avg(), "balanced_accuracy": bal_acc }
net = net.to(device) net_dict = net.state_dict() pretrained_dict = {k: v for k, v in checkpoint['net'].items() if k in net_dict} if 'anchors' not in pretrained_dict.keys(): pretrained_dict['anchors'] = checkpoint['net']['means'] net.load_state_dict(pretrained_dict) net.eval() #find mean anchors for each class anchor_means = find_anchor_means(net, mapping, args.dataset, trial_num, cfg, only_correct = True) net.set_anchors(torch.Tensor(anchor_means)) print('==> Evaluating open set network accuracy for trial {}..'.format(trial_num)) x, y = gather_outputs(net, mapping, knownloader, data_idx = 1, calculate_scores = True) accuracy = metrics.accuracy(x, y) all_accuracy += [accuracy] print('==> Evaluating open set network AUROC for trial {}..'.format(trial_num)) xK, yK = gather_outputs(net, mapping, knownloader, data_idx = 1, calculate_scores = True) xU, yU = gather_outputs(net, mapping, unknownloader, data_idx = 1, calculate_scores = True, unknown = True) auroc = metrics.auroc(xK, xU) all_auroc += [auroc] mean_auroc = np.mean(all_auroc) mean_acc = np.mean(all_accuracy) print('Raw Top-1 Accuracy: {}'.format(all_accuracy)) print('Raw AUROC: {}'.format(all_auroc)) print('Average Top-1 Accuracy: {}'.format(mean_acc))
pcaModel.fit(normalisedTrainData) pcaTrainData = pcaModel.transform(normalisedTrainData) pcaTestData = pcaModel.transform(normalisedTestData) trainLabels = pp.convertLabels(trainLabels) testLabels = pp.convertLabels(testLabels) if (useNaiveBayes): NB = MultinomialNB() nbModel = NB.fit(np.array(normalisedTrainData), np.array(trainLabels)) predictionsMade = nbModel.predict( np.array(normalisedTestData)).tolist() tempAcc = met.accuracy(predictionsMade, testLabels) #tempF1 = met.precision_score(predictionsMade, testLabels) tempF1 = f1_score(testLabels, predictionsMade, average=None) nbAccuracy += tempAcc nbF1Score += tempF1 if (useSupportVectorMachine): svmModel = svm.SVC(gamma="scale") svmModel.fit(np.array(pcaTrainData), np.array(trainLabels)) predictionsMade = svmModel.predict( np.array(pcaTestData)).tolist() tempAcc = met.accuracy(predictionsMade, testLabels) #tempF1 = met.precision_score(predictionsMade, testLabels) tempF1 = f1_score(testLabels, predictionsMade, average=None) svmAccuracy += tempAcc
def main(): visualiseTrainingExamples() nn = NeuralNetwork(config.numLayers, config.numClasses, config.weightInitialisation, config.activationFn, config.weightDecay) nn.initialiseParams(len(x_train[0]) * len(x_train[0]), config.numNeurons) sample = np.random.randint(3 * len(x_train) / 4) nn.forwardPropagate(x_train[sample]) if config.optimizer == "sgd": nn.stochasticGradDesc(x_train, y_train, config.maxIterations, config.learningRate, config.batchSize) elif config.optimizer == "momentum": nn.momentumGradDesc(x_train, y_train, config.maxIterations, config.learningRate, config.batchSize, config.gamma) elif config.optimizer == "nesterov": nn.nesterovAcceleratedGradDesc(x_train, y_train, config.maxIterations, config.learningRate, config.batchSize, config.gamma) elif config.optimizer == "rmsprop": nn.rmsprop(x_train, y_train, config.maxIterations, config.learningRate, config.batchSize, config.eps, config.beta1) elif config.optimizer == "adam": nn.adam(x_train, y_train, config.maxIterations, config.learningRate, config.batchSize, config.eps, config.beta1, config.beta2) elif config.optimizer == "nadam": nn.nadam(x_train, y_train, config.maxIterations, config.learningRate, config.batchSize, config.eps, config.beta1, config.beta2) else: print("No such optimizer available.") predictions = [] predProbs = [] test_acc = 0 test_entropy = 0 test_mse = 0 for i in range(len(x_test)): nn.forwardPropagate(x_test[i]) predictions.append(nn.predictedClass) predProbs.append(nn.output[nn.predictedClass]) test_acc = accuracy(y_test, predictions) test_entropy = crossEntropyLoss(y_test, predProbs) test_mse = MSEloss(y_test, predictions) predictions = [] predProbs = [] valid_acc = 0 valid_entropy = 0 valid_mse = 0 for i in range(len(x_valid)): nn.forwardPropagate(x_valid[i]) predictions.append(nn.predictedClass) predProbs.append(nn.output[nn.predictedClass]) valid_acc = accuracy(y_valid, predictions) valid_entropy = crossEntropyLoss(y_valid, predProbs) valid_mse = MSEloss(y_valid, predictions) print( f"Test Set:\nAccuracy = {test_acc}\nLoss = {test_entropy}\nMSE = {test_mse}" ) print( f"Validation Set:\nAccuracy = {valid_acc}\nLoss = {valid_entropy}\nMSE = {valid_mse}" ) # #Log in wandb metrics = { 'test_acc': test_acc, # 'test_entropy': test_entropy, "test_mse": test_mse, 'valid_acc': valid_acc, # 'valid_entropy': valid_entropy, "valid_mse": valid_mse } wandb.log(metrics) run.finish()
def eval_reranker(res_fname="svm.test.res", pred_fname="svm.train.pred", format="trec", th=50, verbose=False, reranking_th=-100.0, ignore_noanswer=False, ignore_allanswer=False): ir, svm = read_res_pred_files(res_fname, pred_fname, format, verbose, reranking_th=reranking_th, ignore_noanswer=ignore_noanswer, ignore_allanswer=ignore_allanswer) # evaluate IR prec_se = metrics.recall_of_1(ir, th) acc_se = metrics.accuracy(ir, th) acc_se1 = metrics.accuracy1(ir, th) acc_se2 = metrics.accuracy2(ir, th) # evaluate SVM prec_svm = metrics.recall_of_1(svm, th) acc_svm = metrics.accuracy(svm, th) acc_svm1 = metrics.accuracy1(svm, th) acc_svm2 = metrics.accuracy2(svm, th) mrr_se = metrics.mrr(ir, th) mrr_svm = metrics.mrr(svm, th) map_se = metrics.map(ir) map_svm = metrics.map(svm) avg_acc1_svm = metrics.avg_acc1(svm, th) avg_acc1_ir = metrics.avg_acc1(ir, th) ''' print "%13s %5s" %("IR", "SVM") print "MRR: %5.2f %5.2f" %(mrr_se, mrr_svm) print "MAP: %5.4f %5.4f" %(map_se, map_svm) print "AvgRec: %5.2f %5.2f" %(avg_acc1_ir, avg_acc1_svm) print "%16s %6s %14s %6s %14s %6s %12s %4s" % ("IR", "SVM", "IR", "SVM", "IR", "SVM", "IR", "SVM") ''' rec1_se = -10 rec1_svm = -10 for i, (p_se, p_svm, a_se, a_svm, a_se1, a_svm1, a_se2, a_svm2) in enumerate( zip(prec_se, prec_svm, acc_se, acc_svm, acc_se1, acc_svm1, acc_se2, acc_svm2), 1): #print "REC-1@%02d: %6.2f %6.2f ACC@%02d: %6.2f %6.2f AC1@%02d: %6.2f %6.2f AC2@%02d: %4.0f %4.0f" %(i, p_se, p_svm, i, a_se, a_svm, i, a_se1, a_svm1, i, a_se2, a_svm2) if (rec1_se < -5): rec1_se = p_se rec1_svm = p_svm ''' print "REC-1 - percentage of questions with at least 1 correct answer in the top @X positions (useful for tasks were questions have at most one correct answer)" print "ACC - accuracy, i.e. number of correct answers retrieved at rank @X normalized by the rank and the total number of questions" print "AC1 - the number of correct answers at @X normalized by the number of maximum possible answers (perfect re-ranker)" print "AC2 - the absolute number of correct answers at @X" ''' print "Table view" print " MRR MAP P@1" print "REF_FILE %5.2f %5.2f %5.2f" % (mrr_se, map_se * 100, rec1_se) print "SVM %5.2f %5.2f %5.2f" % (mrr_svm, map_svm * 100, rec1_svm)
def accuracy_fn(self, X, y): preds = self.predict(X) return metrics.accuracy(y, preds)
def main(): train_path = sys.argv[1] + '\\train\\' test_path = sys.argv[1] + '\\test\\' # load training data print(f'[INFO] - Loading training data from {train_path}') res = read_data(train_path) train_data = res[0] train_target = res[1] print(f'[INFO] - Total train data: {len(train_data)}') print(f'[INFO] - Loading testing data from {test_path}') res = read_data(test_path) test_data = res[0] test_target = res[1] print(f'[INFO] - Total test data: {len(test_data)}') # 10% of training data will go to developer data set print(f'[INFO] - Splitting training data into training data and developer data (keeping 10% for training data)') res = train_test_split(train_data, train_target, test_size=0.1) train_data = res[0] train_target = res[2] print(f'[INFO] - Total training data after split {len(train_data)}') dev_data = res[1] dev_target = res[3] print(f'[INFO] - Total developer data {len(dev_data)}') nb = MultinomialNaiveBayes() accuracy_train = [] accuracy_test = [] counter = 1 for train_size in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]: print(f'\n[INFO] - Iteration No.{counter} (using {int(train_size*100)}% of 90% of train data).') if train_size != 1.0: res = train_test_split(train_data, train_target, train_size=train_size, shuffle=False) fold_data = res[0] fold_target = res[2] else: fold_data = train_data fold_target = train_target feature_size = 0.007 print(f'[INFO] - Fitting Multinomial Naive Bayes classifier using {feature_size*100:.1f}% of features...') nb.fit(fold_data, fold_target, feature_size) print(f'[INFO] - Predicting with Multinomial Naive Bayes classifier using train data...') nb_targets, _ = nb.predict(fold_data) accuracy_score = accuracy(fold_target, nb_targets) accuracy_train.append(accuracy_score) print(f'[INFO] - Accuracy: {accuracy_score}') print(f'[INFO] - Predicting with Multinomial Naive Bayes classifier using developer data...') nb_targets, _ = nb.predict(dev_data) accuracy_score = accuracy(dev_target, nb_targets) print(f'[INFO] - Accuracy: {accuracy_score}') print(f'[INFO] - Predicting with Multinomial Naive Bayes classifier using test data...') nb_targets, probabilities = nb.predict(test_data) accuracy_score = accuracy(test_target, nb_targets) accuracy_test.append(accuracy_score) print(f'[INFO] - Accuracy: {accuracy_score}') counter += 1 learning_curves_plot = plt.figure(1) plt.plot([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], accuracy_train, label='train') plt.plot([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], accuracy_test, label='test') plt.title('Learning Curves (Multinomial Naive Bayes)') plt.legend(loc='lower right') plt.xlabel('Number of Train Data') plt.ylabel('Accuracy') precision_recall_plot = plt.figure(2) average_precision, average_recall, thresholds = precision_recall(probabilities, test_target, 10) plt.step(average_recall, average_precision, where='post') plt.title('Precision-Recall Curve (Multinomial Naive Bayes)') plt.xlabel('Recall') plt.ylabel('Precision') f1_plot = plt.figure(3) f1_score = f1(average_precision, average_recall) plt.plot(thresholds, f1_score) plt.title('F1 Curve (Multinomial Naive Bayes)') plt.xlabel('Thresholds') plt.ylabel('F1 Measure') plt.show()
def train_stochastic(model, train_loader, test_loader, args, device='cpu'): if args['var_type'] == 'isotropic': trainable_noise_params = { 'params': model.base.sigma, 'lr': args['lr'], 'weight_decay': args['wd'] } elif args['var_type'] == 'anisotropic': trainable_noise_params = { 'params': model.base.L, 'lr': args['lr'], 'weight_decay': args['wd'] } optimizer = Adam([{ 'params': model.base.gen.parameters(), 'lr': args['lr'] }, { 'params': model.base.fc1.parameters(), 'lr': args['lr'] }, trainable_noise_params, { 'params': model.proto.parameters(), 'lr': args['lr'], 'weight_decay': args['wd'] }]) # Uncomment for the "train model and noise separately" ablation. But first train a model with disable_noise=True. # model.freeze_model_params() loss_func = nn.CrossEntropyLoss() if args['dataset'] == 'cifar10': norm_func = normalize_cifar10 elif args['dataset'] == 'cifar100': norm_func = normalize_cifar100 elif args['dataset'] == 'svhn': norm_func = normalize_generic elif args['dataset'] in ('mnist', 'fmnist'): norm_func = None best_test_acc = -1. for epoch in range(args['num_epochs']): for data, target in train_loader: data = data.to(device) target = target.to(device) model.train() if norm_func is not None: data = norm_func(data) logits = model(data) optimizer.zero_grad() if args['reg_type'] == 'wca': if args['var_type'] == 'isotropic': wca = (model.proto.weight @ model.sigma.diag() @ model.proto.weight.T).diagonal().sum() elif args['var_type'] == 'anisotropic': wca = (model.proto.weight @ model.sigma @ model.proto.weight.T).diagonal().sum() loss = loss_func(logits, target) - torch.log(wca) elif args['reg_type'] == 'max_entropy': me = model.base.dist.entropy().mean() loss = loss_func(logits, target) - torch.log(me) elif args['reg_type'] == 'wca+max_entropy': if args['var_type'] == 'isotropic': wca = (model.proto.weight @ model.sigma.diag() @ model.proto.weight.T).diagonal().sum() elif args['var_type'] == 'anisotropic': wca = (model.proto.weight @ model.sigma @ model.proto.weight.T).diagonal().sum() me = model.base.dist.entropy().mean() loss = loss_func(logits, target) - torch.log(wca) - torch.log(me) loss.backward() optimizer.step() if args['var_type'] == 'anisotropic': with torch.no_grad(): model.base.L.data = model.base.L.data.tril() train_acc = accuracy(model, train_loader, device=device, norm=norm_func) test_acc = accuracy(model, test_loader, device=device, norm=norm_func) print('Epoch {:03}, Train acc: {:.3f}, Test acc: {:.3f}'.format( epoch + 1, train_acc, test_acc)) if test_acc > best_test_acc: best_test_acc = test_acc model.save(os.path.join(args['output_path']['models'], 'ckpt_best')) print('Best accuracy achieved on epoch {}.'.format(epoch + 1))
def train(net, net_size, input_size, feature_dim, train_dataset, val_dataset, epochs, learning_rate, batch_size, save_path, pretrained_model=None): # create dataloader train_targets = [sampler[1] for sampler in train_dataset.imgs] weighted_sampler = ScheduledWeightedSampler(len(train_dataset), train_targets, 0.975, True) train_loader = DataLoader(train_dataset, batch_size=batch_size, sampler=weighted_sampler, drop_last=True) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False) # define model model = net(net_size, input_size, feature_dim).cuda() print_msg('Trainable layers: ', ['{}\t{}'.format(k, v) for k, v in model.layer_configs()]) # load pretrained weights if pretrained_model: pretrained_dict = model.load_weights(pretrained_model, ['fc', 'dense']) print_msg('Loaded weights from {}: '.format(pretrained_model), sorted(pretrained_dict.keys())) # define loss and optimizier MSELoss = torch.nn.MSELoss() optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True, weight_decay=0.0005) # optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0.0005) # learning rate warmup and decay milestones = [160, 230] warmup_epoch = 10 warmup_batch = (len(train_loader) // batch_size) * warmup_epoch lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=milestones, gamma=0.1) warmup_scheduler = WarmupLRScheduler(optimizer, warmup_batch, learning_rate) # train max_kappa = 0 record_epochs, accs, losses = [], [], [] model.train() for epoch in range(1, epochs + 1): # resampling weight update weighted_sampler.step() # learning rate update lr_scheduler.step() if epoch in milestones: curr_lr = optimizer.param_groups[0]['lr'] print_msg('Learning rate decayed to {}'.format(curr_lr)) if epoch > 1 and epoch <= warmup_epoch: curr_lr = optimizer.param_groups[0]['lr'] print_msg('Learning rate warmup to {}'.format(curr_lr)) epoch_loss = 0 correct = 0 total = 0 progress = tqdm(enumerate(train_loader)) for step, train_data in progress: if epoch <= warmup_epoch: warmup_scheduler.step() X, y = train_data X, y = X.cuda(), y.float().cuda() # forward y_pred = model(X) loss = MSELoss(y_pred, y) # backward optimizer.zero_grad() loss.backward() optimizer.step() # metrics epoch_loss += loss.item() total += y.size(0) correct += accuracy(y_pred, y) * y.size(0) avg_loss = epoch_loss / (step + 1) avg_acc = correct / total progress.set_description( 'epoch: {}, loss: {:.6f}, acc: {:.4f}'.format( epoch, avg_loss, avg_acc)) # save model c_matrix = np.zeros((5, 5), dtype=int) acc = _eval(model, val_loader, c_matrix) kappa = quadratic_weighted_kappa(c_matrix) print('validation accuracy: {}, kappa: {}'.format(acc, kappa)) if kappa > max_kappa: torch.save(model, save_path) max_kappa = kappa print_msg('Model save at {}'.format(save_path)) # record record_epochs.append(epoch) accs.append(acc) losses.append(avg_loss) return record_epochs, accs, losses
def KLR(K, y, Kval, yval, lambd, maxIter = 100, tresh = 1e-8): # initialize the values assert K.shape[0] == y.shape[0] n = K.shape[0] n_val = Kval.shape[0] y_ = np.ones(n) yval_ = np.ones(n_val) y_[y == 0] = -1 yval_[yval == 0] = -1 alphas = [] for l in lambd : cnt = 0 P_t, W_t = np.eye(n), np.eye(n) z_t = K@ np.ones(n) - y_ alpha_t = np.ones(n) diff_alpha = np.inf while (diff_alpha > tresh) and (cnt < maxIter): old_alpha = alpha_t ## Solving dual using CVXOpt #P = matrix(2*((K @ W_t @ K)/n + l*K)) #q = matrix((-2*K@W_t@y_)/n) #solvers.options['show_progress'] = False #sol=solvers.qp(P, q) #alpha_t = sol['x'] #alpha_t = np.reshape(alpha_t,-1) alpha_t = solveWKRR(K, W_t, z_t, y_, l) m_t = K@alpha_t sigma_m = sigmoid(m_t) sigma_my = sigmoid(-y_*m_t) P_t = - np.diag(sigma_my) W_t = np.diag(sigma_m * (1-sigma_m)) z_t = m_t - (P_t@y_)/(sigma_m * (1-sigma_m)) diff_alpha = np.linalg.norm(alpha_t - old_alpha) cnt+=1 if cnt % 10 == 0: print(l, cnt) loss_lambda = logistic_loss(y_, K@alpha_t) acc_lambda = accuracy(y,K@alpha_t, mode="SVM") loss_lambdaval = logistic_loss(yval_, Kval@alpha_t) acc_lambdaval = accuracy(yval,Kval@alpha_t, mode="SVM") print(f"***********lambda = {l}***********") print(f"Training: loss = {loss_lambda:.4f}, accuracy = {acc_lambda:.6f}") print(f"Validation: loss = {loss_lambdaval:.4f}, accuracy = {acc_lambdaval:.6f}") alphas +=[alpha_t] return(alphas)
adfuller(train_set_diff) acf_pacf(train_set_diff) # ряд стационарен, можно переходить к поиску оптимальных параметров # best_params.append(sarima_best_params(train_set, 7, 1, 0, 7, 3)) # построение модели model = sm.tsa.SARIMAX(train_set, order=(best_params[j][0], 1, best_params[j][1]), seasonal_order=(best_params[j][2], 0, best_params[j][3], 7)).fit(disp=False) # вывод информации о модели print(model.summary()) # проверка остатков модели на случайность ljungbox(model.resid) acf_pacf(model.resid) # SARIMA прогноз на конкретный час hour_pred = model.forecast(2)[-1] # добавление прогноза на час к итоговому прогнозу total_pred.append(hour_pred) # оценка прогноза по метрикам nnf = nnfmetrics(total_pred, test_set, plan_set) mse = mse(test_set, total_pred) mape = mape(test_set, total_pred) acc = accuracy(test_set, total_pred) print('Оценка по NNFMETRICS = ', nnf) print('Оценка по MSE = ', mse) print('Оценка по MAPE = ', mape) print('Точность прогноза = ', acc) # отрисовка графика plot_results(pred_date.strftime('%d-%m-%Y'), test_set, total_pred, plan_set)
def test(dataset): # # load BERT and GAN # load_gan_model(D, G, config['gan_save_path']) # if args.fine_tune: # load_model(E, path=config['bert_save_path'], model_name='bert') # test_dataloader = DataLoader(dataset, batch_size=args.predict_batch_size, shuffle=False, num_workers=2) n_sample = len(test_dataloader) result = dict() # Loss function detection_loss = torch.nn.CrossEntropyLoss().to(device) model.eval() all_detection_preds = [] all_detection_logit = [] total_loss = 0 for sample in tqdm(test_dataloader): sample = (i.to(device) for i in sample) token, mask, type_ids, y = sample batch = len(token) # -------------------------evaluate D------------------------- # # BERT encode sentence to feature vector with torch.no_grad(): logit = model(token, mask, type_ids) all_detection_logit.append(logit) all_detection_preds.append(torch.argmax(logit, 1)) total_loss += detection_loss(logit, y.long()) all_y = LongTensor( dataset.dataset[:, -1].astype(int)).cpu() # [length, n_class] all_binary_y = (all_y != 0).long() # [length, 1] label 0 is oos all_detection_preds = torch.cat(all_detection_preds, 0).cpu() # [length, 1] # all_detection_binary_preds = convert_to_int_by_threshold(all_detection_preds.squeeze()) # [length, 1] all_detection_logit = torch.cat(all_detection_logit, 0).cpu() # 计算损失 result['detection_loss'] = total_loss logger.info( metrics.classification_report(all_binary_y, all_detection_preds, target_names=['oos', 'in'])) # report oos_ind_precision, oos_ind_recall, oos_ind_fscore, _ = metrics.binary_recall_fscore( all_detection_preds, all_binary_y) detection_acc = metrics.accuracy(all_detection_preds, all_binary_y) # y_score = all_detection_preds.squeeze().tolist() y_score = all_detection_logit.softmax(1)[:, 1].tolist() eer = metrics.cal_eer(all_binary_y, y_score) test_logit = all_detection_logit.tolist() result['test_logit'] = test_logit result['eer'] = eer result['all_detection_preds'] = all_detection_preds result['detection_acc'] = detection_acc result['all_binary_y'] = all_binary_y result['oos_ind_precision'] = oos_ind_precision result['oos_ind_recall'] = oos_ind_recall result['oos_ind_f_score'] = oos_ind_fscore result['y_score'] = y_score result['auc'] = roc_auc_score(all_binary_y, y_score) return result
print("Fold number " + str(test_fl)) training, training_class, testing, testing_class = importfile.get_datasets(dir_name[dataset], class_name[dataset], str(test_fl), test_str, train_str) training = np.transpose(utils.normalize_columns(np.transpose(training))) testing = np.transpose(utils.normalize_columns(np.transpose(testing))) # Basic KNN (baseline) start = time.time() predicted = kNNAlgorithm.nearest_neighbor((training, training_class), testing, k, dist, policy) efficiencies[0] = time.time() - start accuracies[0] = metrics.accuracy(testing_class, predicted) recalls[0] = metrics.recall(testing_class, predicted) # Weighted KNN start = time.time() weights_tree = kNNWeightedAlgorithm.calculate_weights(utils.encode_data(training), training_class, 'TreeClassifier') predicted = kNNAlgorithm.nearest_neighbor((training, training_class), testing, k, dist, policy, weights_tree) efficiencies[1] = time.time() - start accuracies[1] = metrics.accuracy(testing_class, predicted) recalls[1] = metrics.recall(testing_class, predicted) start = time.time() weights_relieff = kNNWeightedAlgorithm.calculate_weights(utils.encode_data(training), training_class, 'Relieff')
def run(*options, cfg=None): update_config(config, options=options, config_file=cfg) torch.backends.cudnn.benchmark = config.CUDNN.BENCHMARK if torch.cuda.is_available(): torch.cuda.manual_seed_all(config.SEED) np.random.seed(seed=config.SEED) # Setup Augmentation/Transformation pipeline input_size = config.TRAIN.INPUT_SIZE resize_range_min = config.TRAIN.RESIZE_MIN # Data-parallel devices_lst = list(range(torch.cuda.device_count())) print("Devices {}".format(devices_lst)) if (config.TEST.MODALITY == "RGB") or (config.TEST.MODALITY == "combined"): rgb_loader = torch.utils.data.DataLoader( I3DDataSet(data_root=config.DATASET.DIR, split=config.DATASET.SPLIT, modality="RGB", train_mode=False, sample_frames_at_test=False, transform=torchvision.transforms.Compose([ GroupScale(config.TRAIN.RESIZE_MIN), GroupCenterCrop(config.TRAIN.INPUT_SIZE), GroupNormalize(modality="RGB"), Stack(), ])), batch_size=config.TEST.BATCH_SIZE, shuffle=False, num_workers=config.WORKERS, pin_memory=config.PIN_MEMORY) rgb_model_file = config.TEST.MODEL_RGB if not os.path.exists(rgb_model_file): raise FileNotFoundError(rgb_model_file, " does not exist") rgb_model = load_model(modality="RGB", state_dict_file=rgb_model_file) print("scoring with rgb model") targets, rgb_predictions = test(rgb_model, rgb_loader, "RGB") del rgb_model targets = targets.cuda(non_blocking=True) rgb_top1_accuracy = accuracy(rgb_predictions, targets, topk=(1, )) print("rgb top1 accuracy: ", rgb_top1_accuracy[0].cpu().numpy().tolist()) if (config.TEST.MODALITY == "flow") or (config.TEST.MODALITY == "combined"): flow_loader = torch.utils.data.DataLoader( I3DDataSet(data_root=config.DATASET.DIR, split=config.DATASET.SPLIT, modality="flow", train_mode=False, sample_frames_at_test=False, transform=torchvision.transforms.Compose([ GroupScale(config.TRAIN.RESIZE_MIN), GroupCenterCrop(config.TRAIN.INPUT_SIZE), GroupNormalize(modality="flow"), Stack(), ])), batch_size=config.TEST.BATCH_SIZE, shuffle=False, num_workers=config.WORKERS, pin_memory=config.PIN_MEMORY) flow_model_file = config.TEST.MODEL_FLOW if not os.path.exists(flow_model_file): raise FileNotFoundError(flow_model_file, " does not exist") flow_model = load_model(modality="flow", state_dict_file=flow_model_file) print("scoring with flow model") targets, flow_predictions = test(flow_model, flow_loader, "flow") del flow_model targets = targets.cuda(non_blocking=True) flow_top1_accuracy = accuracy(flow_predictions, targets, topk=(1, )) print("flow top1 accuracy: ", flow_top1_accuracy[0].cpu().numpy().tolist()) if config.TEST.MODALITY == "combined": predictions = torch.stack([rgb_predictions, flow_predictions]) predictions_mean = torch.mean(predictions, dim=0) top1accuracy = accuracy(predictions_mean, targets, topk=(1, )) print("combined top1 accuracy: ", top1accuracy[0].cpu().numpy().tolist())
plot_points(x_test, y_test, class_colors=class_colors, title='Test - correct labels') # Train the SVM classifier on the training data. svm = SVM(kernel_func=kernel_func, C=C) print('Training...') svm.train(x_train, y_train) # Plot the decision boundary. print('Plotting...') plot_svm_decision_boundary(svm, x_train, y_train, title='SVM decision boundary on training data', output_path=output_path, file_name=str(dataset_name) + '_support_vectors_train.png', class_colors=class_colors) # Make predictions on train and test data. y_train_pred = svm.predict(x_train) y_test_pred = svm.predict(x_test) plot_points(x_train, y_train_pred, class_colors=class_colors, title='Your predictions for training data') plot_points(x_test, y_test_pred, class_colors=class_colors, title='Your predictions for test data') # Compute the classification accuracy for train and test. acc_train = accuracy(y_train_pred, y_train) acc_test = accuracy(y_test_pred, y_test) print('Train accuracy: %.2f.' % acc_train) print('Test accuracy: %.2f.' % acc_test) print('Done.')
def score(self, X_test, y_test): y_predict = self.predict(X_test) return accuracy(y_predict.reshape(-1, 1), y_test)
def train(self): """ 训练模型 :return: """ gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9, allow_growth=True) sess_config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True, gpu_options=gpu_options) with tf.Session(config=sess_config) as sess: # 初始化变量值 sess.run(tf.global_variables_initializer()) current_step = 0 for epoch in range(self.config["epochs"]): print("----- Epoch {}/{} -----".format(epoch + 1, self.config["epochs"])) for batch in self.data_obj.next_batch( self.train_queries, self.config["batch_size"]): loss, predictions = self.model.train( sess, batch, self.config["keep_prob"]) acc = accuracy(predictions) print("train: step: {}, loss: {}, acc: {}".format( current_step, loss, acc)) current_step += 1 if current_step % self.config["checkpoint_every"] == 0: eval_losses = [] eval_acc = [] for eval_batch in self.data_obj.next_batch( self.eval_queries, self.config["batch_size"]): eval_loss, eval_predictions = self.model.eval( sess, eval_batch) eval_losses.append(eval_loss) acc = accuracy(eval_predictions) eval_acc.append(acc) print("\n") print("eval: , loss: {}, acc: {}".format( mean(eval_losses), mean(eval_acc))) print("\n") if self.config["ckpt_model_path"]: save_path = os.path.join( os.path.abspath(os.getcwd()), self.config["ckpt_model_path"]) if not os.path.exists(save_path): os.makedirs(save_path) model_save_path = os.path.join( save_path, self.config["model_name"]) self.model.saver.save(sess, model_save_path, global_step=current_step)
def test(dataset): # # load BERT and GAN # load_gan_model(D, G, config['gan_save_path']) # if args.fine_tune: # load_model(E, path=config['bert_save_path'], model_name='bert') # test_dataloader = DataLoader(dataset, batch_size=args.predict_batch_size, shuffle=False, num_workers=2) n_sample = len(test_dataloader) result = dict() # Loss function detection_loss = torch.nn.BCELoss().to(device) classified_loss = torch.nn.CrossEntropyLoss(ignore_index=0).to(device) D_detect.eval() E.eval() all_detection_preds = [] all_class_preds = [] all_features = [] for sample in tqdm(test_dataloader): sample = (i.to(device) for i in sample) token, mask, type_ids, y = sample batch = len(token) # -------------------------evaluate D------------------------- # # BERT encode sentence to feature vector with torch.no_grad(): sequence_output, pooled_output = E(token, mask, type_ids) real_feature = pooled_output discriminator_output, f_vector = D_detect(real_feature) all_detection_preds.append(discriminator_output) if args.do_vis: all_features.append(f_vector) all_y = LongTensor( dataset.dataset[:, -1].astype(int)).cpu() # [length, n_class] all_binary_y = (all_y != 0).long() # [length, 1] label 0 is oos all_detection_preds = torch.cat(all_detection_preds, 0).cpu() # [length, 1] all_detection_binary_preds = convert_to_int_by_threshold( all_detection_preds.squeeze()) # [length, 1] # 计算损失 detection_loss = detection_loss(all_detection_preds, all_binary_y.float()) result['detection_loss'] = detection_loss logger.info( metrics.classification_report(all_binary_y, all_detection_binary_preds, target_names=['oos', 'in'])) # report oos_ind_precision, oos_ind_recall, oos_ind_fscore, _ = metrics.binary_recall_fscore( all_detection_binary_preds, all_binary_y) detection_acc = metrics.accuracy(all_detection_binary_preds, all_binary_y) y_score = all_detection_preds.squeeze().tolist() eer = metrics.cal_eer(all_binary_y, y_score) result['eer'] = eer result['all_detection_binary_preds'] = all_detection_binary_preds result['detection_acc'] = detection_acc result['all_binary_y'] = all_binary_y result['oos_ind_precision'] = oos_ind_precision result['oos_ind_recall'] = oos_ind_recall result['oos_ind_f_score'] = oos_ind_fscore result['y_score'] = y_score result['auc'] = roc_auc_score(all_binary_y, y_score) if args.do_vis: all_features = torch.cat(all_features, 0).cpu().numpy() result['all_features'] = all_features return result
def prepare_student_data(dataset, nb_teachers, save=False): """ Takes a dataset name and the size of the teacher ensemble and prepares training data for the student model, according to parameters indicated in flags above. :param dataset: string corresponding to mnist, cifar10, or svhn :param nb_teachers: number of teachers (in the ensemble) to learn from :param save: if set to True, will dump student training labels predicted by the ensemble of teachers (with Laplacian noise) as npy files. It also dumps the clean votes for each class (without noise) and the labels assigned by teachers :return: pairs of (data, labels) to be used for student training and testing """ assert input.create_dir_if_needed(FLAGS.train_dir) # Load the dataset if dataset == 'svhn': test_data, test_labels = input.ld_svhn(test_only=True) elif dataset == 'cifar10': test_data, test_labels = input.ld_cifar10(test_only=True) elif dataset == 'mnist': test_data, test_labels = input.ld_mnist(test_only=True) else: print("Check value of dataset flag") return False # Make sure there is data leftover to be used as a test set assert FLAGS.stdnt_share < len(test_data) # Prepare [unlabeled] student training data (subset of test set) stdnt_data = test_data[:FLAGS.stdnt_share] # Compute teacher predictions for student training data teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data) # Aggregate teacher predictions to get student training labels if not save: stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale) else: # Request clean votes and clean labels as well stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max( teachers_preds, FLAGS.lap_scale, return_clean_votes=True) #NOLINT(long-line) # Prepare filepath for numpy dump of clean votes filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str( nb_teachers) + '_student_clean_votes_lap_' + str( FLAGS.lap_scale) + '.npy' # NOLINT(long-line) # Prepare filepath for numpy dump of clean labels filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str( nb_teachers) + '_teachers_labels_lap_' + str( FLAGS.lap_scale) + '.npy' # NOLINT(long-line) # Dump clean_votes array with tf.gfile.Open(filepath, mode='w') as file_obj: np.save(file_obj, clean_votes) # Dump labels_for_dump array with tf.gfile.Open(filepath_labels, mode='w') as file_obj: np.save(file_obj, labels_for_dump) # Print accuracy of aggregated labels ac_ag_labels = metrics.accuracy(stdnt_labels, test_labels[:FLAGS.stdnt_share]) print("Accuracy of the aggregated labels: " + str(ac_ag_labels)) # Store unused part of test set for use as a test set after student training stdnt_test_data = test_data[FLAGS.stdnt_share:] stdnt_test_labels = test_labels[FLAGS.stdnt_share:] if save: # Prepare filepath for numpy dump of labels produced by noisy aggregation filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str( nb_teachers) + '_student_labels_lap_' + str( FLAGS.lap_scale) + '.npy' #NOLINT(long-line) # Dump student noisy labels array with tf.gfile.Open(filepath, mode='w') as file_obj: np.save(file_obj, stdnt_labels) return stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels
def eval_reranker(res_fname="svm.test.res", pred_fname="svm.train.pred", format="trec", th=10, verbose=False, reranking_th=0.0, ignore_noanswer=False): ir, svm, conf_matrix = read_res_pred_files(res_fname, pred_fname, format, verbose, reranking_th=reranking_th, ignore_noanswer=ignore_noanswer) # Calculate standard P, R, F1, Acc acc = 1.0 * (conf_matrix['true']['true'] + conf_matrix['false']['false']) / (conf_matrix['true']['true'] + conf_matrix['false']['false'] + conf_matrix['true']['false'] + conf_matrix['false']['true']) p = 0 if (conf_matrix['true']['true'] + conf_matrix['false']['true']) > 0: p = 1.0 * (conf_matrix['true']['true']) / (conf_matrix['true']['true'] + conf_matrix['false']['true']) r = 0 if (conf_matrix['true']['true'] + conf_matrix['true']['false']) > 0: r = 1.0 * (conf_matrix['true']['true']) / (conf_matrix['true']['true'] + conf_matrix['true']['false']) f1 = 0 if (p + r) > 0: f1 = 2.0 * p * r / (p + r) # evaluate IR prec_se = metrics.recall_of_1(ir, th) acc_se = metrics.accuracy(ir, th) acc_se1 = metrics.accuracy1(ir, th) acc_se2 = metrics.accuracy2(ir, th) # evaluate SVM prec_svm = metrics.recall_of_1(svm, th) acc_svm = metrics.accuracy(svm, th) acc_svm1 = metrics.accuracy1(svm, th) acc_svm2 = metrics.accuracy2(svm, th) mrr_se = metrics.mrr(ir, th) mrr_svm = metrics.mrr(svm, th) map_se = metrics.map(ir, th) map_svm = metrics.map(svm, th) avg_acc1_svm = metrics.avg_acc1(svm, th) avg_acc1_ir = metrics.avg_acc1(ir, th) print ("") print ("*** Official score (MAP for SYS): %5.4f" %(map_svm)) print ("") print ("") print( "******************************") print( "*** Classification results ***") print( "******************************") print( "") print( "Acc = %5.4f" %(acc)) print( "P = %5.4f" %(p)) print( "R = %5.4f" %(r)) print( "F1 = %5.4f" %(f1)) print( "") print( "") print( "********************************") print( "*** Detailed ranking results ***") print( "********************************") print( "") print( "IR -- Score for the output of the IR system (baseline).") print( "SYS -- Score for the output of the tested system.") print( "") print( "%13s %5s" %("IR", "SYS")) print( "MAP : %5.4f %5.4f" %(map_se, map_svm)) print( "AvgRec: %5.4f %5.4f" %(avg_acc1_ir, avg_acc1_svm)) print( "MRR : %6.2f %6.2f" %(mrr_se, mrr_svm)) print( "%16s %6s %14s %6s %14s %6s %12s %4s" % ("IR", "SYS", "IR", "SYS", "IR", "SYS", "IR", "SYS")) for i, (p_se, p_svm, a_se, a_svm, a_se1, a_svm1, a_se2, a_svm2) in enumerate(zip(prec_se, prec_svm, acc_se, acc_svm, acc_se1, acc_svm1, acc_se2, acc_svm2), 1): print( "REC-1@%02d: %6.2f %6.2f ACC@%02d: %6.2f %6.2f AC1@%02d: %6.2f %6.2f AC2@%02d: %4.0f %4.0f" %(i, p_se, p_svm, i, a_se, a_svm, i, a_se1, a_svm1, i, a_se2, a_svm2)) print( "REC-1 - percentage of questions with at least 1 correct answer in the top @X positions (useful for tasks where questions have at most one correct answer)") print( "ACC - accuracy, i.e., number of correct answers retrieved at rank @X normalized by the rank and the total number of questions") print( "AC1 - the number of correct answers at @X normalized by the number of maximum possible answers (perfect re-ranker)") print( "AC2 - the absolute number of correct answers at @X") return map_svm
def train_teacher(dataset, nb_teachers, teacher_id): """ This function trains a teacher (teacher id) among an ensemble of nb_teachers models for the dataset specified. :param dataset: string corresponding to dataset (svhn, cifar10) :param nb_teachers: total number of teachers in the ensemble :param teacher_id: id of the teacher being trained :return: True if everything went well """ # If working directories do not exist, create them assert input.create_dir_if_needed(FLAGS.data_dir) assert input.create_dir_if_needed(FLAGS.train_dir) print("teacher {}:".format(teacher_id)) # Load the dataset if dataset == 'svhn': train_data,train_labels,test_data,test_labels = input.ld_svhn(extended=True) elif dataset == 'cifar10': train_data, train_labels, test_data, test_labels = input.ld_cifar10() elif dataset == 'mnist': train_data, train_labels, test_data, test_labels = input.ld_mnist() else: print("Check value of dataset flag") return False path = os.path.abspath('.') path1 = path + '\\plts_nodisturb\\' # 对标签进行干扰 import copy train_labels1 = copy.copy(train_labels) train_labels2 = disturb(train_labels, 0.3) disturb(test_labels, 0.3) #path1 = path + '\\plts_withdisturb\\' # Retrieve subset of data for this teacher #干扰前 data, labels = input.partition_dataset(train_data, train_labels, nb_teachers, teacher_id) from pca import K_S import operator print(operator.eq(train_labels1, train_labels2)) print("干扰前: ", K_S.tst_norm(train_labels1)) print("干扰后: ", K_S.tst_norm(train_labels2)) print(K_S.tst_samp(train_labels1, train_labels2)) print("Length of training data: " + str(len(labels))) # Define teacher checkpoint filename and full path if FLAGS.deeper: filename = str(nb_teachers) + '_teachers_' + str(teacher_id) + '_deep.ckpt' else: filename = str(nb_teachers) + '_teachers_' + str(teacher_id) + '.ckpt' ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + filename # Perform teacher training losses = deep_cnn.train(data, labels, ckpt_path) # Append final step value to checkpoint for evaluation ckpt_path_final = ckpt_path + '-' + str(FLAGS.max_steps - 1) # Retrieve teacher probability estimates on the test data teacher_preds = deep_cnn.softmax_preds(test_data, ckpt_path_final) # Compute teacher accuracy precision = metrics.accuracy(teacher_preds, test_labels) print('Precision of teacher after training: ' + str(precision)) print("each n step loss: ", losses) #x = list(range(1, len(losses)+1)) #plt.plot(x, losses, 'bo-', markersize=20) #plt.savefig(path1 + 'loss' + str(teacher_id) + '.jpg') #plt.show() #print("x: ",x) #print("loss: ", losses) return True
def train(args, train_loader, all_models, optimizer, epoch): """ Trains for one epoch of the train_loader dataset. """ # switch all models to train mode for m in all_models: m.train() (trunk_model, incident_layer, place_layer) = all_models # holds some metrics a_v_batch_time = AverageMeter() a_v_data_time = AverageMeter() a_v_losses = AverageMeter() a_v_incident_top1 = AverageMeter() a_v_place_top1 = AverageMeter() a_v_incident_top5 = AverageMeter() a_v_place_top5 = AverageMeter() # set end time as current time before training on a batch end_time = time.time() for batch_iteration, (input_data, target_p_v, target_d_v, weight_p_v, weight_d_v) in enumerate(train_loader): # measure data loading time a_v_data_time.update(time.time() - end_time) image_v = input_data.cuda(non_blocking=True) target_p_v = target_p_v.cuda(non_blocking=True) target_d_v = target_d_v.cuda(non_blocking=True) weight_p_v = weight_p_v.cuda(non_blocking=True) weight_d_v = weight_d_v.cuda(non_blocking=True) # input_v = torch.autograd.Variable(image_v) # target_p_v = torch.autograd.Variable(target_p_v) # target_d_v = torch.autograd.Variable(target_d_v) # weight_p_v = torch.autograd.Variable(weight_p_v) # weight_d_v = torch.autograd.Variable(weight_d_v) # compute output output = trunk_model(image_v) place_output = place_layer(output) incident_output = incident_layer(output) # get the loss according to parameters loss, incident_output, place_output = get_loss(args, incident_output, target_d_v, weight_d_v, place_output, target_p_v, weight_p_v) # measure accuracy and record loss incident_prec1, incident_prec5 = accuracy(incident_output.data, target_d_v, topk=1), \ accuracy(incident_output.data, target_d_v, topk=5) place_prec1, place_prec5 = accuracy(place_output.data, target_p_v, topk=1), \ accuracy(place_output.data, target_p_v, topk=5) a_v_losses.update(loss.data, input_data.size(0)) a_v_place_top1.update(place_prec1, input_data.size(0)) a_v_incident_top1.update(incident_prec1, input_data.size(0)) a_v_place_top5.update(place_prec5, input_data.size(0)) a_v_incident_top5.update(incident_prec5, input_data.size(0)) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time a_v_batch_time.update(time.time() - end_time) end_time = time.time() if batch_iteration % args.print_freq == 0: print( 'Epoch: [{0}][{1}/{2}]\t' 'Time {a_v_batch_time.val:.3f} ({a_v_batch_time.avg:.3f})\t' 'Data {a_v_data_time.val:.3f} ({a_v_data_time.avg:.3f})\t' 'Loss {a_v_losses.val:.4f} ({a_v_losses.avg:.4f})\t' 'Incident Prec@1 {a_v_incident_top1.val:.3f} ({a_v_incident_top1.avg:.3f})\t' 'Place Prec@1 {a_v_place_top1.val:.3f} ({a_v_place_top1.avg:.3f})\t' 'Place Prec@5 {a_v_place_top5.val:.3f} ({a_v_place_top5.avg:.3f})\t' 'Incident Prec@5 {a_v_incident_top5.val:.3f} ({a_v_incident_top5.avg:.3f})\t' .format(epoch, batch_iteration, len(train_loader), a_v_batch_time=a_v_batch_time, a_v_data_time=a_v_data_time, a_v_losses=a_v_losses, a_v_incident_top1=a_v_incident_top1, a_v_place_top1=a_v_place_top1, a_v_incident_top5=a_v_incident_top5, a_v_place_top5=a_v_place_top5)) # TODO: add more metrics here writer.add_scalar('Loss/train', a_v_losses.avg, batch_iteration + epoch * len(train_loader)) writer.add_scalar('Accuracy/train_place_1', a_v_place_top1.avg, batch_iteration + epoch * len(train_loader)) writer.add_scalar('Accuracy/train_place_5', a_v_place_top5.avg, batch_iteration + epoch * len(train_loader)) writer.add_scalar('Accuracy/train_incident_1', a_v_incident_top1.avg, batch_iteration + epoch * len(train_loader)) writer.add_scalar('Accuracy/train_incident_5', a_v_incident_top5.avg, batch_iteration + epoch * len(train_loader))
with np.load(weight_path) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] nlayers = len(lasagne.layers.get_all_params(simple_net_output)) lasagne.layers.set_all_param_values(simple_net_output, param_values[:nlayers]) print 'Done assigning weights' # In[33]: print "Defining and compiling test functions" test_prediction = lasagne.layers.get_output(simple_net_output[0], deterministic=True) test_loss = categorical_crossentropy(test_prediction, target_var) test_loss = test_loss.mean() test_acc, test_acc_per_sample = accuracy(test_prediction, target_var, void_labels) test_jacc = jaccard(test_prediction, target_var, n_classes) test_fn = theano.function( [input_var, target_var], [test_loss, test_acc, test_jacc, test_acc_per_sample]) print "Done" # In[34]: #Function computing the prediction with current parameters (for visualization) pred = theano.function([input_var], lasagne.layers.get_output(net['probs_reshape'], deterministic=True)) # In[35]:
def train(): """Training""" opt = FLAGS tf.logging.info("Build CleanNet...") batch_size = opt.batch_size_sup + opt.batch_size_unsup model = CleanNet(opt.num_ref, opt.img_dim, opt.embed_norm, opt.dropout_rate, opt.weight_decay) # phi_s: class embedding (batch_size, embed_size) # v_q: query image feature (batch_size, img_dim) # phi_q: query embedding (batch_size, embed_size) # v_qr: reconstructed query image feature (batch_size, img_dim) phi_s, v_q, phi_q, v_qr = model.forward(is_training=True) # verification labels vlabel = tf.placeholder(tf.float32, shape=(None,), name="vlabel") # verification flags indicating a sample is for supervised(1) or unsupervised(0) training vflag = tf.placeholder(tf.float32, shape=(None,), name="vflag") cos_sim = similarity(phi_s, phi_q) acc = accuracy(vlabel[:opt.batch_size_sup], cos_sim[:opt.batch_size_sup], threshold=0.1, scope="train_acc") val_acc = accuracy(vlabel, cos_sim, threshold=opt.val_sim_thres, scope="val_acc_at_{}".format(opt.val_sim_thres)) tf.summary.scalar('train/accuracy', acc) objective_loss = tf.reduce_mean(total_loss(vlabel, cos_sim, phi_s, v_q, phi_q, v_qr, vflag, opt.neg_weight, beta=0.1, gamma=0.1)) tf.summary.scalar('train/objective_loss', objective_loss) regularization_loss = tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) tf.summary.scalar('train/regularization_loss', regularization_loss) loss = objective_loss + regularization_loss tf.summary.scalar('train/loss', loss) lr = tf.train.exponential_decay(opt.learning_rate, model.global_step, opt.lr_update, opt.lr_decay, staircase=True) tf.summary.scalar('train/lr', lr) merged = tf.summary.merge_all() optimizer = tf.train.MomentumOptimizer(lr, opt.momentum) train_op = optimizer.minimize(loss, global_step=model.global_step) tf.logging.info("Get data batcher...") supervised_data = data_provider_factory.get_data_batcher('trainval', 'train', opt) val_data = data_provider_factory.get_data_batcher('trainval', 'val', opt) if opt.batch_size_unsup > 0: unsupervised_data = data_provider_factory.get_data_batcher('trainval', 'unverified', opt) saver = tf.train.Saver() init_op = tf.global_variables_initializer() with tf.Session() as sess: train_summary_writer = tf.summary.FileWriter(opt.log_dir + '/train', sess.graph) val_summary_writer = tf.summary.FileWriter(opt.log_dir + '/val') cur_step = 0 best_avg_val_acc = 0.0 sess.run(init_op) # recover from latest checkpoint and run validation if available ckpt = tf.train.get_checkpoint_state(opt.checkpoint_dir) if ckpt: saver.restore(sess, ckpt.model_checkpoint_path) saver.recover_last_checkpoints(ckpt.all_model_checkpoint_paths) cur_step, avg_val_acc = validation(sess, model, loss, val_acc, vlabel, vflag, opt.val_batch_size, val_data, val_summary_writer) best_avg_val_acc = avg_val_acc tf.logging.info("Recover model at global step = %d.", cur_step) else: tf.logging.info("Training from scratch.") while cur_step < opt.n_step: # data for supervised training _, batch_vlabel, batch_q, batch_vflag, batch_ref = supervised_data.get_batch(batch_size=opt.batch_size_sup) # data for unsupervised training if opt.batch_size_unsup > 0: # ubatch_vlabel_u is a dummy zero tensor since unsupervised samples don't have verification labels _, ubatch_vlabel_u, ubatch_q, ubatch_vflag, ubatch_ref = unsupervised_data.get_batch(batch_size=opt.batch_size_unsup) # concate supervised and unsupervied training data batch_vlabel = np.concatenate([batch_vlabel, ubatch_vlabel_u], axis=0) batch_q = np.concatenate([batch_q, ubatch_q], axis=0) batch_vflag = np.concatenate([batch_vflag, ubatch_vflag], axis=0) batch_ref = np.concatenate([batch_ref, ubatch_ref], axis=0) _, cur_step, cur_loss, cur_acc, summary = sess.run([train_op, model.global_step, loss, acc, merged], feed_dict={model.reference: batch_ref, model.query: batch_q, vlabel: batch_vlabel, vflag: batch_vflag}) train_summary_writer.add_summary(summary, cur_step) if cur_step % opt.log_interval == 0: tf.logging.info('step {}: train/loss = {}, train/acc = {}'.format(cur_step, cur_loss, cur_acc)) if cur_step % opt.val_interval == 0 and cur_step != 0: _, avg_val_acc = validation(sess, model, loss, val_acc, vlabel, vflag, opt.val_batch_size, val_data, val_summary_writer) if not os.path.exists(opt.checkpoint_dir): os.mkdir(opt.checkpoint_dir) save_path = saver.save(sess, opt.checkpoint_dir) print("Model saved in path: %s" % save_path) if avg_val_acc > best_avg_val_acc: best_avg_val_acc = avg_val_acc model_path = os.path.join(save_path, "checkpoint") best_model_path = os.path.join(save_path, "best_model_{}".format(cur_step)) shutil.copy(model_path, best_model_path) print("Best model saved in path: %s" % best_model_path)
def test_regression(model, test_features, test_labels): model.eval() return accuracy(model(test_features), test_labels)
""" Dataset Preprocessing """ from sklearn.datasets import load_breast_cancer data = load_breast_cancer() X = data.data y = data.target scalar = MinMaxScaler() scalar.fit(X) X = scalar.transform(X) print("######### Q1 Part A") LR = LogisticRegression(lr=0.1,epoch=1000) LR.fit(X,y) y_hat = LR.predict(X) print("Accuracy: ",end = "") print(accuracy(y_hat,y)) print("######### Q1 Part B") LR = LogisticRegression(lr=0.1,epoch=1000) LR.fit_autograd(X,y) y_hat = LR.predict(X) print("Accuracy: ",end = "") print(accuracy(y_hat,y)) print("######### Q1 Part C") acc=0 X = np.append(X, np.matrix(y).T, axis=1) k_fold = KFold(3, shuffle=True, random_state=1) for train, test in k_fold.split(X): LR = LogisticRegression(lr=0.1,epoch=1000) train_all = X[train]
def prepare_student_data(dataset, nb_teachers, save=False): """ Takes a dataset name and the size of the teacher ensemble and prepares training data for the student model, according to parameters indicated in flags above. :param dataset: string corresponding to mnist, cifar10, or svhn :param nb_teachers: number of teachers (in the ensemble) to learn from :param save: if set to True, will dump student training labels predicted by the ensemble of teachers (with Laplacian noise) as npy files. It also dumps the clean votes for each class (without noise) and the labels assigned by teachers :return: pairs of (data, labels) to be used for student training and testing """ assert input.create_dir_if_needed(FLAGS.train_dir) # Load the dataset if dataset == 'svhn': test_data, test_labels = input.ld_svhn(test_only=True) elif dataset == 'cifar10': test_data, test_labels = input.ld_cifar10(test_only=True) elif dataset == 'mnist': test_data, test_labels = input.ld_mnist(test_only=True) elif dataset == 'digit': test_data, test_labels = input.ld_digit_test(test_name=FLAGS.test_name, num=2000) else: print("Check value of dataset flag") return False # Make sure there is data leftover to be used as a test set assert FLAGS.stdnt_share < len(test_data) # Prepare [unlabeled] student training data (subset of test set) if (FLAGS.d_stu > -1): # stdnt_data = [] # for i in range(FLAGS.stdnt_share): # new_img = transform.resize(skimage.img_as_ubyte(test_data[i].astype(int)),(28,28)) # if FLAGS.d_stu == 3: # new_img = color.rgb2gray(new_img) # else: # new_img = new_img[ :,:, FLAGS.d_stu] # stdnt_data.append(new_img.reshape(28,28,1).astype(np.float32)) # stdnt_data = np.array(stdnt_data) trimmed = test_data[:FLAGS.stdnt_share, 2:30, 2:30, :] # grey scale if (FLAGS.d_stu == 3): stdnt_data = 0.2125 * trimmed[:, :, :, 0] + 0.7154 * trimmed[:, :, :, 1] + 0.0721 * trimmed[:, :, :, 2] else: stdnt_data = trimmed[:, :, :, FLAGS.d_stu] stdnt_data = stdnt_data.reshape((-1, 28, 28, 1)) else: stdnt_data = test_data[:FLAGS.stdnt_share] # Compute teacher predictions for student training data teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data) # Aggregate teacher predictions to get student training labels if not save: stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale) else: # Request clean votes and clean labels as well stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max( teachers_preds, FLAGS.lap_scale, return_clean_votes=True) #NOLINT(long-line) # Prepare filepath for numpy dump of clean votes filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str( nb_teachers) + '_student_clean_votes_lap_' + str( FLAGS.lap_scale) + '.npy' # NOLINT(long-line) # Prepare filepath for numpy dump of clean labels filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str( nb_teachers) + '_teachers_labels_lap_' + str( FLAGS.lap_scale) + '.npy' # NOLINT(long-line) # Dump clean_votes array with tf.gfile.Open(filepath, mode='w') as file_obj: np.save(file_obj, clean_votes) # Dump labels_for_dump array with tf.gfile.Open(filepath_labels, mode='w') as file_obj: np.save(file_obj, labels_for_dump) # Print accuracy of aggregated labels ac_ag_labels = metrics.accuracy(stdnt_labels, test_labels[:FLAGS.stdnt_share]) print("Accuracy of the aggregated labels: " + str(ac_ag_labels)) # Store unused part of test set for use as a test set after student training if FLAGS.dataset_teacher == 'mnist': test_data, test_labels = input.ld_mnist(test_only=True) else: assert 0 == 1, "Non implemented error: dataset_teacher not equals to mnist" # if FLAGS.d_stu > -1: # stdnt_test_data = test_data[FLAGS.stdnt_share:, 2:30, 2:30, FLAGS.d_stu : FLAGS.d_stu+1] # else: stdnt_test_data = test_data[FLAGS.stdnt_share:] stdnt_test_labels = test_labels[FLAGS.stdnt_share:] if save: # Prepare filepath for numpy dump of labels produced by noisy aggregation filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str( nb_teachers) + '_student_labels_lap_' + str( FLAGS.lap_scale) + '.npy' #NOLINT(long-line) # Dump student noisy labels array with tf.gfile.Open(filepath, mode='w') as file_obj: np.save(file_obj, stdnt_labels) return stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels
from sklearn.model_selection import train_test_split from metrics import accuracy import matplotlib.pyplot as plt from sklearn.metrics import roc_curve digits = datasets.load_digits() # print(digits.keys()) X = digits.data y = digits.target.copy() y[digits.target == 9] = 1 y[digits.target != 9] = 0 X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666) lr = LogisticRegression() lr.fit(X_train, y_train) y_predict = lr.predict(X_test) print(accuracy(y_test, y_predict)) def TN(y_true, y_predict): assert len(y_true) == len( y_predict), "the size of y_true must be equal to y_predict" return np.sum((y_true == 0) & (y_predict == 0)) # print(TN(y_test, y_predict)) def TP(y_true, y_predict): assert len(y_true) == len(y_predict) return np.sum((y_true == 1) & (y_predict == 1)) def FN(y_true, y_predict):
x_shared.set_value(x_chunk_eval) outputs_chunk = [] for b in xrange(num_batches_chunk_eval): out = compute_output(b) outputs_chunk.append(out) outputs_chunk = np.vstack(outputs_chunk) outputs_chunk = outputs_chunk[:chunk_length_eval] outputs.append(outputs_chunk) outputs = np.vstack(outputs) outputs_labels = np.argmax(outputs, axis=1) loss = np.mean(log_losses(outputs, labels)) acc = accuracy(outputs_labels, labels) kappa_eval = continuous_kappa( outputs_labels, labels, ) metric, conf_mat, \ hist_rater_a, hist_rater_b, \ nom, denom = kappa_eval try: kappa_cont_eval = continuous_kappa(outputs, labels, y_pow=model.y_pow) except: kappa_cont_eval = [1, 1, 1, 1, 1, 1]
def test(dataset): # load BERT and GAN load_gan_model(D, G, config['gan_save_path']) if args.fine_tune: load_model(E, path=config['bert_save_path'], model_name='bert') test_dataloader = DataLoader(dataset, batch_size=args.predict_batch_size, shuffle=False, num_workers=2) n_sample = len(test_dataloader) result = dict() # Loss function detection_loss = torch.nn.BCELoss().to(device) classified_loss = torch.nn.CrossEntropyLoss(ignore_index=0).to(device) G.eval() D.eval() E.eval() all_detection_preds = [] all_class_preds = [] all_features = [] for sample in tqdm.tqdm(test_dataloader): sample = (i.to(device) for i in sample) token, mask, type_ids, y = sample batch = len(token) # -------------------------evaluate D------------------------- # # BERT encode sentence to feature vector with torch.no_grad(): sequence_output, pooled_output = E(token, mask, type_ids) real_feature = pooled_output # 大于2表示除了训练判别器还要训练分类器 if n_class > 2: f_vector, discriminator_output, classification_output = D( real_feature, return_feature=True) all_detection_preds.append(discriminator_output) all_class_preds.append(classification_output) # 只预测判别器 else: f_vector, discriminator_output = D.detect_only( real_feature, return_feature=True) all_detection_preds.append(discriminator_output) if args.do_vis: all_features.append(f_vector) all_y = LongTensor( dataset.dataset[:, -1].astype(int)).cpu() # [length, n_class] all_binary_y = (all_y != 0).long() # [length, 1] label 0 is oos all_detection_preds = torch.cat(all_detection_preds, 0).cpu() # [length, 1] all_detection_binary_preds = convert_to_int_by_threshold( all_detection_preds.squeeze()) # [length, 1] # 计算损失 detection_loss = detection_loss(all_detection_preds, all_binary_y.float()) result['detection_loss'] = detection_loss if n_class > 2: class_one_hot_preds = torch.cat(all_class_preds, 0).detach().cpu() # one hot label class_loss = classified_loss(class_one_hot_preds, all_y) # compute loss all_class_preds = torch.argmax(class_one_hot_preds, 1) # label class_acc = metrics.ind_class_accuracy( all_class_preds, all_y, oos_index=0) # accuracy for ind class logger.info( metrics.classification_report( all_y, all_class_preds, target_names=processor.id_to_label)) logger.info( metrics.classification_report(all_binary_y, all_detection_binary_preds, target_names=['oos', 'in'])) # report oos_ind_precision, oos_ind_recall, oos_ind_fscore, _ = metrics.binary_recall_fscore( all_detection_binary_preds, all_binary_y) detection_acc = metrics.accuracy(all_detection_binary_preds, all_binary_y) y_score = all_detection_preds.squeeze().tolist() eer = metrics.cal_eer(all_binary_y, y_score) result['eer'] = eer result['all_detection_binary_preds'] = all_detection_binary_preds result['detection_acc'] = detection_acc result['all_binary_y'] = all_binary_y result['all_y'] = all_y result['oos_ind_precision'] = oos_ind_precision result['oos_ind_recall'] = oos_ind_recall result['oos_ind_f_score'] = oos_ind_fscore result['score'] = y_score result['y_score'] = y_score result['auc'] = roc_auc_score(all_binary_y, y_score) if n_class > 2: result['class_loss'] = class_loss result['class_acc'] = class_acc if args.do_vis: all_features = torch.cat(all_features, 0).cpu().numpy() result['all_features'] = all_features freeze_data['test_all_y'] = all_y.tolist() freeze_data['test_all_pred'] = all_detection_binary_preds.tolist() freeze_data['test_score'] = y_score return result
def main(args): """Run training and validation. 1. Build graphs 1.1 Training graph to run on multiple GPUs 1.2 Validation graph to run on multiple GPUs 2. Configure sessions 2.1 Train 2.2 Validate 3. Main loop 3.1 Train 3.2 Write summary 3.3 Save model 3.4 Validate model Author: Ashley Gritzman """ # Set reproduciable random seed tf.set_random_seed(1234) # Directories train_dir, train_summary_dir = conf.setup_train_directories() # Logger conf.setup_logger(logger_dir=train_dir, name="logger_train.txt") # Hyperparameters conf.load_or_save_hyperparams(train_dir) # Get dataset hyperparameters logger.info('Using dataset: {}'.format(FLAGS.dataset)) dataset_size_train = conf.get_dataset_size_train(FLAGS.dataset) dataset_size_val = conf.get_dataset_size_validate(FLAGS.dataset) build_arch = conf.get_dataset_architecture(FLAGS.dataset) num_classes = conf.get_num_classes(FLAGS.dataset) create_inputs_train = conf.get_create_inputs(FLAGS.dataset, mode="train") create_inputs_train_wholeset = conf.get_create_inputs(FLAGS.dataset, mode="train_whole") if dataset_size_val > 0: create_inputs_val = conf.get_create_inputs(FLAGS.dataset, mode="validate") #***************************************************************************** # 1. BUILD GRAPHS #***************************************************************************** #---------------------------------------------------------------------------- # GRAPH - TRAIN #---------------------------------------------------------------------------- logger.info('BUILD TRAIN GRAPH') g_train = tf.Graph() with g_train.as_default(), tf.device('/cpu:0'): # Get global_step global_step = tf.train.get_or_create_global_step() # Get batches per epoch num_batches_per_epoch = int(dataset_size_train / FLAGS.batch_size) # In response to a question on OpenReview, Hinton et al. wrote the # following: # "We use an exponential decay with learning rate: 3e-3, decay_steps: 20000, # decay rate: 0.96." # https://openreview.net/forum?id=HJWLfGWRb¬eId=ryxTPFDe2X lrn_rate = tf.train.exponential_decay(learning_rate=FLAGS.lrn_rate, global_step=global_step, decay_steps=20000, decay_rate=0.96) tf.summary.scalar('learning_rate', lrn_rate) opt = tf.train.AdamOptimizer(learning_rate=lrn_rate) # Get batch from data queue. Batch size is FLAGS.batch_size, which is then # divided across multiple GPUs input_dict = create_inputs_train() batch_x = input_dict['image'] batch_labels = input_dict['label'] # AG 03/10/2018: Split batch for multi gpu implementation # Each split is of size FLAGS.batch_size / FLAGS.num_gpus # See: https://github.com/naturomics/CapsNet-Tensorflow/blob/master/ # dist_version/distributed_train.py splits_x = tf.split(axis=0, num_or_size_splits=FLAGS.num_gpus, value=batch_x) splits_labels = tf.split(axis=0, num_or_size_splits=FLAGS.num_gpus, value=batch_labels) #-------------------------------------------------------------------------- # MULTI GPU - TRAIN #-------------------------------------------------------------------------- # Calculate the gradients for each model tower tower_grads = [] tower_losses = [] tower_logits = [] reuse_variables = None for i in range(FLAGS.num_gpus): with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % i) as scope: logger.info('TOWER %d' % i) #with slim.arg_scope([slim.model_variable, slim.variable], # device='/cpu:0'): with slim.arg_scope([slim.variable], device='/cpu:0'): loss, logits = tower_fn( build_arch, splits_x[i], splits_labels[i], scope, num_classes, reuse_variables=reuse_variables, is_train=True) # Don't reuse variable for first GPU, but do reuse for others reuse_variables = True # Compute gradients for one GPU grads = opt.compute_gradients(loss) # Keep track of the gradients across all towers. tower_grads.append(grads) # Keep track of losses and logits across for each tower tower_logits.append(logits) tower_losses.append(loss) # Loss for each tower tf.summary.scalar("loss", loss) # We must calculate the mean of each gradient. Note that this is the # synchronization point across all towers. grad = average_gradients(tower_grads) # See: https://stackoverflow.com/questions/40701712/how-to-check-nan-in- # gradients-in-tensorflow-when-updating grad_check = ([ tf.check_numerics(g, message='Gradient NaN Found!') for g, _ in grad if g is not None ] + [tf.check_numerics(loss, message='Loss NaN Found')]) # Apply the gradients to adjust the shared variables with tf.control_dependencies(grad_check): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op = opt.apply_gradients(grad, global_step=global_step) # Calculate mean loss loss = tf.reduce_mean(tower_losses) # Calculate accuracy logits = tf.concat(tower_logits, axis=0) acc = met.accuracy(logits, batch_labels) # Prepare predictions and one-hot labels probs = tf.nn.softmax(logits=logits) labels_oh = tf.one_hot(batch_labels, num_classes) # Group metrics together # See: https://cs230-stanford.github.io/tensorflow-model.html trn_metrics = { 'loss': loss, 'labels': batch_labels, 'labels_oh': labels_oh, 'logits': logits, 'probs': probs, 'acc': acc, } # Reset and read operations for streaming metrics go here trn_reset = {} trn_read = {} # Logging tf.summary.scalar('batch_loss', loss) tf.summary.scalar('batch_acc', acc) # Set Saver # AG 26/09/2018: Save all variables including Adam so that we can continue # training from where we left off # max_to_keep=None should keep all checkpoints saver = tf.train.Saver(tf.global_variables(), max_to_keep=None) # Display number of parameters train_params = np.sum([ np.prod(v.get_shape().as_list()) for v in tf.trainable_variables() ]).astype(np.int32) logger.info('Trainable Parameters: {}'.format(train_params)) # Set summary op trn_summary = tf.summary.merge_all() #---------------------------------------------------------------------------- # GRAPH - TRAINING SET ACCURACY #---------------------------------------------------------------------------- logger.info('BUILD TRAINING SET ACCURACY GRAPH') g_trn_acc = tf.Graph() with g_trn_acc.as_default(): # Get global_step global_step = tf.train.get_or_create_global_step() # Get data input_dict = create_inputs_train_wholeset() batch_x = input_dict['image'] batch_labels = input_dict['label'] # AG 10/12/2018: Split batch for multi gpu implementation # Each split is of size FLAGS.batch_size / FLAGS.num_gpus # See: https://github.com/naturomics/CapsNet- # Tensorflow/blob/master/dist_version/distributed_train.py splits_x = tf.split(axis=0, num_or_size_splits=FLAGS.num_gpus, value=batch_x) splits_labels = tf.split(axis=0, num_or_size_splits=FLAGS.num_gpus, value=batch_labels) #-------------------------------------------------------------------------- # MULTI GPU - TRAINING SET ACCURACY #-------------------------------------------------------------------------- # Calculate the logits for each model tower tower_logits = [] reuse_variables = None for i in range(FLAGS.num_gpus): with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % i) as scope: with slim.arg_scope([slim.variable], device='/cpu:0'): loss, logits = tower_fn( build_arch, splits_x[i], splits_labels[i], scope, num_classes, reuse_variables=reuse_variables, is_train=False) # Don't reuse variable for first GPU, but do reuse for others reuse_variables = True # Keep track of losses and logits across for each tower tower_logits.append(logits) # Loss for each tower tf.summary.histogram("train_set_logits", logits) # Combine logits from all towers logits = tf.concat(tower_logits, axis=0) # Calculate metrics train_set_loss = mod.spread_loss(logits, batch_labels) train_set_acc = met.accuracy(logits, batch_labels) # Prepare predictions and one-hot labels train_set_probs = tf.nn.softmax(logits=logits) train_set_labels_oh = tf.one_hot(batch_labels, num_classes) # Group metrics together # See: https://cs230-stanford.github.io/tensorflow-model.html train_set_metrics = { 'loss': train_set_loss, 'labels': batch_labels, 'labels_oh': train_set_labels_oh, 'logits': logits, 'probs': train_set_probs, 'acc': train_set_acc, } # Reset and read operations for streaming metrics go here train_set_reset = {} train_set_read = {} saver = tf.train.Saver(max_to_keep=None) tf.summary.scalar("train_set_loss", train_set_loss) tf.summary.scalar("train_set_acc", train_set_acc) trn_acc_summary = tf.summary.merge_all() if dataset_size_val > 0: #---------------------------------------------------------------------------- # GRAPH - VALIDATION #---------------------------------------------------------------------------- logger.info('BUILD VALIDATION GRAPH') g_val = tf.Graph() with g_val.as_default(): # Get global_step global_step = tf.train.get_or_create_global_step() num_batches_val = int(dataset_size_val / FLAGS.batch_size) # Get data input_dict = create_inputs_val() batch_x = input_dict['image'] batch_labels = input_dict['label'] # AG 10/12/2018: Split batch for multi gpu implementation # Each split is of size FLAGS.batch_size / FLAGS.num_gpus # See: https://github.com/naturomics/CapsNet- # Tensorflow/blob/master/dist_version/distributed_train.py splits_x = tf.split(axis=0, num_or_size_splits=FLAGS.num_gpus, value=batch_x) splits_labels = tf.split(axis=0, num_or_size_splits=FLAGS.num_gpus, value=batch_labels) #-------------------------------------------------------------------------- # MULTI GPU - VALIDATE #-------------------------------------------------------------------------- # Calculate the logits for each model tower tower_logits = [] reuse_variables = None for i in range(FLAGS.num_gpus): with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % i) as scope: with slim.arg_scope([slim.variable], device='/cpu:0'): loss, logits = tower_fn( build_arch, splits_x[i], splits_labels[i], scope, num_classes, reuse_variables=reuse_variables, is_train=False) # Don't reuse variable for first GPU, but do reuse for others reuse_variables = True # Keep track of losses and logits across for each tower tower_logits.append(logits) # Loss for each tower tf.summary.histogram("val_logits", logits) # Combine logits from all towers logits = tf.concat(tower_logits, axis=0) # Calculate metrics val_loss = mod.spread_loss(logits, batch_labels) val_acc = met.accuracy(logits, batch_labels) # Prepare predictions and one-hot labels val_probs = tf.nn.softmax(logits=logits) val_labels_oh = tf.one_hot(batch_labels, num_classes) # Group metrics together # See: https://cs230-stanford.github.io/tensorflow-model.html val_metrics = { 'loss': val_loss, 'labels': batch_labels, 'labels_oh': val_labels_oh, 'logits': logits, 'probs': val_probs, 'acc': val_acc, } # Reset and read operations for streaming metrics go here val_reset = {} val_read = {} tf.summary.scalar("val_loss", val_loss) tf.summary.scalar("val_acc", val_acc) # Saver saver = tf.train.Saver(max_to_keep=None) # Set summary op val_summary = tf.summary.merge_all() #**************************************************************************** # 2. SESSIONS #**************************************************************************** #----- SESSION TRAIN -----# # Session settings #sess_train = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, # log_device_placement=False), # graph=g_train) # Perry: added in for RTX 2070 incompatibility workaround config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) config.gpu_options.allow_growth = True sess_train = tf.Session(config=config, graph=g_train) # Debugger # AG 05/06/2018: Debugging using either command line or TensorBoard if FLAGS.debugger is not None: # sess = tf_debug.LocalCLIDebugWrapperSession(sess) sess_train = tf_debug.TensorBoardDebugWrapperSession( sess_train, FLAGS.debugger) with g_train.as_default(): sess_train.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) # Restore previous checkpoint # AG 26/09/2018: where should this go??? if FLAGS.load_dir is not None: prev_step = load_training(saver, sess_train, FLAGS.load_dir) else: prev_step = 0 # Create summary writer, and write the train graph summary_writer = tf.summary.FileWriter(train_summary_dir, graph=sess_train.graph) #----- SESSION TRAIN SET ACCURACY -----# #sess_val = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, # log_device_placement=False), # graph=g_val) # Perry: added in for RTX 2070 incompatibility workaround config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) config.gpu_options.allow_growth = True sess_train_acc = tf.Session(config=config, graph=g_trn_acc) with g_trn_acc.as_default(): sess_train_acc.run([ tf.local_variables_initializer(), tf.global_variables_initializer() ]) if dataset_size_val > 0: #----- SESSION VALIDATION -----# #sess_val = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, # log_device_placement=False), # graph=g_val) # Perry: added in for RTX 2070 incompatibility workaround config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) config.gpu_options.allow_growth = True sess_val = tf.Session(config=config, graph=g_val) with g_val.as_default(): sess_val.run([ tf.local_variables_initializer(), tf.global_variables_initializer() ]) #**************************************************************************** # 3. MAIN LOOP #**************************************************************************** SUMMARY_FREQ = 100 SAVE_MODEL_FREQ = num_batches_per_epoch # 500 VAL_FREQ = num_batches_per_epoch # 500 PROFILE_FREQ = 5 for step in range(prev_step, FLAGS.epoch * num_batches_per_epoch + 1): #for step in range(0,3): # AG 23/05/2018: limit number of iterations for testing # for step in range(100): epoch_decimal = step / num_batches_per_epoch epoch = int(np.floor(epoch_decimal)) # TF queue would pop batch until no file try: # TRAIN with g_train.as_default(): # With profiling if (FLAGS.profile is True) and ((step % PROFILE_FREQ) == 0): logger.info("Train with Profiling") run_options = tf.RunOptions( trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() # Without profiling else: run_options = None run_metadata = None # Reset streaming metrics if step % (num_batches_per_epoch / 4) == 1: logger.info("Reset streaming metrics") sess_train.run([trn_reset]) # MAIN RUN tic = time.time() train_op_v, trn_metrics_v, trn_summary_v = sess_train.run( [train_op, trn_metrics, trn_summary], options=run_options, run_metadata=run_metadata) toc = time.time() # Read streaming metrics trn_read_v = sess_train.run(trn_read) # Write summary for profiling if run_options is not None: summary_writer.add_run_metadata( run_metadata, 'epoch{:f}'.format(epoch_decimal)) # Logging #logger.info('TRN' # + ' e-{:d}'.format(epoch) # + ' stp-{:d}'.format(step) # + ' {:.2f}s'.format(toc - tic) # + ' loss: {:.4f}'.format(trn_metrics_v['loss']) # + ' acc: {:.2f}%'.format(trn_metrics_v['acc']*100) # ) except KeyboardInterrupt: sess_train.close() sess_val.close() sys.exit() except tf.errors.InvalidArgumentError as e: logger.warning('%d iteration contains NaN gradients. Discard.' % step) logger.error(str(e)) continue else: # WRITE SUMMARY if (step % SUMMARY_FREQ) == 0: logger.info("Write Train Summary") with g_train.as_default(): # Summaries from graph summary_writer.add_summary(trn_summary_v, step) # SAVE MODEL if (step % SAVE_MODEL_FREQ) == 0: logger.info("Save Model") with g_train.as_default(): train_checkpoint_dir = train_dir + '/checkpoint' if not os.path.exists(train_checkpoint_dir): os.makedirs(train_checkpoint_dir) # Save ckpt from train session ckpt_path = os.path.join(train_checkpoint_dir, 'model.ckpt' + str(epoch)) saver.save(sess_train, ckpt_path, global_step=step) if (step % VAL_FREQ) == 0: # calculate metrics every epoch with g_trn_acc.as_default(): logger.info("Start Train Set Accuracy") # Restore ckpt to val session latest_ckpt = tf.train.latest_checkpoint( train_checkpoint_dir) saver.restore(sess_train_acc, latest_ckpt) # Reset accumulators accuracy_sum = 0 loss_sum = 0 sess_train_acc.run(train_set_reset) for i in range(num_batches_per_epoch): train_set_metrics_v, train_set_summary_str_v = sess_train_acc.run( [train_set_metrics, trn_acc_summary]) # Update accuracy_sum += train_set_metrics_v['acc'] loss_sum += train_set_metrics_v['loss'] # Read trn_read_v = sess_train_acc.run(val_read) # Get checkpoint number ckpt_num = re.split('-', latest_ckpt)[-1] # Average across batches ave_acc = accuracy_sum / num_batches_per_epoch ave_loss = loss_sum / num_batches_per_epoch logger.info('TRN ckpt-{}'.format(ckpt_num) + ' avg_acc: {:.2f}%'.format(ave_acc * 100) + ' avg_loss: {:.4f}'.format(ave_loss)) logger.info("Write Train Summary") summary_train = tf.Summary() summary_train.value.add(tag="trn_acc", simple_value=ave_acc) summary_train.value.add(tag="trn_loss", simple_value=ave_loss) summary_writer.add_summary(summary_train, epoch) if dataset_size_val > 0: #----- Validation -----# with g_val.as_default(): logger.info("Start Validation") # Restore ckpt to val session latest_ckpt = tf.train.latest_checkpoint( train_checkpoint_dir) saver.restore(sess_val, latest_ckpt) # Reset accumulators accuracy_sum = 0 loss_sum = 0 sess_val.run(val_reset) for i in range(num_batches_val): val_metrics_v, val_summary_str_v = sess_val.run( [val_metrics, val_summary]) # Update accuracy_sum += val_metrics_v['acc'] loss_sum += val_metrics_v['loss'] # Read val_read_v = sess_val.run(val_read) # Get checkpoint number ckpt_num = re.split('-', latest_ckpt)[-1] # Logging #logger.info('VAL ckpt-{}'.format(ckpt_num) # + ' bch-{:d}'.format(i) # + ' cum_acc: {:.2f}%'.format(accuracy_sum/(i+1)*100) # + ' cum_loss: {:.4f}'.format(loss_sum/(i+1)) # ) # Average across batches ave_acc = accuracy_sum / num_batches_val ave_loss = loss_sum / num_batches_val logger.info('VAL ckpt-{}'.format(ckpt_num) + ' avg_acc: {:.2f}%'.format(ave_acc * 100) + ' avg_loss: {:.4f}'.format(ave_loss)) logger.info("Write Val Summary") summary_val = tf.Summary() summary_val.value.add(tag="val_acc", simple_value=ave_acc) summary_val.value.add(tag="val_loss", simple_value=ave_loss) summary_writer.add_summary(summary_val, epoch) # Close (main loop) sess_train.close() sess_val.close() sys.exit()
training_steps = config.training_steps start_train = time.perf_counter() start_step = time.perf_counter() for epoch in range(epochs_finished, epochs): for step, (input_data, labels) in enumerate(train_loader): current_step = step + epoch * steps_per_epoch start_step = time.perf_counter() losses, logits = train_model(input_data, labels) step_length = time.perf_counter() - start_step scheduler.step() train_model.setOptimizer(optimizer) mean_loss = losses.mean().item() preds = torch.argmax(logits, dim=-1) acc = accuracy(preds, labels) step_throughput = config.samples_per_step / step_length msg = ("Epoch: {:.2f}/{} " "Step: {}/{} " "Lr: {:.6f} " "Loss: {:.3f} " "Acc: {:.3f} " "Throughput: {:.2f} samples/sec").format( epoch, epochs, current_step, training_steps, scheduler.get_last_lr()[0], mean_loss, acc, step_throughput) logger.info(msg) if config.wandb: wandb.log({ "LR": scheduler.get_last_lr()[0],