def computenlogStud(self, student_out, teacher_out, studentgrad_params, teachergrad_params, y_discriminator, target, out, isCorrect): teacherprec1 = precision(teacher_out.data, target) studentprec1 = precision(student_out.data, target) discadversarialLoss = self.opt.wdiscAdv * self.advCriterion( out, Variable(isCorrect)) disccrossentropyLoss = self.opt.wdiscClassify * self.discclassifyCriterion( y_discriminator, Variable(target)) studreconstructionLoss = self.opt.wstudSim * self.similarityCriterion( student_out, teacher_out.detach()) studderivativeLoss = self.opt.wstudDeriv * self.derivativeCriterion( studentgrad_params, teachergrad_params.detach()) studtotalLoss = studreconstructionLoss + studderivativeLoss + discadversarialLoss + disccrossentropyLoss self.teachertop1.update(teacherprec1[0], target.size(0)) self.studenttop1.update(studentprec1[0], target.size(0)) self.studadversariallossLog.update(discadversarialLoss.data[0], target.size(0)) self.studcrossentropylossLog.update(disccrossentropyLoss.data[0], target.size(0)) self.studreconstructionlossLog.update(studreconstructionLoss.data[0], target.size(0)) self.studderivativelossLog.update(studderivativeLoss.data[0], target.size(0)) self.studtotallossLog.update(studtotalLoss.data[0], target.size(0)) return studtotalLoss
def validate(self, valloader, epoch, opt): self.teacher.eval() self.student.eval() self.discriminator.eval() self.teachertop1.reset() self.studenttop1.reset() self.discriminatortop1.reset() self.data_time.reset() self.batch_time.reset() for i, (input, target) in enumerate(valloader, 0): end = time.time() if opt.cuda: input = input.cuda(async=True) target = target.cuda(async=True) input, target_var = Variable(input, volatile=True), Variable( target, volatile=True) self.data_time.update(time.time() - end) teacher_out = self.teacher(input) student_out = self.student(input) discriminator_out = self.discriminator(student_out) self.batch_time.update(time.time() - end) teacherprec1, teacherprec5 = precision(teacher_out.data, target, topk=(1, 5)) studentprec1, studentprec5 = precision(student_out.data, target, topk=(1, 5)) discriminatorprec1, discriminatorprec5 = precision( discriminator_out.data, target, topk=(1, 5)) self.teachertop1.update(teacherprec1[0], input.size(0)) self.studenttop1.update(studentprec1[0], input.size(0)) self.discriminatortop1.update(discriminatorprec1[0], input.size(0)) if opt.tensorboard: self.logger.scalar_summary('Teacher Accuracy', self.teachertop1.avg, epoch) self.logger.scalar_summary('Student Accuracy', self.studenttop1.avg, epoch) self.logger.scalar_summary('Discriminator Accuracy', self.discriminatortop1.avg, epoch) print('Val: [{0}]\t' 'Time {batch_time.sum:.3f}\t' 'Data {data_time.sum:.3f}\t' 'DiscriminatorPrec@1 {discriminatortop1.avg:.4f}\t' 'TeacherPrec@1 {teachertop1.avg:.4f}\t' 'StudentPrec@1 {studenttop1.avg:.4f}\t'.format( epoch, batch_time=self.batch_time, data_time=self.data_time, discriminatortop1=self.discriminatortop1, teachertop1=self.teachertop1, studenttop1=self.studenttop1)) return self.studenttop1.avg
def simple_lstm(x_train, y_train, x_test, y_test, dataset="default"): model = train(x_train, y_train, dataset=dataset) print(x_train.shape) print(x_test.shape) # print(model.summary()) pred_x_train = model.predict(x_train) pred_x_test = model.predict(x_test) precision_test = precision(np.argmax(y_test, axis=1), np.argmax(pred_x_test, axis=1), labels=(0, 1, 2)) recall_test = recall(np.argmax(y_test, axis=1), np.argmax(pred_x_test, axis=1), labels=(0, 1, 2)) f1_test = f1(np.argmax(y_test, axis=1), np.argmax(pred_x_test, axis=1), labels=(0, 1, 2)) # print( "Accuracy training accuracy = ", accuracy(np.argmax(y_train, axis=1), np.argmax(pred_x_train, axis=1))) print("Accuracy testing accuracy =", accuracy(np.argmax(y_test, axis=1), np.argmax(pred_x_test, axis=1)), "\n") # print("Precision (test data) =", precision_test) print("Recall (test data) =", recall_test) print("F1 (test data) =", f1_test, "\n")
def evaluate(self, Xt, Xc, y): """ Evaluates the model's accuracy, precision, recall, F1 score and loss value on a dataset. :param Xt: Matrix containing title input. :param Xc: Matrix containing content input. :param y: Label vector. :return: Accuracy, lists containing each topic's precision, recall, F1 score followed by the macro-average across topics, and the model's loss value. """ probs = self.model.predict([Xt, Xc], verbose=1, batch_size=100) preds = np.argmax(probs, axis=1) true = np.argmax(y, 1) acc = np.sum(np.equal(preds, true)) / np.size(true, 0) p, r, f1 = [], [], [] for i in range(len(self.classes)): p.append(utils.precision(preds, true, i)) r.append(utils.recall(preds, true, i)) f1.append(utils.f1_score(p[i], r[i])) p2 = [x for x in p if x is not None] r2 = [x for x in r if x is not None] f2 = [x for x in f1 if x is not None] p.append(np.mean(p2)) r.append(np.mean(r2)) f1.append(np.mean(f2)) print('\nCalculating loss...') self.compile() loss = self.model.evaluate([Xt, Xc], y, batch_size=100)[0] return acc, p, r, f1, loss
def vectorized_rf(x_train, y_train, x_test, y_test, checktrain=True, ngram_range=(1, 1), vector_type="count", dataset="default"): vectorizer = CountVectorizer(tokenizer=tokenize, ngram_range=ngram_range) if vector_type=="count" \ else TfidfVectorizer(tokenizer=tokenize, ngram_range=ngram_range) vectorized_x_train = vectorizer.fit_transform(x_train) vectorized_x_test = vectorizer.transform(x_test) model = train(vectorized_x_train, y_train, checktrain, ngram_range, vector_type=vector_type, dataset=dataset) pred_x_train = predict(model, vectorized_x_train) pred_x_test = predict(model, vectorized_x_test) precision_test = precision(y_test, pred_x_test) recall_test = recall(y_test, pred_x_test) f1_test = f1(y_test, pred_x_test) print("Accuracy training accuracy (" + dataset, vector_type, " vectorized joint data) =", accuracy(y_train, pred_x_train)) print("Accuracy testing accuracy (" + dataset, vector_type, "vectorized joint data) =", accuracy(y_test, pred_x_test), "\n") print("Precision (" + dataset, vector_type + " vectorized test data) =", precision_test) print("Recall (" + dataset, vector_type + " test data) =", recall_test) print("F1 (" + dataset, vector_type + " test data) =", f1_test, "\n")
def train(epochs): print("Train start") writer = tensorboard.SummaryWriter(log_dir='./log', comment='Train loop') for ep in range(1, epochs + 1): epoch_loss, epoch_accuracy, epoch_precision = 0, 0, 0 epoch_f1, idx = 0, 0 for idx, (inp, label) in enumerate(train_loader): optimizer.zero_grad() op = model(inp) loss = criterion(op, label) loss.backward() optimizer.step() epoch_loss += loss.item() epoch_accuracy += accuracy(op, label) epoch_precision += precision(op, label) epoch_f1 += f1(op, label) writer.add_scalars( 'Training', { 'Accuracy': epoch_accuracy / idx, 'Precision': epoch_precision / idx, 'F1': epoch_f1 / idx }, ep) writer.add_scalars('Loss', {'Training': epoch_loss / idx}, ep) writer.close() torch.save(model.state_dict(), PATH) print("Done training")
def calculate_p(self, lines, traffic, sources): """ engset, calculate blocking rate :param lines: number of servers :param traffic: the traffic in Erlangs :param sources: the number of points generating the traffic :return: blocking rate """ block = 0 right = self.basic_engset(lines, traffic, block, sources) while block != right: block += .0001 right = precision(right, 4) block = precision(block, 4) return block
def validate(self): ''' method to validate model on current epoch ''' # meters losses = AverageMeter() accuracy = AverageMeter() # switch to evaluation mode and inference the model self.model.eval() loop = tqdm(enumerate(self.val_loader), total=len(self.val_loader), leave=False) criterion = self.criterion[ 0] if self.config.multi_task_learning else self.criterion for i, (input_, target) in loop: if i == self.config.test_steps: break input_ = input_.to(self.device) target = target.to(self.device) if len(target.shape) > 1: target = target[:, 0].reshape(-1) # computing output and loss with torch.no_grad(): features = self.model(input_) if self.data_parallel: model1 = self.model.module else: model1 = self.model output = model1.make_logits(features, all=True) if isinstance(output, tuple): output = output[0] new_target = F.one_hot(target, num_classes=2) loss = criterion(output, new_target) # measure accuracy and record loss acc = precision(output, target, s=self.config.loss.amsoftmax.s) losses.update(loss.item(), input_.size(0)) accuracy.update(acc, input_.size(0)) # update progress bar loop.set_postfix(loss=loss.item(), avr_loss=losses.avg, acc=acc, avr_acc=accuracy.avg) print( f'val accuracy on epoch: {round(accuracy.avg, 3)}, loss on epoch:{round(losses.avg, 3)}' ) # write val in writer self.writer.add_scalar('Val/loss', losses.avg, global_step=self.val_step) self.writer.add_scalar('Val/accuracy', accuracy.avg, global_step=self.val_step) self.val_step += 1 return accuracy.avg
def validate(self, valloader, epoch, opt): self.model.eval() self.losses.reset() self.top1.reset() self.top5.reset() self.data_time.reset() self.batch_time.reset() end = time.time() for i, (input, target) in enumerate(valloader, 0): if opt.tenCrop: input = input.view(input.size(0)*input.size(1), input.size(2), input.size(3), input.size(4)) if opt.cuda: input = input.cuda(async=True) target = target.cuda(async=True) input_var = Variable(input, volatile=True) target_var = Variable(target, volatile=True) self.data_time.update(time.time() - end) output = self.model(input_var) if opt.tenCrop: # print("Doing 10crop") output = output.view(output.size(0) // 10, 10, output.size(1)).sum(1).squeeze(1).div(10.0) loss = self.criterion(output, target_var) prec1, prec5 = precision(output.data, target, topk=(1,5)) self.losses.update(loss.data[0], input.size(0)) self.top1.update(prec1[0], input.size(0)) self.top5.update(prec5[0], input.size(0)) # measure elapsed time self.batch_time.update(time.time() - end) end = time.time() # log to TensorBoard if opt.tensorboard: self.logger.scalar_summary('val_loss', self.losses.avg, epoch) self.logger.scalar_summary('val_acc', self.top1.avg, epoch) print('Val: [{0}]\t' 'Time {batch_time.sum:.3f}\t' 'Data {data_time.sum:.3f}\t' 'Loss {loss.avg:.3f}\t' 'Prec@1 {top1.avg:.4f}\t' 'Prec@5 {top5.avg:.4f}\t'.format( epoch, batch_time=self.batch_time, data_time= self.data_time, loss=self.losses, top1=self.top1, top5=self.top5)) return self.top1.avg
def test_F2Score(): print(true_positive(actual, estimated) == 2) print(false_positive(actual, estimated) == 3) print(false_negative(actual, estimated) == 1) expected_prec = 2 / (2 + 3) print(precision(actual, estimated) == expected_prec) expected_recall = 2 / (2 + 1) print(recall(actual, estimated) == expected_recall) print( F1score(actual, estimated) == 2 * expected_recall * expected_prec / (expected_recall + expected_prec))
def generate_results(args, ignore_last=False): ''' Collects precision information on a given embedding and original graph information. ''' graph = np.load('%s/../graph/%s_hidden.npy' % (c, args.graph)) hidden_links = np.load('%s/../graph/%s_testlinks.npy' % (c, args.hidden_links)) emb = np.load('%s/../emb/%s.npy' % (c, args.embedding)) # file_path = '%s/../emb/%s.mat' % (c, args.embedding) # emb = scipy.io.loadmat(file_path)['embedding'] if ignore_last: # ignore the last node in embedding and in graph (for arxiv) m = len(emb) - 1 emb = emb[:m] n = len(graph) - 1 graph = np.delete(graph, n, 0) graph = np.delete(graph, n, 1) test_mat = np.zeros((len(graph), len(graph))) for row in hidden_links: x = row[0] y = row[1] graph[x][y] = graph[y][x] = 1 test_mat[x][y] = test_mat[y][x] = 1 np.fill_diagonal(graph, 0) sim_scores = similarity_scores(emb, method='dot') k_ = [2**j for j in range(0, 14)] # use for arxiv # k_ = [ 2**j for j in range(0,15) ] # use for blog # k_ = [ 2**j for j in range(0,19,2) ] # use for flickr # uncomment these lines for binned results # ranges = [range(1,2), range(2,3), range(3,5), range(5,10), range(10,26)] # use for arxiv # ranges = [range(1,2), range(2,4), range(4,8), range(8,19), range(19,839)] # use for blog # ranges = [range(1,3), range(3,8), range(8,18), range(18,49), range(49,1178)] # use for flickr # for n_links in ranges: # precisions = [precision_rm(k, sim_scores, graph, test_mat, n_links) for k in k_] precisions = [precision(k, sim_scores, graph) for k in k_] str_precisions = [("%f" % x) for x in precisions] print '\t'.join(str_precisions) return
def test(): test_loss = 0 test_accuracy = 0 test_precision = 0 test_f1 = 0 idx = 0 with torch.no_grad(): for idx, (inp, label) in enumerate(test_loader): op = model(inp) loss = criterion(op, label) test_loss += loss.item() test_accuracy += accuracy(op, label) test_precision += precision(op, label) test_f1 += f1(op, label) print(f"Test loss: {test_loss / idx}") print(f"Test accuracy: {test_accuracy / idx}") print(f"Test precision: {test_precision / idx}") print(f"Test f1: {test_f1 / idx}")
def computenlogDisc(self, y_discriminator, target, out, isCorrect): discriminatortop1 = precision(y_discriminator.data, target) #discriminatorisreal = precision(out.data, isCorrect) discadversarialLoss = self.opt.wdiscAdv * self.advCriterion( out, Variable(isCorrect)) disccrossentropyLoss = self.opt.wdiscClassify * self.discclassifyCriterion( y_discriminator, Variable(target)) disctotalLoss = discadversarialLoss + disccrossentropyLoss self.disctop1.update(discriminatortop1[0], target.size(0)) #self.discisreal.update(discriminatorisreal[0], target.size(0)) self.discadversariallossLog.update(discadversarialLoss.data[0], target.size(0)) self.disccrossentropylossLog.update(disccrossentropyLoss.data[0], target.size(0)) self.disctotallossLog.update(disctotalLoss.data[0], target.size(0)) return disctotalLoss
def kfold(classification_algorithm, k): res = {"accuracy": 0, "precision": 0, "recall": 0, "f1": 0} for i in range(1, k + 1): validation = utils.load_train(i) validation = validation["plus"] + validation["minus"] train = {"plus": [], "minus": []} for j in range(1, k + 1): if j != i: extension = utils.load_train(j) train["plus"].extend(extension["plus"]) train["minus"].extend(extension["minus"]) classification = classification_algorithm(train, validation) res["accuracy"] += utils.accuracy(classification) res["precision"] += utils.precision(classification) res["recall"] += utils.recall(classification) res["f1"] += utils.F1_score(classification) for k in res: res[k] /= k print res return res
def main(path_detection, path_truth, mode, iou): print("Path to detections:", path_detection) print("Path to ground truth:", path_truth) print("Chosen mode:", mode) print("IoU-threshold:", iou) # acquire detections and false_negatives for each class # detections: {class: [(1,0.95), (0,0.87), (0,0.6), (1,0.67), ...., (0,0.9), (1,0.7)]} # false_negatives: int detections, false_negatives = loop_detection_files(path_detection, path_truth, iou, mode) # store APs for calculating mAP average_precisions = [] # analyse results of each class for classification in detections.keys(): classification_detections = [ d[0] for d in sorted(detections.get(classification), key=lambda tup: tup[1], reverse=True) ] # print precision, recall, AP and show precision-recall curve print("-------------") print("CLASS", classification) print("Precision:", precision(classification_detections)) print( "Recall:", recall(classification_detections, false_negatives.get(classification))) average_precision, smoothed_precision, recall_samples = AP( classification, classification_detections, false_negatives.get(classification)) average_precisions.append(average_precision) print("AP:", average_precision) visualise_results(classification, classification_detections, false_negatives.get(classification), smoothed_precision, recall_samples) print("-------------") print("********") print("mAP:", np.mean(average_precisions)) print("********")
def compute_test(): model.eval() output = model(features, adjs) loss_test = F.nll_loss(output[idx_test], labels[idx_test]) acc_test = accuracy(output[idx_test], labels[idx_test]) # mac = macro_f1(output[idx_test], labels[idx_test]) # mac_pre = macro_precision(output[idx_test], labels[idx_test]) # mac_rec = macro_recall(output[idx_test], labels[idx_test]) # print("Test set results:", # "loss= {:.4f}".format(loss_test.item()), # "accuracy= {:.4f}".format(acc_test.item()), # "macro_f1= {:.4f}".format(mac), # "macro_precision= {:.4f}".format(mac_pre), # "macro_recall= {:.4f}".format(mac_rec)) mac = f1(output[idx_test], labels[idx_test]) mac_pre = precision(output[idx_test], labels[idx_test]) mac_rec = recall(output[idx_test], labels[idx_test]) print("Test set results:", "loss= {:.4f}".format(loss_test.item()), "accuracy= {:.4f}".format(acc_test.item()), "macro_f1= {:.4f}".format(mac), "macro_precision= {:.4f}".format(mac_pre), "macro_recall= {:.4f}".format(mac_rec))
x2 = D.get_data('Data1/Class2.txt') x3 = D.get_data('Data1/Class3.txt') if not os.path.isdir("plots_multi"): os.mkdir("plots_multi") w1 = [200.0, -200.0, 200.0] w2 = [-200.0, 200.0, 200.0] w3 = [200.0, 200.0, -200.0] perceptron(w1, x1, x2, x3, 0.5, 100, 1) perceptron(w2, x2, x3, x1, 0.5, 100, 2) perceptron(w3, x3, x1, x2, 0.5, 100, 3) plt.figure() decision_boundary(plt, w1, w2, w3) conf_mat = get_conf(x1, x2, x3, w1, w2, w3, plt) plt.savefig('plots_multi/decision_boundary.png') print(conf_mat) print("Accuracy: ", U.accuracy(conf_mat)) print("Precision for class 1: ", U.precision(conf_mat, 0)) print("Precision for class 2: ", U.precision(conf_mat, 1)) print("Precision for class 3: ", U.precision(conf_mat, 2)) print("Recall for class 1: ", U.recall(conf_mat, 0)) print("Recall for class 2: ", U.recall(conf_mat, 1)) print("Recall for class 3: ", U.recall(conf_mat, 2)) print("F-Score for class 1: ", U.f_score(conf_mat, 0)) print("F-Score for class 2: ", U.f_score(conf_mat, 1)) print("F-Score for class 3: ", U.f_score(conf_mat, 2))
# para evitar a dependência entre as variáveis. Isso pode ser feito através do parâmetro "drop_first = True". X = pd.get_dummies(df.iloc[:, :-1], prefix=['city', 'sex'], drop_first=True).values y = df.iloc[:, 4].values # Aplicando a normalização/escalonamento das features X = feature_scaling(X) # Definindo os valores da matriz de confusão: tp, fp, fn, tn = [1, 1, 1, 2] # Calculando os valores da acurácia accuracy(tp, fp, fn, tn) # Calculando os valores da precisão precision(tp, fp) # Calculando os valores do recall recall(tp, fn) # Calculando os valores do informedness informedness(tp, fp, fn, tn) # Calculando os valores do markdness markdness(tp, fp, fn, tn) # Definindo os valores para calculo do ROC-AUC # Valores reais (ground truth) y_true = np.array([1, 1, 2, 2]) #Probabilidade de estimar as classes positivas:
best_test = test_score acc_score[train_times] = round(best_test, 4) save_model_path = os.path.join( save_time_fold, preprocess_path.split('/')[-1] + '_times_' + str(train_times) + '_' + str(round(best_test, 4)) + '.pth.tar') torch.save(model.state_dict(), save_model_path) save_predict_target_path = os.path.join( save_time_fold, preprocess_path.split('/')[-1] + '_times_' + str(train_times) + '_' + str(round(best_test, 4)) + '.txt') predict_target = torch.cat((predicts, targets), dim=0).detach().cpu().numpy() np.savetxt(save_predict_target_path, predict_target) precision_score[train_times] = round(precision(predicts, targets), 4) recall_score[train_times] = round(recall(predicts, targets), 4) specificity_score[train_times] = round( specificity(predicts, targets), 4) mcc_score[train_times] = round(mcc(predicts, targets), 4) auc_score = round(auc(predicts, targets), 4) aupr_score = round(aupr(predicts, targets), 4) print('Epoch: {:04d}'.format(epoch + 1), 'Train_times:', train_times) print( "*****************test_score {:.4f} best_socre {:.4f}****************" .format(test_score, best_test)) print("All Test Score:", acc_score) print(args.dataset, " Optimization Finished!") print("Total time elapsed: {:.4f}s".format(time.time() - t_total)) print("acc Score:", acc_score)
cv2.COLOR_BGR2RGB)[:args.crop_height, :args.crop_width]), axis=0) / 255.0 gt = cv2.imread(val_output_names[ind], -1)[:args.crop_height, :args.crop_width] st = time.time() output_image = sess.run(network, feed_dict={input: input_image}) output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) out_vis_image = helpers.colour_code_segmentation(output_image) accuracy = utils.compute_avg_accuracy(output_image, gt) class_accuracies = utils.compute_class_accuracies( output_image, gt, num_classes) prec = utils.precision(output_image, gt) rec = utils.recall(output_image, gt) f1 = utils.f1score(output_image, gt) iou = utils.compute_mean_iou(output_image, gt) file_name = utils.filepath_to_name(val_input_names[ind]) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec)
theta = np.zeros((X.shape[1], classes)) l2_lambda = 0 # Calculate optimal theta using Newton Conjugate Gradient (closest alternative to fminunc) X_train, X_test, y_train, y_test = split_data(X, y) # stratified split theta = opt.minimize(fun=calculate_cost_and_gradient, x0=theta, method='TNC', args=(X_train, y_train, classes, l2_lambda), jac=True, options={'maxiter': 50}) theta.x = theta.x.reshape( X.shape[1], classes) # reshape theta into necessary dimensions # Print out the cost and calculated theta print('Testing cost: %f' % theta.fun) print('----------------------------------------------------') # separator # Calculate accuracy, precision and recall print('Accuracy (percentage of guessed samples): %f' % multiclass_accuracy(theta.x, X_test, y_test)) conf = confusion_matrix_c(y_test, softmax(X_test.dot(theta.x)), True) plot_confusion_matrix(conf, 'Confusion matrix (actual vs predicted values)') print('Precision (true positives / predicted positives): %f' % precision(conf)) print('Recall (true positives / actual positives): %f' % recall(conf)) plt.show()
gt = cv2.imread(val_output_names[ind], -1)[:args.crop_height, :args.crop_width] st = time.time() output_image = sess.run(network, feed_dict={input: input_image}) output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) out_eval_image = output_image[:, :, 0] out_vis_image = helpers.colour_code_segmentation(output_image) accuracy = utils.compute_avg_accuracy(out_eval_image, gt) class_accuracies = utils.compute_class_accuracies( out_eval_image, gt, num_classes) prec = utils.precision(out_eval_image, gt) rec = utils.recall(out_eval_image, gt) f1 = utils.f1score(out_eval_image, gt) iou = utils.compute_mean_iou(out_eval_image, gt) file_name = utils.filepath_to_name(val_input_names[ind]) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec)
def train(self, trainloader, epoch, opt): self.model.train() self.losses.reset() self.top1.reset() self.top5.reset() self.data_time.reset() self.batch_time.reset() end = time.time() for i, (input, target) in enumerate(trainloader, 0): if opt.cuda: input = input.cuda(async=True) target = target.cuda(async=True) input_var = Variable(input) target_var = Variable(target) self.data_time.update(time.time() - end) output = self.model(input_var) loss = self.criterion(output, target_var) #Broadcast based function. Adapted from:- https://github.com/pytorch/examples/blob/master/imagenet/main.py prec1, prec5 = precision(output.data, target, topk=(1,5)) self.optimizer.zero_grad() loss.backward() self.optimizer.step() self.optimizer.zero_grad() self.losses.update(loss.data[0], input.size(0)) self.top1.update(prec1[0], input.size(0)) self.top5.update(prec5[0], input.size(0)) # measure elapsed time self.batch_time.update(time.time() - end) end = time.time() if opt.verbose == True and i % opt.printfreq == 0: print('Train: [{0}][{1}/{2}]\t' 'Time {batch_time.avg:.3f} ({batch_time.sum:.3f})\t' 'Data {data_time.avg:.3f} ({data_time.sum:.3f})\t' 'Loss {loss.avg:.3f}\t' 'Prec@1 {top1.avg:.4f}\t' 'Prec@5 {top5.avg:.4f}'.format( epoch, i, len(trainloader), batch_time=self.batch_time, data_time=self.data_time, loss=self.losses, top1=self.top1, top5=self.top5)) # log to TensorBoard if opt.tensorboard: self.logger.scalar_summary('train_loss', self.losses.avg, epoch) self.logger.scalar_summary('train_acc', self.top1.avg, epoch) print('Train: [{0}]\t' 'Time {batch_time.sum:.3f}\t' 'Data {data_time.sum:.3f}\t' 'Loss {loss.avg:.3f}\t' 'Prec@1 {top1.avg:.4f}\t' 'Prec@5 {top5.avg:.4f}\t'.format( epoch, batch_time=self.batch_time, data_time= self.data_time, loss=self.losses, top1=self.top1, top5=self.top5))
for x, target in batch: x = x.to(device) target = target.to(device) y = classifier(x) loss = criterion(y, target) ll.append(loss.detach().item()) batch.set_description( f'Epoch: {epoch} Test Loss: {stats.mean(ll)}') _, predicted = y.detach().max(1) for item in zip(predicted, target): confusion[item[0], item[1]] += 1 precis, ave_precis = precision(confusion) print('') print( f'{Fore.CYAN}RESULTS FOR EPOCH {Fore.LIGHTYELLOW_EX}{epoch}{Style.RESET_ALL}' ) for i, cls in enumerate(classes): print( f'{Fore.LIGHTMAGENTA_EX}{cls} : {precis[i].item()}{Style.RESET_ALL}' ) best_precision = ave_precis if ave_precis > best_precision else best_precision print( f'{Fore.GREEN}ave precision : {ave_precis} best: {best_precision} {Style.RESET_ALL}' ) scheduler.step(ave_precis)
FP = 0 FN = 0 for index, name in enumerate(a_names): prediction = cv2.imread(name) ground_truth = gt[index] pe = pixel_evaluation(ground_truth, prediction) TP += pe[0] TN += pe[1] FP += pe[2] FN += pe[3] a_pe = np.array([TP, TN, FP, FN]) a_precision = precision(a_pe) a_recall = recall(a_pe) a_f1_score = f1_score(a_pe) # test B results regex = re.compile(".*(test_B).*") b_names = [m.group(0) for l in glob.glob(TestDirectory + '*') for m in [regex.search(l)] if m] b_names.sort() TP = 0 TN = 0 FP = 0 FN = 0 for index, name in enumerate(b_names): prediction = cv2.imread(name)
def train(self, epoch: int): ''' method to train your model for epoch ''' losses = AverageMeter() accuracy = AverageMeter() # switch to train mode and train one epoch self.model.train() loop = tqdm(enumerate(self.train_loader), total=len(self.train_loader), leave=False) for i, (input_, target) in loop: if i == self.config.test_steps: break input_ = input_.to(self.device) target = target.to(self.device) # compute output and loss if self.config.aug.type_aug: if self.config.aug.type_aug == 'mixup': aug_output = mixup_target(input_, target, self.config, self.device) else: assert self.config.aug.type_aug == 'cutmix' aug_output = cutmix(input_, target, self.config, self.device) input_, target_a, target_b, lam = aug_output tuple_target = (target_a, target_b, lam) if self.config.multi_task_learning: hot_target = lam*F.one_hot(target_a[:,0], 2) + (1-lam)*F.one_hot(target_b[:,0], 2) else: hot_target = lam*F.one_hot(target_a, 2) + (1-lam)*F.one_hot(target_b, 2) output = self.make_output(input_, hot_target) if self.config.multi_task_learning: loss = self.multi_task_criterion(output, tuple_target) else: loss = self.mixup_criterion(self.criterion, output, target_a, target_b, lam, 2) else: new_target = (F.one_hot(target[:,0], num_classes=2) if self.config.multi_task_learning else F.one_hot(target, num_classes=2)) output = self.make_output(input_, new_target) loss = (self.multi_task_criterion(output, target) if self.config.multi_task_learning else self.criterion(output, new_target)) # compute gradient and do SGD step self.optimizer.zero_grad() loss.backward() self.optimizer.step() # measure accuracy s = self.config.loss.amsoftmax.s acc = (precision(output[0], target[:,0].reshape(-1), s) if self.config.multi_task_learning else precision(output, target, s)) # record loss losses.update(loss.item(), input_.size(0)) accuracy.update(acc, input_.size(0)) # write to writer for tensorboard self.writer.add_scalar('Train/loss', loss, global_step=self.train_step) self.writer.add_scalar('Train/accuracy', accuracy.avg, global_step=self.train_step) self.train_step += 1 # update progress bar max_epochs = self.config.epochs.max_epoch loop.set_description(f'Epoch [{epoch}/{max_epochs}]') loop.set_postfix(loss=loss.item(), avr_loss = losses.avg, acc=acc, avr_acc=accuracy.avg, lr=self.optimizer.param_groups[0]['lr']) return losses.avg, accuracy.avg
# Get mesurments for current frame fscore, iou, map, bboxTP, bboxFN, bboxFP = bg.compute_metrics_general( cbbox, bboxes_in_img, k=5, iou_thresh=0.5) # collect of the measures fscore_tot.append(fscore) iou_tot.append(iou) map_tot.append(map) bboxTP_tot += bboxTP bboxFN_tot += bboxFN bboxFP_tot += bboxFP # Precision -Recall of this Experiment precision.append(ut.precision(bboxTP_tot, bboxFP_tot)) recall.append(ut.recall(bboxTP_tot, bboxFN_tot)) fsc.append(ut.fscore(bboxTP_tot, bboxFP_tot, bboxFN_tot)) # Compute the mAP over the precision and Recall mAp = compute_mAP(precision, recall) print('mAP : {}'.format(mAp)) # If there are bounding boxes in the ground truth #if any(cbbox): # check if it is a good example for ploting: #if PLOT_FLAG: # Plot different thresholds #fig, axs = plt.subplots(2, 3, figsize=(15, 6), facecolor='w', edgecolor='g') #fig.subplots_adjust(hspace=.5, wspace=.01)
def validate(self, valloader, epoch, opt): """ Validates the specified model on the validation data """ self.model.eval() self.losses.reset() self.top1.reset() self.top5.reset() self.acc.reset() self.data_time.reset() self.batch_time.reset() end = time.time() if opt.binaryWeight: # Mean center and clamp weights of the conv layers within the range [binStart, binEnd] count = 1 for m in self.model.modules(): if isinstance(m, nn.Conv2d): if count >= opt.binStart and count <= opt.binEnd: utils.meancenterConvParams(m) utils.clampConvParams(m) count += 1 if opt.binaryWeight: # Make a copy of the weights for use later self.realparams = deepcopy(self.model.parameters) if opt.binaryWeight: # Binarize weights of the conv layers within the range [binStart, binEnd] count = 1 for m in self.model.modules(): if isinstance(m, nn.Conv2d): if count >= opt.binStart and count <= opt.binEnd: utils.binarizeConvParams(m, opt.bnnModel) count += 1 """ A capture hook that allows us to analyze layer weights and produce a list of binarization losses based on the weights of each layer. Use on a trained FBin model which would allow us to produce a list of losses for each of the conv layers based on which a hybrid architecture can be designed. """ def activ_forward_hook(self, inputs, outputs): feature_size = outputs.size() l2loss_tot = 0.0 oneminusWloss_tot = 0.0 w_tot = 0.0 for batch_elem in range(feature_size[0]): l2loss_cur = (outputs[batch_elem] - inputs[0][batch_elem]).pow(2).sum().div( feature_size[1] * feature_size[2] * feature_size[3]) l2loss_tot += l2loss_cur.data[0] inp_squared = (inputs[0][batch_elem].clamp(min=-1.0, max=1.0)).pow(2) oneminusWloss_cur = (1 - inp_squared).abs().sum().div( feature_size[1] * feature_size[2] * feature_size[3]) oneminusWloss_tot += oneminusWloss_cur.data[0] w_tot += (inputs[0][batch_elem]).pow(2).sum().div( feature_size[1] * feature_size[2] * feature_size[3]).data[0] l2loss_tot /= feature_size[0] oneminusWloss_tot /= feature_size[0] w_tot /= feature_size[0] self.that.binarizationLosses[self.layer_num] += l2loss_tot self.that.binarizationLosses2[self.layer_num] += l2loss_tot / w_tot self.that.binarizationLosses3[self.layer_num] += oneminusWloss_tot layer_num = 0 # Use hook to compute binarization losses if selected if opt.calculateBinarizationLosses: for m in self.model.modules(): if isinstance(m, Active): m.layer_num = layer_num m.that = self m.register_forward_hook(activ_forward_hook) layer_num += 1 for i in range(0, layer_num): self.binarizationLosses.append(0.0) self.binarizationLosses2.append(0.0) self.binarizationLosses3.append(0.0) for i, data in enumerate(valloader, 0): self.itersforbin += 1 if opt.cuda: inputs, targets = data inputs = inputs.cuda(async=True) targets = targets.cuda(async=True) inputs, targets = Variable(inputs, volatile=True), Variable(targets, volatile=True) # Arrange inputs for ten crop validation where each input is converted to 5 inputs if opt.tenCrop: inputs = inputs.view( inputs.size(0) * inputs.size(1), inputs.size(2), inputs.size(3), inputs.size(4)) self.data_time.update(time.time() - end) outputs = self.model(inputs) # Tweak outputs for ten crop validation where each output is duplicated based on the number of inputs if opt.tenCrop: outputs = outputs.view( outputs.size(0) // 10, 10, outputs.size(1)).sum(1).squeeze(1).div(10.0) loss = self.criterion(outputs, targets) acc = accuracy(outputs.data.max(1)[1], targets.data, opt) self.losses.update(loss.data[0], inputs[0].size(0)) prec1, prec5 = precision(outputs.data, targets.data, topk=(1, 5)) prec1, prec5 = prec1[0], prec5[0] inputs_size = inputs.size(0) acc = accuracy(outputs.data.max(1)[1], targets.data, opt) self.acc.update(acc, inputs_size) self.top1.update(prec1, inputs_size) self.top5.update(prec5, inputs_size) # measure elapsed time self.batch_time.update(time.time() - end) end = time.time() if i % opt.printfreq == 0 and opt.verbose == True: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Accuracy {acc.val:.4f} ({acc.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})\t'.format( epoch, i, len(valloader), batch_time=self.batch_time, data_time=self.data_time, loss=self.losses, acc=self.acc, top1=self.top1, top5=self.top5)) if opt.calculateBinarizationLosses: for i in range(0, layer_num): self.binarizationLosses[i] /= self.itersforbin self.binarizationLosses2[i] /= self.itersforbin self.binarizationLosses3[i] /= self.itersforbin self.binarizationLosses[i] = math.sqrt( self.binarizationLosses[i]) self.binarizationLosses2[i] = math.sqrt( self.binarizationLosses2[i]) print('Root Mean square error per convolution layer') print(self.binarizationLosses) print('Weight Normalized RMSE per convolution layer') print(self.binarizationLosses2) print('One minus W^2 error per convolution layer') print(self.binarizationLosses3) # Copy back the original set of weights if opt.binaryWeight: current_parameters_list = list(self.realparams()) current_count = 0 for p in self.model.parameters(): p.data = current_parameters_list[current_count].data current_count += 1 finalacc = self.acc.avg #if opt.tensorboard: #self.logger.scalar_summary('val_loss', self.losses.avg, epoch) #self.logger.scalar_summary('val_acc', self.top1.avg, epoch) print('Val: [{0}]\t' 'Time {batch_time.sum:.3f}\t' 'Data {data_time.sum:.3f}\t' 'Loss {loss.avg:.3f}\t' 'Accuracy {acc:.4f}\t' 'Prec@1 {top1.avg:.4f}\t' 'Prec@5 {top5.avg:.4f}\t'.format(epoch, batch_time=self.batch_time, data_time=self.data_time, loss=self.losses, acc=finalacc, top1=self.top1, top5=self.top5)) return self.top1.avg
start = time.time() # recommend_list = recommend_multi_thread(train_data, item_sims, nearest_k, top_n) recommend_list = recommend_multi_process(train_data, item_sims, nearest_k, top_n) # recommend_list = recommend_single_thread(train_data, item_sims, nearest_k, top_n) print("recommend done, cost " + str(time.time() - start) + " s") return recommend_list if __name__ == '__main__': start_time = time.time() train, test = utils.split_data(utils.load_data("./data/ratings.dat"), 8, 1) W = item_similarity(train) recommends = recommend(train, W, nearest_k=10, top_n=10) p = utils.precision(train, test, recommends) r = utils.recall(train, test, recommends) c = utils.coverage(train, recommends) po = utils.popularity(train, recommends) cost_time = time.time() - start_time print(p, r, c, po) print("cost time " + str(cost_time) + " s")
def train(self, trainloader, epoch, opt): """ Trains the specified model for a single epoch on the training data """ self.model.train() self.losses.reset() self.top1.reset() self.top5.reset() self.acc.reset() self.data_time.reset() self.batch_time.reset() end = time.time() for i, data in enumerate(trainloader, 0): if opt.binaryWeight: # Mean center and clamp weights of the conv layers within the range [binStart, binEnd] count = 1 for m in self.model.modules(): if isinstance(m, nn.Conv2d): if count >= opt.binStart and count <= opt.binEnd: utils.meancenterConvParams(m) utils.clampConvParams(m) count += 1 # Make a copy of the model parameters for use later self.realparams = deepcopy(self.model.parameters) # Binarize weights of the conv layers within the range [binStart, binEnd] count = 1 for m in self.model.modules(): if isinstance(m, nn.Conv2d): if count >= opt.binStart and count <= opt.binEnd: utils.binarizeConvParams(m, opt.bnnModel) count += 1 self.optimizer.zero_grad() if opt.cuda: inputs, targets = data inputs = inputs.cuda(async=True) targets = targets.cuda(async=True) inputs, targets = Variable(inputs), Variable(targets) self.data_time.update(time.time() - end) outputs = self.model(inputs) loss = self.criterion(outputs, targets) prec1, prec5 = precision(outputs.data, targets.data, topk=(1, 5)) acc = accuracy(outputs.data.max(1)[1], targets.data, opt) prec1, prec5 = prec1[0], prec5[0] loss.backward() if opt.binaryWeight: # Copy back the real weights stored earlier to all the conv layers current_parameters_list = list(self.realparams()) current_count = 0 for p in self.model.parameters(): p.data = current_parameters_list[current_count].data current_count += 1 # Apply gradient updates to all the layers count = 1 for m in self.model.modules(): if isinstance(m, nn.Conv2d): if count >= opt.binStart and count <= opt.binEnd: utils.updateBinaryGradWeight(m, opt.bnnModel) count += 1 self.optimizer.step() inputs_size = inputs.size(0) self.losses.update(loss.data[0], inputs_size) self.acc.update(acc, inputs_size) self.top1.update(prec1, inputs_size) self.top5.update(prec5, inputs_size) # measure elapsed time self.batch_time.update(time.time() - end) end = time.time() if i % opt.printfreq == 0 and opt.verbose == True: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.avg:.3f} ({batch_time.sum:.3f})\t' 'Data {data_time.avg:.3f} ({data_time.sum:.3f})\t' 'Loss {loss.avg:.3f}\t' 'Accuracy {acc.avg:.4f}\t' 'Prec@1 {top1.avg:.4f}\t' 'Prec@5 {top5.avg:.4f}'.format( epoch, i, len(trainloader), batch_time=self.batch_time, data_time=self.data_time, loss=self.losses, acc=self.acc, top1=self.top1, top5=self.top5)) # log to TensorBoard #if opt.tensorboard: #self.logger.scalar_summary('train_loss', self.losses.avg, epoch) #self.logger.scalar_summary('train_acc', self.top1.avg, epoch) print('Train: [{0}]\t' 'Time {batch_time.sum:.3f}\t' 'Data {data_time.sum:.3f}\t' 'Loss {loss.avg:.3f}\t' 'Accuracy {acc.avg:.4f}\t' 'Prec@1 {top1.avg:.4f}\t' 'Prec@5 {top5.avg:.4f}\t'.format(epoch, batch_time=self.batch_time, data_time=self.data_time, loss=self.losses, acc=self.acc, top1=self.top1, top5=self.top5))
image_names, _, image_matches = mirflickr_images() absolute_matches = {} for label, s in results.iteritems(): absolute_matches[label] = {} for target_image, retrieved in s.iteritems(): if dataset == 'ukbench': retrieved_filenames = retrieved[:3] recall_value = recall(target_image, retrieved_filenames, retrieved, method=result_test_ukbench ) precision_value = precision(target_image, retrieved_filenames, retrieved, method=result_test_ukbench ) recalls[label][target_image] = recall_value precisions[label][target_image] = precision_value relevant, retrieved = result_test_ukbench(target_image, retrieved_filenames, retrieved) elif dataset == 'mirflickr': retrieved_filenames = retrieved[:image_matches[target_image]] recall_value = recall(target_image, retrieved_filenames, retrieved, method=result_test_mirflickr ) precision_value = precision(target_image, retrieved_filenames, retrieved,
def test(self, data): preds = [] actuals = [] source_data, source_loc_data, target_data, target_label = data N = int(np.ceil(len(source_data) / self.batch_size)) cost = 0 x = np.ndarray([self.batch_size, 1], dtype=np.int32) time = np.ndarray([self.batch_size, self.mem_size], dtype=np.int32) target = np.zeros([self.batch_size], dtype=np.int32) context = np.ndarray([self.batch_size, self.mem_size], dtype=np.int32) mask = np.ndarray([self.batch_size, self.mem_size]) context.fill(self.pad_idx) m, acc = 0, 0 for i in range(N): target.fill(0) time.fill(self.mem_size) context.fill(self.pad_idx) mask.fill(-1.0 * np.inf) raw_labels = [] for b in range(self.batch_size): x[b][0] = target_data[m] target[b] = target_label[m] time[b, :len(source_loc_data[m])] = source_loc_data[m] context[b, :len(source_data[m])] = source_data[m] mask[b, :len(source_data[m])].fill(0) raw_labels.append(target_label[m]) m += 1 loss = self.sess.run( [self.loss], feed_dict={ self.input: x, self.time: time, self.target: target, self.context: context, self.mask: mask }) cost += np.sum(loss) predictions = self.sess.run(self.correct_prediction, feed_dict={ self.input: x, self.time: time, self.target: target, self.context: context, self.mask: mask }) for b in range(self.batch_size): preds.append(predictions[b]) actuals.append(raw_labels[b]) if raw_labels[b] == predictions[b]: acc = acc + 1 prec = precision(actuals, preds, labels=[0, 1, 2]) rec = recall(actuals, preds, labels=[0, 1, 2]) f = f1(actuals, preds, labels=[0, 1, 2]) return cost, acc / float(len(source_data)), prec, rec, f