def test(net, recall_ids): net.eval() with torch.no_grad(): # obtain feature vectors for all data for key in eval_dict.keys(): eval_dict[key]['features'] = [] for inputs, labels in tqdm(eval_dict[key]['data_loader'], desc='processing {} data'.format(key)): inputs, labels = inputs.cuda(), labels.cuda() features, classes = net(inputs) eval_dict[key]['features'].append(features) eval_dict[key]['features'] = torch.cat(eval_dict[key]['features'], dim=0) # compute recall metric if data_name == 'isc': acc_list = recall(eval_dict['test']['features'], test_data_set.labels, recall_ids, eval_dict['gallery']['features'], gallery_data_set.labels) else: acc_list = recall(eval_dict['test']['features'], test_data_set.labels, recall_ids) desc = 'Test Epoch {}/{} '.format(epoch, num_epochs) for index, rank_id in enumerate(recall_ids): desc += 'R@{}:{:.2f}% '.format(rank_id, acc_list[index] * 100) results['test_recall@{}'.format(rank_id)].append(acc_list[index] * 100) print(desc) return acc_list[0]
def eval(net, data_dict, ensemble_num, recalls): net.eval() data_set = ImageReader(data_dict, get_transform(DATA_NAME, 'test')) data_loader = DataLoader(data_set, BATCH_SIZE, shuffle=False, num_workers=8) features = [] with torch.no_grad(): for inputs, labels in data_loader: out = net(inputs.to(DEVICE)) out = F.normalize(out) features.append(out.cpu()) features = torch.cat(features, 0) torch.save( features, 'results/{}_test_features_{:03}.pth'.format(DATA_NAME, ensemble_num)) # load feature vectors features = [ torch.load('results/{}_test_features_{:03}.pth'.format(DATA_NAME, d)) for d in range(1, ensemble_num + 1) ] features = torch.cat(features, 1) acc_list = recall(features, data_set.labels, rank=recalls) desc = '' for index, recall_id in enumerate(recalls): desc += 'R@{}:{:.2f}% '.format(recall_id, acc_list[index] * 100) print(desc)
def eval(net, loader, ep): net.eval() n_node = graph_net.n_node graph_net.n_node = 1 net.return_base = False test_iter = tqdm(loader) embeddings_all, labels_all = [], [] test_iter.set_description("[Eval][Epoch %d]" % ep) with torch.no_grad(): for images, labels in test_iter: images, labels = images.cuda(), labels.cuda() embedding = net(images) if graph_net is not None: embedding = graph_net(embedding) embeddings_all.append(embedding.data) labels_all.append(labels.data) embeddings_all = torch.cat(embeddings_all).cpu() labels_all = torch.cat(labels_all).cpu() rec = recall(embeddings_all, labels_all) print('[Epoch %d] Recall@1: [%.4f]\n' % (ep, 100 * rec)) graph_net.n_node = n_node return rec
def vectorized_rf(x_train, y_train, x_test, y_test, checktrain=True, ngram_range=(1, 1), vector_type="count", dataset="default"): vectorizer = CountVectorizer(tokenizer=tokenize, ngram_range=ngram_range) if vector_type=="count" \ else TfidfVectorizer(tokenizer=tokenize, ngram_range=ngram_range) vectorized_x_train = vectorizer.fit_transform(x_train) vectorized_x_test = vectorizer.transform(x_test) model = train(vectorized_x_train, y_train, checktrain, ngram_range, vector_type=vector_type, dataset=dataset) pred_x_train = predict(model, vectorized_x_train) pred_x_test = predict(model, vectorized_x_test) precision_test = precision(y_test, pred_x_test) recall_test = recall(y_test, pred_x_test) f1_test = f1(y_test, pred_x_test) print("Accuracy training accuracy (" + dataset, vector_type, " vectorized joint data) =", accuracy(y_train, pred_x_train)) print("Accuracy testing accuracy (" + dataset, vector_type, "vectorized joint data) =", accuracy(y_test, pred_x_test), "\n") print("Precision (" + dataset, vector_type + " vectorized test data) =", precision_test) print("Recall (" + dataset, vector_type + " test data) =", recall_test) print("F1 (" + dataset, vector_type + " test data) =", f1_test, "\n")
def simple_lstm(x_train, y_train, x_test, y_test, dataset="default"): model = train(x_train, y_train, dataset=dataset) print(x_train.shape) print(x_test.shape) # print(model.summary()) pred_x_train = model.predict(x_train) pred_x_test = model.predict(x_test) precision_test = precision(np.argmax(y_test, axis=1), np.argmax(pred_x_test, axis=1), labels=(0, 1, 2)) recall_test = recall(np.argmax(y_test, axis=1), np.argmax(pred_x_test, axis=1), labels=(0, 1, 2)) f1_test = f1(np.argmax(y_test, axis=1), np.argmax(pred_x_test, axis=1), labels=(0, 1, 2)) # print( "Accuracy training accuracy = ", accuracy(np.argmax(y_train, axis=1), np.argmax(pred_x_train, axis=1))) print("Accuracy testing accuracy =", accuracy(np.argmax(y_test, axis=1), np.argmax(pred_x_test, axis=1)), "\n") # print("Precision (test data) =", precision_test) print("Recall (test data) =", recall_test) print("F1 (test data) =", f1_test, "\n")
def evaluate(self, Xt, Xc, y): """ Evaluates the model's accuracy, precision, recall, F1 score and loss value on a dataset. :param Xt: Matrix containing title input. :param Xc: Matrix containing content input. :param y: Label vector. :return: Accuracy, lists containing each topic's precision, recall, F1 score followed by the macro-average across topics, and the model's loss value. """ probs = self.model.predict([Xt, Xc], verbose=1, batch_size=100) preds = np.argmax(probs, axis=1) true = np.argmax(y, 1) acc = np.sum(np.equal(preds, true)) / np.size(true, 0) p, r, f1 = [], [], [] for i in range(len(self.classes)): p.append(utils.precision(preds, true, i)) r.append(utils.recall(preds, true, i)) f1.append(utils.f1_score(p[i], r[i])) p2 = [x for x in p if x is not None] r2 = [x for x in r if x is not None] f2 = [x for x in f1 if x is not None] p.append(np.mean(p2)) r.append(np.mean(r2)) f1.append(np.mean(f2)) print('\nCalculating loss...') self.compile() loss = self.model.evaluate([Xt, Xc], y, batch_size=100)[0] return acc, p, r, f1, loss
def eval_graph(net, loader, ep): net.eval() graph_net.eval() test_iter = tqdm(loader) embeddings_all, labels_all = [], [] test_iter.set_description("[Eval][Epoch %d]" % ep) with torch.no_grad(): for images, labels in test_iter: images, labels = images.cuda(), labels.cuda() embedding = net(images) embeddings_all.append(embedding.data) labels_all.append(labels.data) embeddings_all = torch.cat(embeddings_all) labels_all = torch.cat(labels_all) d = pdist(embeddings_all) pos_idx = d.topk(11, dim=1, largest=False)[1][:, 1:] neg_idx = torch.randint(0, len(d), (len(d), 1), device=d.device, dtype=torch.int64) graph_embedding = [] for i, e in enumerate(embeddings_all): pos_embedding = embeddings_all[pos_idx[i][1:]] neg_embedding = embeddings_all[torch.cat([pos_idx[j] for j in range(i-3, i-1)])] e = torch.cat((e.unsqueeze(0), pos_embedding, neg_embedding), dim=0) e = graph_net(e) graph_embedding.append(e[0]) graph_embedding = torch.stack(graph_embedding, dim=0) rec = recall(graph_embedding, labels_all) print('[Epoch %d] Recall@1: [%.4f]\n' % (ep, 100 * rec)) return rec
def eval(net, loader, ep): K = [1, 10, 100, 1000] net.eval() test_iter = tqdm(loader) embeddings_all, labels_all = [], [] test_iter.set_description("[Eval][Epoch %d]" % ep) with torch.no_grad(): for images, labels in test_iter: images, labels = images.cuda(), labels.cuda() embedding = net(images) embeddings_all.append(embedding.data) labels_all.append(labels.data) print(cuda.memory_allocated(cuda.current_device())) embeddings_all = torch.cat(embeddings_all).cpu() labels_all = torch.cat(labels_all).cpu() rec = recall(embeddings_all, labels_all, K=K) print("Embedding Size: %d" % len(embeddings_all)) print(labels_all.sum()) for k, r in zip(K, rec): print('[Epoch %d] Recall@%d: [%.4f]\n' % (ep, k+1, 100 * r)) return rec[0]
def test(net, recall_ids): net.eval() with torch.no_grad(): # obtain feature vectors for all data for key in eval_dict.keys(): eval_dict[key]['features'] = [] for inputs, labels in tqdm(eval_dict[key]['data_loader'], desc='processing {} data'.format(key), dynamic_ncols=True): features, classes = net(inputs.cuda()) eval_dict[key]['features'].append(features) eval_dict[key]['features'] = torch.cat(eval_dict[key]['features'], dim=0) test_features = torch.sign(eval_dict['test']['features']).cpu() # compute recall metric if data_name == 'isc': dense_acc_list = recall(eval_dict['test']['features'].cpu(), test_data_set.labels, recall_ids, eval_dict['gallery']['features'].cpu(), gallery_data_set.labels) gallery_features = torch.sign( eval_dict['gallery']['features']).cpu() binary_acc_list = recall(test_features, test_data_set.labels, recall_ids, gallery_features, gallery_data_set.labels, binary=True) else: dense_acc_list = recall(eval_dict['test']['features'].cpu(), test_data_set.labels, recall_ids) binary_acc_list = recall(test_features, test_data_set.labels, recall_ids, binary=True) desc = 'Test Epoch {}/{} '.format(epoch, num_epochs + 1) for index, rank_id in enumerate(recall_ids): desc += 'R@{}:{:.2f}%[{:.2f}%] '.format(rank_id, dense_acc_list[index] * 100, binary_acc_list[index] * 100) results['test_dense_recall@{}'.format(rank_id)].append( dense_acc_list[index] * 100) results['test_binary_recall@{}'.format(rank_id)].append( binary_acc_list[index] * 100) print(desc) return dense_acc_list[0]
def test_F2Score(): print(true_positive(actual, estimated) == 2) print(false_positive(actual, estimated) == 3) print(false_negative(actual, estimated) == 1) expected_prec = 2 / (2 + 3) print(precision(actual, estimated) == expected_prec) expected_recall = 2 / (2 + 1) print(recall(actual, estimated) == expected_recall) print( F1score(actual, estimated) == 2 * expected_recall * expected_prec / (expected_recall + expected_prec))
def eval(net, loader, ep): net.eval() test_iter = tqdm(loader) embeddings_all, labels_all = [], [] with torch.no_grad(): for images, labels in test_iter: images, labels = images.cuda(), labels.cuda() output = net(images) embeddings_all.append(output.data) labels_all.append(labels.data) test_iter.set_description("[Eval][Epoch %d]" % ep) embeddings_all = torch.cat(embeddings_all).cpu() labels_all = torch.cat(labels_all).cpu() print('[Epoch %d] Recall@1: [%.6f]\n' % (ep, recall(embeddings_all, labels_all)))
def eval(ep, loader=loader_eval): model.eval() test_iter = tqdm(loader) embeddings_all, labels_all = [], [] test_iter.set_description("[Eval][Epoch %d]" % ep) with torch.no_grad(): for images, labels in test_iter: images, labels = images.cuda(), labels.cuda() embedding = model(images) embeddings_all.append(embedding.data) labels_all.append(labels.data) embeddings_all = torch.cat(embeddings_all).cpu() labels_all = torch.cat(labels_all).cpu() rec = recall(embeddings_all, labels_all) print('[Epoch %d] Recall@1: [%.4f]\n' % (ep, 100 * rec)) return rec
def test(net, recall_ids): net.eval() # obtain feature vectors for all data with torch.no_grad(): features = [] for inputs, labels in tqdm(test_data_loader, desc='processing test data', dynamic_ncols=True): feature, _ = net(inputs.cuda()) features.append(feature) features = torch.cat(features, dim=0) # compute recall metric acc_list = recall(features, test_data_set.labels, recall_ids) desc = 'Test Epoch {}/{} '.format(epoch, num_epochs) for index, rank_id in enumerate(recall_ids): desc += 'R@{}:{:.2f}% '.format(rank_id, acc_list[index] * 100) results['test_recall@{}'.format(rank_id)].append(acc_list[index] * 100) print(desc) data_base['test_features'] = features return acc_list[0]
def eval(net, loader, ep): net.eval() embeddings_all, labels_all = [], [] with torch.no_grad(): for images, labels in loader: images, labels = images.to(device), labels.to(device) output = avgpool(net(images, True)[-1]) embeddings_all.append(output.data) labels_all.append(labels.data) embeddings_all = torch.cat(embeddings_all).cpu() labels_all = torch.cat(labels_all).cpu() rec = recall(embeddings_all, labels_all, K=[1]) for k, r in enumerate(rec): print('[Epoch %d] Recall@%d: [%.4f]\n' % (ep, k + 1, 100 * r)) return rec[0]
def kfold(classification_algorithm, k): res = {"accuracy": 0, "precision": 0, "recall": 0, "f1": 0} for i in range(1, k + 1): validation = utils.load_train(i) validation = validation["plus"] + validation["minus"] train = {"plus": [], "minus": []} for j in range(1, k + 1): if j != i: extension = utils.load_train(j) train["plus"].extend(extension["plus"]) train["minus"].extend(extension["minus"]) classification = classification_algorithm(train, validation) res["accuracy"] += utils.accuracy(classification) res["precision"] += utils.precision(classification) res["recall"] += utils.recall(classification) res["f1"] += utils.F1_score(classification) for k in res: res[k] /= k print res return res
def eval(net, net_normalize, loader, ep): net.eval() test_iter = tqdm(loader) embeddings_all, labels_all = [], [] with torch.no_grad(): for images, labels in test_iter: images, labels = images.cuda(), labels.cuda() output = net(net_normalize(images)) embeddings_all.append(output.data) labels_all.append(labels.data) test_iter.set_description("[Eval][Epoch %d]" % ep) embeddings_all = torch.cat(embeddings_all).cpu() labels_all = torch.cat(labels_all).cpu() rec = recall(embeddings_all, labels_all, K=[1]) for k, r in enumerate(rec): print('[Epoch %d] Recall@%d: [%.4f]\n' % (ep, k + 1, 100 * r)) return rec[0]
def main(path_detection, path_truth, mode, iou): print("Path to detections:", path_detection) print("Path to ground truth:", path_truth) print("Chosen mode:", mode) print("IoU-threshold:", iou) # acquire detections and false_negatives for each class # detections: {class: [(1,0.95), (0,0.87), (0,0.6), (1,0.67), ...., (0,0.9), (1,0.7)]} # false_negatives: int detections, false_negatives = loop_detection_files(path_detection, path_truth, iou, mode) # store APs for calculating mAP average_precisions = [] # analyse results of each class for classification in detections.keys(): classification_detections = [ d[0] for d in sorted(detections.get(classification), key=lambda tup: tup[1], reverse=True) ] # print precision, recall, AP and show precision-recall curve print("-------------") print("CLASS", classification) print("Precision:", precision(classification_detections)) print( "Recall:", recall(classification_detections, false_negatives.get(classification))) average_precision, smoothed_precision, recall_samples = AP( classification, classification_detections, false_negatives.get(classification)) average_precisions.append(average_precision) print("AP:", average_precision) visualise_results(classification, classification_detections, false_negatives.get(classification), smoothed_precision, recall_samples) print("-------------") print("********") print("mAP:", np.mean(average_precisions)) print("********")
def compute_test(): model.eval() output = model(features, adjs) loss_test = F.nll_loss(output[idx_test], labels[idx_test]) acc_test = accuracy(output[idx_test], labels[idx_test]) # mac = macro_f1(output[idx_test], labels[idx_test]) # mac_pre = macro_precision(output[idx_test], labels[idx_test]) # mac_rec = macro_recall(output[idx_test], labels[idx_test]) # print("Test set results:", # "loss= {:.4f}".format(loss_test.item()), # "accuracy= {:.4f}".format(acc_test.item()), # "macro_f1= {:.4f}".format(mac), # "macro_precision= {:.4f}".format(mac_pre), # "macro_recall= {:.4f}".format(mac_rec)) mac = f1(output[idx_test], labels[idx_test]) mac_pre = precision(output[idx_test], labels[idx_test]) mac_rec = recall(output[idx_test], labels[idx_test]) print("Test set results:", "loss= {:.4f}".format(loss_test.item()), "accuracy= {:.4f}".format(acc_test.item()), "macro_f1= {:.4f}".format(mac), "macro_precision= {:.4f}".format(mac_pre), "macro_recall= {:.4f}".format(mac_rec))
def two_eval(net, loader1, loader2, ep): net.eval() test_iter = tqdm(loader1) test_iter2 = tqdm(loader2) embeddings_all, labels_all = [], [] test_iter.set_description("[Eval][Epoch %d]" % ep) labels_in = [] with torch.no_grad(): for images, labels in test_iter: images, labels = images.cuda(), labels.cuda() embedding = net(images) embeddings_all.append(embedding.data) labels_all.append(labels.data) #embeddings_all = torch.cat(embeddings_all).cpu() #labels_all = torch.cat(labels_all).cpu() for images, labels in test_iter2: images, labels = images.cuda(), labels.cuda() embedding = net(images) embeddings_all.append(embedding.data) labels_all.append(labels.data) embeddings_all = torch.cat(embeddings_all).cpu() labels_all = torch.cat(labels_all).cpu() rec = recall(embeddings_all, labels_all, K=4) print("Embedding Size: %d" % len(embeddings_all)) print(labels_all.sum()) for k, r in enumerate(rec): print('[Epoch %d] Recall@%d: [%.4f]\n' % (ep, k + 1, 100 * r)) return rec[0]
-1)[:args.crop_height, :args.crop_width] st = time.time() output_image = sess.run(network, feed_dict={input: input_image}) output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) out_eval_image = output_image[:, :, 0] out_vis_image = helpers.colour_code_segmentation(output_image) accuracy = utils.compute_avg_accuracy(out_eval_image, gt) class_accuracies = utils.compute_class_accuracies( out_eval_image, gt, num_classes) prec = utils.precision(out_eval_image, gt) rec = utils.recall(out_eval_image, gt) f1 = utils.f1score(out_eval_image, gt) iou = utils.compute_mean_iou(out_eval_image, gt) file_name = utils.filepath_to_name(val_input_names[ind]) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec) f1_list.append(f1)
# Get mesurments for current frame fscore, iou, map, bboxTP, bboxFN, bboxFP = bg.compute_metrics_general( cbbox, bboxes_in_img, k=5, iou_thresh=0.5) # collect of the measures fscore_tot.append(fscore) iou_tot.append(iou) map_tot.append(map) bboxTP_tot += bboxTP bboxFN_tot += bboxFN bboxFP_tot += bboxFP # Precision -Recall of this Experiment precision.append(ut.precision(bboxTP_tot, bboxFP_tot)) recall.append(ut.recall(bboxTP_tot, bboxFN_tot)) fsc.append(ut.fscore(bboxTP_tot, bboxFP_tot, bboxFN_tot)) # Compute the mAP over the precision and Recall mAp = compute_mAP(precision, recall) print('mAP : {}'.format(mAp)) # If there are bounding boxes in the ground truth #if any(cbbox): # check if it is a good example for ploting: #if PLOT_FLAG: # Plot different thresholds #fig, axs = plt.subplots(2, 3, figsize=(15, 6), facecolor='w', edgecolor='g') #fig.subplots_adjust(hspace=.5, wspace=.01)
axis=0) / 255.0 st = time.time() output_image = sess.run(network, feed_dict={z: input_image}) gt_map = cv2.imread(test_output_names[ind], -1)[:args.crop_height, :args.crop_width] output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) output_image = output_image[:, :, 0] out_vis_image = helpers.colour_code_segmentation(output_image) accuracy = utils.compute_avg_accuracy(output_image, gt_map) class_accuracies = utils.compute_class_accuracies(output_image, gt_map) prec = utils.precision(output_image, gt_map) rec = utils.recall(output_image, gt_map) f1 = utils.f1score(output_image, gt_map) iou = utils.compute_mean_iou(output_image, gt_map) file_name = utils.filepath_to_name(test_input_names[ind]) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec) f1_list.append(f1)
FN = 0 for index, name in enumerate(a_names): prediction = cv2.imread(name) ground_truth = gt[index] pe = pixel_evaluation(ground_truth, prediction) TP += pe[0] TN += pe[1] FP += pe[2] FN += pe[3] a_pe = np.array([TP, TN, FP, FN]) a_precision = precision(a_pe) a_recall = recall(a_pe) a_f1_score = f1_score(a_pe) # test B results regex = re.compile(".*(test_B).*") b_names = [m.group(0) for l in glob.glob(TestDirectory + '*') for m in [regex.search(l)] if m] b_names.sort() TP = 0 TN = 0 FP = 0 FN = 0 for index, name in enumerate(b_names): prediction = cv2.imread(name) ground_truth = gt[index]
def test(self, data): preds = [] actuals = [] source_data, source_loc_data, target_data, target_label = data N = int(np.ceil(len(source_data) / self.batch_size)) cost = 0 x = np.ndarray([self.batch_size, 1], dtype=np.int32) time = np.ndarray([self.batch_size, self.mem_size], dtype=np.int32) target = np.zeros([self.batch_size], dtype=np.int32) context = np.ndarray([self.batch_size, self.mem_size], dtype=np.int32) mask = np.ndarray([self.batch_size, self.mem_size]) context.fill(self.pad_idx) m, acc = 0, 0 for i in range(N): target.fill(0) time.fill(self.mem_size) context.fill(self.pad_idx) mask.fill(-1.0 * np.inf) raw_labels = [] for b in range(self.batch_size): x[b][0] = target_data[m] target[b] = target_label[m] time[b, :len(source_loc_data[m])] = source_loc_data[m] context[b, :len(source_data[m])] = source_data[m] mask[b, :len(source_data[m])].fill(0) raw_labels.append(target_label[m]) m += 1 loss = self.sess.run( [self.loss], feed_dict={ self.input: x, self.time: time, self.target: target, self.context: context, self.mask: mask }) cost += np.sum(loss) predictions = self.sess.run(self.correct_prediction, feed_dict={ self.input: x, self.time: time, self.target: target, self.context: context, self.mask: mask }) for b in range(self.batch_size): preds.append(predictions[b]) actuals.append(raw_labels[b]) if raw_labels[b] == predictions[b]: acc = acc + 1 prec = precision(actuals, preds, labels=[0, 1, 2]) rec = recall(actuals, preds, labels=[0, 1, 2]) f = f1(actuals, preds, labels=[0, 1, 2]) return cost, acc / float(len(source_data)), prec, rec, f
def main(): parser = get_parser("generation", MODEL_CLASSES, ALL_MODELS) args = parser.parse_args() if args.local_rank == -1: args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = torch.cuda.device_count() else: torch.cuda.set_device(args.local_rank) args.device = torch.device("cuda", args.local_rank) torch.distributed.init_process_group(backend='nccl') args.n_gpu = 1 logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN) logger.warning( "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s", args.local_rank, args.device, args.n_gpu, bool(args.local_rank != -1)) set_seed(args) if args.local_rank not in [-1, 0]: torch.distributed.barrier() args.model_type = args.model_type.lower() model_class, tokenizer_class = MODEL_CLASSES[args.model_type] gpt2_model, tokenizer = init_gpt2_model( checkpoint_dir=args.model_name_or_path, args=args, model_class=model_class, tokenizer_class=tokenizer_class) gpt2_model.eval() if args.local_rank == 0: torch.distributed.barrier() config = gpt2_model.gpt2.config if args.length < 0 and config.max_position_embeddings > 0: args.length = config.max_position_embeddings elif 0 < config.max_position_embeddings < args.length: args.length = config.max_position_embeddings # No generation bigger than model size elif args.length < 0: args.length = MAX_LENGTH # avoid infinite loop logger.info(args) if args.local_rank not in [-1, 0]: torch.distributed.barrier( ) # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache eval_dataset = load_and_cache_examples(args, tokenizer) if args.local_rank == 0: torch.distributed.barrier() eval_sampler = SequentialSampler( eval_dataset) if args.local_rank == -1 else DistributedSampler( eval_dataset, shuffle=False) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.per_gpu_eval_batch_size) output_log = { "true_text": [], "generated_text": [], "context": [], "recall_score": [], "context_suffix_styles": [], "original_styles": [], "metadata": [] } for batch in tqdm(eval_dataloader, desc="Evaluating", disable=args.local_rank not in [-1, 0]): sentences = batch["sentence"].to(args.device) segments = batch["segment"].to(args.device) global_dense_vectors = batch["global_dense_vectors"].to(args.device) suffix_styles = batch["suffix_style"] original_styles = batch["original_style"] metadata = batch["metadata"] # Assume init_context_size is same for all examples in minibatch init_context_size = batch["init_context_size"][0].item() out, dense_length, scores = gpt2_model.generate( gpt2_sentences=sentences, segments=segments, eos_token_id=tokenizer.eos_token_id, global_dense_vectors=global_dense_vectors, init_context_size=init_context_size) for sent_num in range(sentences.shape[0]): output_sequence = out[sent_num][init_context_size:].tolist() if tokenizer.eos_token_id in output_sequence: output_sequence = output_sequence[:output_sequence.index( tokenizer.eos_token_id)] true_text = tokenizer.decode( sentences[sent_num, init_context_size:].tolist(), clean_up_tokenization_spaces=True, skip_special_tokens=True) generated_text = tokenizer.decode( output_sequence, clean_up_tokenization_spaces=True, skip_special_tokens=True) context = tokenizer.decode( sentences[sent_num, :init_context_size].tolist(), clean_up_tokenization_spaces=True, skip_special_tokens=True) recall_score = recall(true_text, context) output_log["true_text"].append(true_text) output_log["generated_text"].append(generated_text) output_log["context"].append(context) output_log["recall_score"].append("%.4f" % recall_score) output_log["metadata"].append(metadata[sent_num]) if hasattr(eval_dataset, "reverse_label_dict"): output_log["context_suffix_styles"].append( class_number_to_str(eval_dataset, suffix_styles[sent_num])) output_log["original_styles"].append( class_number_to_str(eval_dataset, original_styles[sent_num])) else: output_log["context_suffix_styles"].append("<none>") output_log["original_styles"].append("<none>") with open( os.path.join(args.generation_output_dir, "reference_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["true_text"]) + "\n") with open( os.path.join(args.generation_output_dir, "generated_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["generated_text"]) + "\n") with open( os.path.join(args.generation_output_dir, "context_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["context"]) + "\n") with open( os.path.join(args.generation_output_dir, "recall_score_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["recall_score"]) + "\n") with open( os.path.join( args.generation_output_dir, "context_suffix_styles_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["context_suffix_styles"]) + "\n") with open( os.path.join(args.generation_output_dir, "original_styles_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["original_styles"]) + "\n") with open( os.path.join(args.generation_output_dir, "metadata_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["metadata"]) + "\n")
x2 = D.get_data('Data1/Class2.txt') x3 = D.get_data('Data1/Class3.txt') if not os.path.isdir("plots_multi"): os.mkdir("plots_multi") w1 = [200.0, -200.0, 200.0] w2 = [-200.0, 200.0, 200.0] w3 = [200.0, 200.0, -200.0] perceptron(w1, x1, x2, x3, 0.5, 100, 1) perceptron(w2, x2, x3, x1, 0.5, 100, 2) perceptron(w3, x3, x1, x2, 0.5, 100, 3) plt.figure() decision_boundary(plt, w1, w2, w3) conf_mat = get_conf(x1, x2, x3, w1, w2, w3, plt) plt.savefig('plots_multi/decision_boundary.png') print(conf_mat) print("Accuracy: ", U.accuracy(conf_mat)) print("Precision for class 1: ", U.precision(conf_mat, 0)) print("Precision for class 2: ", U.precision(conf_mat, 1)) print("Precision for class 3: ", U.precision(conf_mat, 2)) print("Recall for class 1: ", U.recall(conf_mat, 0)) print("Recall for class 2: ", U.recall(conf_mat, 1)) print("Recall for class 3: ", U.recall(conf_mat, 2)) print("F-Score for class 1: ", U.f_score(conf_mat, 0)) print("F-Score for class 2: ", U.f_score(conf_mat, 1)) print("F-Score for class 3: ", U.f_score(conf_mat, 2))
axis=0) / 255.0 gt = cv2.imread(val_output_names[ind], -1)[:args.crop_height, :args.crop_width] st = time.time() output_image = sess.run(network, feed_dict={input: input_image}) output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) out_vis_image = helpers.colour_code_segmentation(output_image) accuracy = utils.compute_avg_accuracy(output_image, gt) class_accuracies = utils.compute_class_accuracies( output_image, gt, num_classes) prec = utils.precision(output_image, gt) rec = utils.recall(output_image, gt) f1 = utils.f1score(output_image, gt) iou = utils.compute_mean_iou(output_image, gt) file_name = utils.filepath_to_name(val_input_names[ind]) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec) f1_list.append(f1)
theta = np.zeros((X.shape[1], classes)) l2_lambda = 0 # Calculate optimal theta using Newton Conjugate Gradient (closest alternative to fminunc) X_train, X_test, y_train, y_test = split_data(X, y) # stratified split theta = opt.minimize(fun=calculate_cost_and_gradient, x0=theta, method='TNC', args=(X_train, y_train, classes, l2_lambda), jac=True, options={'maxiter': 50}) theta.x = theta.x.reshape( X.shape[1], classes) # reshape theta into necessary dimensions # Print out the cost and calculated theta print('Testing cost: %f' % theta.fun) print('----------------------------------------------------') # separator # Calculate accuracy, precision and recall print('Accuracy (percentage of guessed samples): %f' % multiclass_accuracy(theta.x, X_test, y_test)) conf = confusion_matrix_c(y_test, softmax(X_test.dot(theta.x)), True) plot_confusion_matrix(conf, 'Confusion matrix (actual vs predicted values)') print('Precision (true positives / predicted positives): %f' % precision(conf)) print('Recall (true positives / actual positives): %f' % recall(conf)) plt.show()
start = time.time() # recommend_list = recommend_multi_thread(train_data, item_sims, nearest_k, top_n) recommend_list = recommend_multi_process(train_data, item_sims, nearest_k, top_n) # recommend_list = recommend_single_thread(train_data, item_sims, nearest_k, top_n) print("recommend done, cost " + str(time.time() - start) + " s") return recommend_list if __name__ == '__main__': start_time = time.time() train, test = utils.split_data(utils.load_data("./data/ratings.dat"), 8, 1) W = item_similarity(train) recommends = recommend(train, W, nearest_k=10, top_n=10) p = utils.precision(train, test, recommends) r = utils.recall(train, test, recommends) c = utils.coverage(train, recommends) po = utils.popularity(train, recommends) cost_time = time.time() - start_time print(p, r, c, po) print("cost time " + str(cost_time) + " s")
save_model_path = os.path.join( save_time_fold, preprocess_path.split('/')[-1] + '_times_' + str(train_times) + '_' + str(round(best_test, 4)) + '.pth.tar') torch.save(model.state_dict(), save_model_path) save_predict_target_path = os.path.join( save_time_fold, preprocess_path.split('/')[-1] + '_times_' + str(train_times) + '_' + str(round(best_test, 4)) + '.txt') predict_target = torch.cat((predicts, targets), dim=0).detach().cpu().numpy() np.savetxt(save_predict_target_path, predict_target) precision_score[train_times] = round(precision(predicts, targets), 4) recall_score[train_times] = round(recall(predicts, targets), 4) specificity_score[train_times] = round( specificity(predicts, targets), 4) mcc_score[train_times] = round(mcc(predicts, targets), 4) auc_score = round(auc(predicts, targets), 4) aupr_score = round(aupr(predicts, targets), 4) print('Epoch: {:04d}'.format(epoch + 1), 'Train_times:', train_times) print( "*****************test_score {:.4f} best_socre {:.4f}****************" .format(test_score, best_test)) print("All Test Score:", acc_score) print(args.dataset, " Optimization Finished!") print("Total time elapsed: {:.4f}s".format(time.time() - t_total)) print("acc Score:", acc_score) print("precision Score:", precision_score) print("recall score", recall_score)
for image_name in image_names: recalls_image_names[image_name] = [] precisions_image_names[image_name] = [] if dataset == 'mirflickr': image_names, _, image_matches = mirflickr_images() absolute_matches = {} for label, s in results.iteritems(): absolute_matches[label] = {} for target_image, retrieved in s.iteritems(): if dataset == 'ukbench': retrieved_filenames = retrieved[:3] recall_value = recall(target_image, retrieved_filenames, retrieved, method=result_test_ukbench ) precision_value = precision(target_image, retrieved_filenames, retrieved, method=result_test_ukbench ) recalls[label][target_image] = recall_value precisions[label][target_image] = precision_value relevant, retrieved = result_test_ukbench(target_image, retrieved_filenames, retrieved) elif dataset == 'mirflickr': retrieved_filenames = retrieved[:image_matches[target_image]] recall_value = recall(target_image, retrieved_filenames, retrieved,
y = df.iloc[:, 4].values # Aplicando a normalização/escalonamento das features X = feature_scaling(X) # Definindo os valores da matriz de confusão: tp, fp, fn, tn = [1, 1, 1, 2] # Calculando os valores da acurácia accuracy(tp, fp, fn, tn) # Calculando os valores da precisão precision(tp, fp) # Calculando os valores do recall recall(tp, fn) # Calculando os valores do informedness informedness(tp, fp, fn, tn) # Calculando os valores do markdness markdness(tp, fp, fn, tn) # Definindo os valores para calculo do ROC-AUC # Valores reais (ground truth) y_true = np.array([1, 1, 2, 2]) #Probabilidade de estimar as classes positivas: y_scores = np.array([0.1, 0.4, 0.35, 0.8]) # A probabilidade de predição de cada classe retornada por um classificador: