def val_epoch(epoch, data_loader, model, criterion, opt, logger): print('validation at epoch {}'.format(epoch)) model.eval() batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() accuracies = AverageMeter() precisions = AverageMeter() # recalls = AverageMeter() printer = 50 if opt.dataset == 'jester' else 5 end_time = time.time() for i, (inputs, targets) in enumerate(data_loader): data_time.update(time.time() - end_time) if not opt.no_cuda: targets = targets.cuda(async=True) with torch.no_grad(): inputs = Variable(inputs) targets = Variable(targets) outputs = model(inputs) loss = criterion(outputs, targets) acc = calculate_accuracy(outputs, targets) precision = calculate_precision(outputs, targets) # recall = calculate_recall(outputs, targets) losses.update(loss.item(), inputs.size(0)) accuracies.update(acc, inputs.size(0)) precisions.update(precision, inputs.size(0)) recalls.update(recall, inputs.size(0)) batch_time.update(time.time() - end_time) end_time = time.time() if i % printer == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.5f} ({batch_time.avg:.5f})\t' 'Data {data_time.val:.5f} ({data_time.avg:.5f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc {acc.val:.3f} ({acc.avg:.3f})'.format( epoch, i + 1, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses, acc=accuracies)) logger.log({ 'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg, 'precision': precisions.avg, 'recall': recalls.avg }) return losses.avg, accuracies.avg
def score_claim(claim): cleaned_claim = claim.replace("/", " ") choices = utils.query_customized_lucene(cleaned_claim, k=max(k), jar_name=args.jar) retrieved = utils.process_lucene_output(choices) relevant = claim_to_article[claim] mAP = {} for i in k: precision = utils.calculate_precision(retrieved=retrieved, relevant=relevant, k=i) recall = utils.calculate_recall(retrieved=retrieved, relevant=relevant, k=i) mAP[i] = {} mAP[i]['precision'] = precision mAP[i]['recall'] = recall return mAP
def train_epoch(epoch, data_loader, model, criterion, optimizer, opt, epoch_logger, batch_logger): print('train at epoch {}'.format(epoch)) model.train() batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() accuracies = AverageMeter() precisions = AverageMeter() # recalls = AverageMeter() end_time = time.time() # i, (inputs, targets) = next(iter(enumerate(data_loader))) for i, (inputs, targets) in enumerate(data_loader): data_time.update(time.time() - end_time) if not opt.no_cuda: #targets = targets.cuda(non_blocking=True) targets = targets.to('cuda:0', non_blocking=True) inputs = inputs.to('cuda:0', non_blocking=True) #print("NO CUDA") inputs = Variable(inputs) targets = Variable(targets) #pdb.set_trace() #print(model) #print(inputs) outputs = model(inputs) loss = criterion(outputs, targets) acc = calculate_accuracy(outputs, targets) precision = calculate_precision(outputs, targets) # recall = calculate_recall(outputs, targets) losses.update(loss.item(), inputs.size(0)) accuracies.update(acc, inputs.size(0)) precisions.update(precision, inputs.size(0)) recalls.update(recall, inputs.size(0)) optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end_time) end_time = time.time() batch_logger.log({ 'epoch': epoch, 'batch': i + 1, 'iter': (epoch - 1) * len(data_loader) + (i + 1), 'loss': losses.val, 'acc': accuracies.val, 'precision': precisions.val, 'recall': recalls.val, 'lr': optimizer.param_groups[0]['lr'] }) if i % 10 == 0: print('Epoch: [{0}][{1}/{2}]\t lr: {lr:.5f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc {acc.val:.3f} ({acc.avg:.3f})\t' 'Precision {precision.val:.3f}({precision.avg:.3f})\t' 'Recall {recall.val:.3f}({recall.avg:.3f})'.format( epoch, i, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses, lr=optimizer.param_groups[0]['lr'], acc=accuracies, precision=precisions, recall=recalls)) epoch_logger.log({ 'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg, 'precision': precisions.avg, 'recall': recalls.avg, 'lr': optimizer.param_groups[0]['lr'] })
def __init__(self, words_per_document, num_classes, vocabulary_size, embedding_size, filter_sizes, num_filters, l2_reg_lambda, train_size, batch_size): # Placeholders self.emb_place_holder = tf.placeholder(tf.float32, [None, embedding_size], name="emb_place_holder") self.x_place_holder = tf.placeholder(tf.int32, [None, words_per_document], name="x") self.y_place_holder = tf.placeholder(tf.float32, [None, num_classes], name="labels") self.dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob") self.learning_rate = tf.placeholder(tf.float32, name="learning_rate") self.decay_rate = tf.placeholder(tf.float32, name="decay_rate") l2_loss = tf.constant(0.0, name="l2_loss") # First Layer Embeddings [MiniBatchSize, Largest Size, embSize] with tf.device('/cpu:0'): self.W = tf.Variable(tf.constant( 0.0, shape=[vocabulary_size, embedding_size]), trainable=True, name="W") embedding_init = self.W.assign(value=self.emb_place_holder) self.embedded_chars = tf.nn.embedding_lookup( embedding_init, self.x_place_holder) self.embedded_chars_expanded = tf.expand_dims( self.embedded_chars, -1) # Second Layer Convolutional filter size [3, 128] [4, 128] [5, 128] pooled_outputs = [] for i, filter_size in enumerate(filter_sizes): with tf.name_scope("conv-maxpool-%s" % filter_size): filter_shape = [filter_size, embedding_size, 1, num_filters] W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W") b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b") conv = tf.nn.conv2d(self.embedded_chars_expanded, W, strides=[1, 1, 1, 1], padding="VALID", name="conv") h = tf.nn.relu(tf.nn.bias_add(conv, b), name="h") # Maxpooling layer pooled = tf.nn.max_pool( h, ksize=[1, words_per_document - filter_size + 1, 1, 1], strides=[1, 1, 1, 1], padding="VALID", name="pool") tf.summary.histogram("weights", W) tf.summary.histogram("biases", b) tf.summary.histogram("activations", h) pooled_outputs.append(pooled) num_filters_total = num_filters * len(filter_sizes) self.h_pool = tf.concat(pooled_outputs, 3) self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total]) # Dropout with tf.name_scope("dropout"): self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob) tf.summary.histogram("dropout", self.h_drop) # Last layer # xw_plus_b: compute matmul(x, weights) + biases with tf.name_scope("output"): W = tf.Variable(tf.truncated_normal( [num_filters_total, num_classes], stddev=0.1), name="W") b = tf.Variable(tf.constant(0.1, shape=[num_classes]), name="b") self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name="scores") self.predictions = tf.argmax(self.scores, 1, name="predictions") l2_loss += tf.nn.l2_loss(W, name="l2_loss") l2_loss += tf.nn.l2_loss(b, name="l2_loss") tf.summary.histogram("l2", l2_loss) tf.summary.histogram("weigths", W) tf.summary.histogram("biases", b) # Loss function # cross entropy between what we got and our labels with tf.name_scope("loss"): cross_entropy_r = tf.nn.softmax_cross_entropy_with_logits( logits=self.scores, labels=self.y_place_holder) self.cross_entropy = tf.reduce_mean( cross_entropy_r, name="cross_entropy") + (l2_reg_lambda * l2_loss) self.stream_loss, self.stream_loss_update = tf.contrib.metrics.streaming_mean( self.cross_entropy) tf.summary.scalar("loss_tr", self.stream_loss) # tf.summary.scalar("loss_tr", self.cross_entropy) # Train Step # using learning rate 0.0004 # minimize is it the same as compute_gradients and apply gradients with tf.name_scope("train"): self.global_step_ = tf.Variable(0, name="global_step", trainable=False) lr = tf.train.exponential_decay(self.learning_rate, self.global_step_ * batch_size, train_size, self.decay_rate) self.train_step = tf.train.AdamOptimizer(lr).minimize( self.cross_entropy, global_step=self.global_step_, name="train_oper") with tf.name_scope("accuracy"): correct_prediction = tf.equal(self.predictions, tf.argmax(self.y_place_holder, 1), name="correct_prediction") self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name="accuracy") self.stream_accuracy, self.stream_accuracy_update = tf.contrib.metrics.streaming_mean( self.accuracy) tf.summary.scalar("accuracy", self.stream_accuracy) # tf.summary.scalar("accuracy", self.accuracy) with tf.name_scope("confussion_matrix"): labels_max = tf.argmax(self.y_place_holder, 1, name="label_max") # self.matrix = tf.contrib.metrics.confusion_matrix(labels_max, self.predictions, num_classes=2, name="matrix") self.matrix = tf.confusion_matrix(labels_max, self.predictions, num_classes=2, name="matrix") true_positive = self.matrix[1, 1] true_negative = self.matrix[0, 0] false_positive = self.matrix[0, 1] false_negative = self.matrix[1, 0] self.precision_mini_batch = utils.calculate_precision( true_positive, false_positive) self.recall_mini_batch = utils.calculate_recall( true_positive, false_negative) self.f1_score_min_batch = utils.calculate_f1( self.precision_mini_batch, self.recall_mini_batch) self.stream_precision, self.stream_precision_update = tf.contrib.metrics.streaming_mean( self.precision_mini_batch) self.stream_recall, self.stream_recall_update = tf.contrib.metrics.streaming_mean( self.recall_mini_batch) self.stream_f1, self.stream_f1_update = tf.contrib.metrics.streaming_mean( self.f1_score_min_batch) tf.summary.scalar("Precision", self.stream_precision) tf.summary.scalar("Recall", self.stream_recall) tf.summary.scalar("F1", self.stream_f1) # tf.summary.scalar("Precision", self.precision_mini_batch) # tf.summary.scalar("Recall", self.recall_mini_batch) # tf.summary.scalar("F1", self.f1_score_min_batch) # if should_load: # log("Data processing ok load network...") # saver = tf.train.Saver() # try: # saver.restore(sess, checkpoint_file_path) # except Exception as e: # log("Not able to load file") # summaries self.summary = tf.summary.merge_all() self.accuracy_validation_placeholder = tf.placeholder( tf.float32, name="acc_val_placeholder") self.loss_validation_placeholder = tf.placeholder( tf.float32, name="loss_val_placeholder") self.precision_validation_placeholder = tf.placeholder( tf.float32, name="prec_val_placeholder") self.recall_validation_placeholder = tf.placeholder( tf.float32, name="recall_val_placeholder") self.f1_validation_placeholder = tf.placeholder( tf.float32, name="f1_val_placeholder") with tf.name_scope("validation"): self.acc_validation_mean = tf.reduce_mean( self.accuracy_validation_placeholder) self.loss_validation_mean = tf.reduce_mean( self.loss_validation_placeholder) self.prec_validation_mean = tf.reduce_mean( self.precision_validation_placeholder) self.recall_validation_mean = tf.reduce_mean( self.recall_validation_placeholder) self.f1_validation_mean = tf.reduce_mean( self.f1_validation_placeholder) loss_val = tf.summary.scalar("loss_val", self.loss_validation_mean) accuracy_val = tf.summary.scalar("accuracy_val", self.acc_validation_mean) precission_val = tf.summary.scalar("Precision_val", self.prec_validation_mean) recall_val = tf.summary.scalar("Recall_val", self.recall_validation_mean) f1_val = tf.summary.scalar("F1_val", self.f1_validation_mean) self.summary_val = tf.summary.merge( [loss_val, accuracy_val, precission_val, recall_val, f1_val])
inputs = Variable(inputs) targets = Variable(targets) outputs = model(inputs) if not opt.no_softmax_in_test: outputs = F.softmax(outputs) recorder.append(outputs.data.cpu().numpy().copy()) y_true.extend(targets.cpu().numpy().tolist()) y_pred.extend(outputs.argmax(1).cpu().numpy().tolist()) #outputs = torch.unsqueeze(torch.mean(outputs, 0), 0) #pdb.set_trace() # print(outputs.shape, targets.shape) if outputs.size(1) <= 4: prec1 = calculate_accuracy(outputs, targets, topk=(1, )) precision = calculate_precision(outputs, targets) # recall = calculate_recall(outputs, targets) top1.update(prec1[0], inputs.size(0)) precisions.update(precision, inputs.size(0)) recalls.update(recall, inputs.size(0)) batch_time.update(time.time() - end_time) end_time = time.time() print('[{0}/{1}]\t' 'Time {batch_time.val:.5f} ({batch_time.avg:.5f})\t' 'prec@1 {top1.avg:.5f} \t' 'precision {precision.val:.5f} ({precision.avg:.5f})\t' 'recall {recall.val:.5f} ({recall.avg:.5f})'.format( i + 1,
def test_calculate_precision(self): precision_value = calculate_precision(self.relevant_entries, self.retrieved_entries) assert precision_value >= 0.85 and precision_value <= 0.858
def testing_step(checkpoint_dir, batch_size, x_test, y_test, vocab_inv_emb_dset, LOG_FILE, parameters, early_stop_log): checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) graph = tf.Graph() with graph.as_default(): session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) sess = tf.Session(config=session_conf) with sess.as_default(): saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) # all_vars = graph.get_operations() x_placeholder = graph.get_operation_by_name("x").outputs[0] y_placeholder = graph.get_operation_by_name("labels").outputs[0] embedding_placeholder = graph.get_operation_by_name("emb_place_holder").outputs[0] dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0] accuracies = graph.get_operation_by_name("accuracy/accuracy").outputs[0] loss = graph.get_operation_by_name("loss/cross_entropy").outputs[0] label_match = graph.get_operation_by_name("confussion_matrix/label_max").outputs[0] predictions = graph.get_operation_by_name("output/predictions").outputs[0] l2_loss = graph.get_operation_by_name("l2_loss").outputs[0] testing_data = (Dataset.from_tensor_slices((x_test, y_test)) .shuffle(buffer_size=10) .batch(parameters.batch_size) .make_initializable_iterator()) next_element_testing = testing_data.get_next() epochs = 1 step = 0 accuracies_test = [] losses_test = [] precisions_test = [] recalls_test = [] f1_scores_test = [] for actual_epoch in range(epochs): sess.run(testing_data.initializer) while True: try: batch_testing = sess.run(next_element_testing) feed_dict = {x_placeholder: batch_testing[0], y_placeholder: batch_testing[1], embedding_placeholder: vocab_inv_emb_dset, dropout_keep_prob: 1.0} acc_batch, loss_batch, label_op, pred_op, l2_loss_op = \ sess.run([accuracies, loss, label_match, predictions, l2_loss], feed_dict) matrix_batch = confusion_matrix(label_op, pred_op) true_positive = matrix_batch[1, 1] true_negative = matrix_batch[0, 0] false_positive = matrix_batch[0, 1] false_negative = matrix_batch[1, 0] precision_mini_batch = utils.calculate_precision(true_positive, false_positive) recall_mini_batch = utils.calculate_recall(true_positive, false_negative) f1_score_min_batch = utils.calculate_f1(precision_mini_batch, recall_mini_batch) accuracies_test.append(acc_batch) losses_test.append(loss_batch) precisions_test.append(precision_mini_batch) recalls_test.append(recall_mini_batch) f1_scores_test.append(f1_score_min_batch) log("Step " + str(step) + "(epoch " + str(epochs) + ")" + "Test accuracy: " + str( acc_batch) + " test loss: " + str(loss_batch) + " test precission: " + str(precision_mini_batch) + " test recall: " + str(recall_mini_batch) + "test F1: " + str(f1_score_min_batch), LOG_FILE) except tf.errors.OutOfRangeError: avg_accuracy = np.mean(accuracies_test) avg_losses = np.mean(losses_test) avg_precision = np.mean(precisions_test) avg_recall = np.mean(recalls_test) avg_f1 = np.mean(f1_scores_test) log(str(parameters), LOG_FILE) log("Final results, test accuracy: " + str(avg_accuracy) + " test loss: " + str(avg_losses) + " test precission: " + str(avg_precision) + " test recall: " + str( avg_recall) + " test f1: " + str(avg_f1), LOG_FILE) log("End training dataset epoch: " + str(early_stop_log), LOG_FILE) break
def train_epoch(epoch, data_loader, model, criterion, optimizer, opt, epoch_logger, batch_logger, global_step, tsboard_writer=None): print('train at epoch {}'.format(epoch)) sys.stdout.flush() model.train() batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() accuracies = AverageMeter() precisions = AverageMeter() # recalls = AverageMeter() printer = 100 if opt.dataset == 'jester' else 10 end_time = time.time() # i, (inputs, targets) = next(iter(enumerate(data_loader))) for i, (inputs, gravity_position, targets) in enumerate(data_loader): data_time.update(time.time() - end_time) if not opt.no_cuda: targets = targets.cuda(non_blocking=True) inputs = Variable(inputs) positions = Variable(gravity_position) targets = Variable(targets) #pdb.set_trace() outputs = model(inputs, positions) loss = criterion(outputs, targets) acc = calculate_accuracy(outputs, targets) precision = calculate_precision(outputs, targets) # recall = calculate_recall(outputs, targets) losses.update(loss.item(), inputs.size(0)) accuracies.update(acc, inputs.size(0)) precisions.update(precision, inputs.size(0)) recalls.update(recall, inputs.size(0)) optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end_time) end_time = time.time() batch_logger.log({ 'epoch': epoch, 'batch': i + 1, 'iter': (epoch - 1) * len(data_loader) + (i + 1), 'loss': losses.val, 'acc': accuracies.val, 'precision': precisions.val, 'recall': recalls.val, 'lr': optimizer.param_groups[0]['lr'] }) if i % printer == 0: print('Epoch: [{0}][{1}/{2}]\t lr: {lr:.5f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc {acc.val:.3f} ({acc.avg:.3f})\t' 'Precision {precision.val:.3f}({precision.avg:.3f})\t' 'Recall {recall.val:.3f}({recall.avg:.3f})'.format( epoch, i, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses, lr=optimizer.param_groups[0]['lr'], acc=accuracies, precision=precisions, recall=recalls)) sys.stdout.flush() if tsboard_writer is not None: tsboard_writer.add_scalar("training loss", losses.val, global_step) tsboard_writer.add_scalar("training acc", accuracies.val, global_step) tsboard_writer.add_scalar("training precision", precisions.val, global_step) tsboard_writer.add_scalar("training recall", recalls.val, global_step) global_step += 1 epoch_logger.log({ 'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg, 'precision': precisions.avg, 'recall': recalls.avg, 'lr': optimizer.param_groups[0]['lr'] }) return global_step
def main(**kwargs): fp16 = kwargs['fp16'] batch_size = kwargs['batch_size'] num_workers = kwargs['num_workers'] pin_memory = kwargs['pin_memory'] patience_limit = kwargs['patience'] initial_lr = kwargs['learning_rate'] gradient_accumulation_steps = kwargs['grad_accumulation_steps'] device = kwargs['device'] epochs = kwargs['epochs'] freeze_bert = kwargs['freeze_bert_layers'] slots = kwargs['slots'] if fp16: scaler = GradScaler() tokenizer = BertTokenizer.from_pretrained( 'bert-base-uncased', model_max_length=128 ) # for TM_1, out of 303066 samples, 5 are above 128 tokens if kwargs['dataset'] == "TM": train_data, val_data = load_taskmaster_datasets( utils.datasets, tokenizer, train_percent=0.9, for_testing_purposes=kwargs['testing_for_bugs']) if kwargs['dataset'] == "MW": train_data = load_multiwoz_dataset("multi-woz/train_dials.json", tokenizer, slots, kwargs['testing_for_bugs']) val_data = load_multiwoz_dataset("multi-woz/dev_dials.json", tokenizer, slots, kwargs['testing_for_bugs']) if kwargs['dataset'] == "MW22": train_data = load_MW_22_dataset_training(tokenizer, slots, kwargs['testing_for_bugs']) val_data = load_MW_22_dataset_validation(tokenizer, slots, kwargs['testing_for_bugs']) train_dataset = VE_dataset(train_data) val_dataset = VE_dataset(val_data) collator = collate_class(tokenizer.pad_token_id) train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collator, num_workers=num_workers, pin_memory=pin_memory) val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=int(batch_size / 2), shuffle=True, collate_fn=collator, num_workers=num_workers, pin_memory=pin_memory) # BertForValueExtraction if kwargs['model_path'] and os.path.isdir(kwargs['model_path']): from_pretrained = kwargs['model_path'] else: from_pretrained = 'bert-base-uncased' model = BertForValueExtraction(num_labels=len(label2id.keys()), from_pretrained=from_pretrained, freeze_bert=freeze_bert) model.to(device) model.train() optimizer = torch.optim.Adam(params=model.parameters(), lr=initial_lr) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'max', factor=0.5, patience=1, min_lr=initial_lr / 100, verbose=True) best_acc, count = 0, 0 for epoch in range(epochs): # train loop total_loss = 0 optimizer.zero_grad() pbar = tqdm(enumerate(train_dataloader), total=len(train_dataloader)) for i, batch in pbar: input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) token_type_ids = batch['token_type_ids'].to(device) labels = batch['labels'].to(device) text = batch['text'] # print(f"input_ids: {input_ids.shape} - {input_ids}") # print(f"attention_mask: {attention_mask.shape} - {attention_mask}") # print(f"token_type_ids: {token_type_ids.shape} - {token_type_ids}") # print(f"labels: {labels.shape} - {labels}") if fp16: with autocast(): loss = model.calculate_loss(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, labels=labels) total_loss += loss.item() loss = loss / gradient_accumulation_steps scaler.scale(loss).backward() else: loss = model.calculate_loss(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, labels=labels) loss.backward() total_loss += loss.item() if ((i + 1) % gradient_accumulation_steps) == 0: if fp16: scaler.step(optimizer) scaler.update() else: optimizer.step() optimizer.zero_grad() batch_num = ((i + 1) / gradient_accumulation_steps) pbar.set_description(f"Loss: {total_loss/batch_num:.4f}") # validation loop model.eval() with torch.no_grad(): TP, FP, FN, TN = model.evaluate(val_dataloader, device) pr = utils.calculate_precision(TP, FP) re = utils.calculate_recall(TP, FN) F1 = utils.calculate_F1(TP, FP, FN) acc = utils.calculate_accuracy(TP, FP, FN, TN) balanced_acc = utils.calculate_balanced_accuracy(TP, FP, FN, TN) print( f"Validation: pr {pr:.4f} - re {re:.4f} - F1 {F1:.4f} - acc {acc:.4f} - balanced acc {balanced_acc:.4f}" ) scheduler.step(balanced_acc) if balanced_acc > best_acc: best_acc = balanced_acc count = 0 if kwargs['model_path']: model.save_(f"{kwargs['model_path']}-ACC{best_acc:.4f}") else: count += 1 if count == patience_limit: print("ran out of patience stopping early") break model.train()
def val_epoch_true(epoch, data_loader, model, criterion, opt, logger, global_step, tsboard_writer=None): print('validation at epoch {}'.format(epoch)) model.eval() batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() accuracies = AverageMeter() precisions = AverageMeter() # recalls = AverageMeter() end_time = time.time() vid_ids = data_loader.dataset.ids vid_ids.append(-1) out_queue = [] for i, (inputs, targets) in enumerate(data_loader): data_time.update(time.time() - end_time) if not opt.no_cuda: targets = targets.cuda(non_blocking=True) with torch.no_grad(): inputs = Variable(inputs) targets = Variable(targets) outputs = model(inputs) outputs = F.softmax(outputs, dim=1) outputs = outputs.cpu().numpy()[0].reshape(-1, ) out_queue.append(outputs) if vid_ids[i + 1] != vid_ids[i]: output = np.mean(out_queue, 0) outputs = torch.from_numpy(output).float().unsqueeze_(0).cuda() loss = criterion(outputs, targets) acc = calculate_accuracy(outputs, targets) precision = calculate_precision(outputs, targets) recall = calculate_recall(outputs, targets) losses.update(loss.item(), inputs.size(0)) accuracies.update(acc, inputs.size(0)) precisions.update(precision, inputs.size(0)) recalls.update(recall, inputs.size(0)) batch_time.update(time.time() - end_time) end_time = time.time() out_queue = [] if i % 100 == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.5f} ({batch_time.avg:.5f})\t' 'Data {data_time.val:.5f} ({data_time.avg:.5f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc {acc.val:.3f} ({acc.avg:.3f})'.format( epoch, i + 1, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses, acc=accuracies)) logger.log({ 'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg, 'precision': precisions.avg, 'recall': recalls.avg }) if tsboard_writer is not None: tsboard_writer.add_scalar("validation loss", losses.avg, global_step) tsboard_writer.add_scalar("validation acc", accuracies.avg, global_step) tsboard_writer.add_scalar("validation precision", precisions.avg, global_step) tsboard_writer.add_scalar("validation recall", recalls.avg, global_step) return losses.avg, accuracies.avg
def test(data_loader, model, opt, class_names): print('test') model.eval() batch_time = AverageMeter() data_time = AverageMeter() end_time = time.time() output_buffer = [] previous_video_id = '' test_results = {'results': {}} total_output_buffer = [] total_target_buffer = [] for i, (inputs, positions, targets) in enumerate(data_loader): data_time.update(time.time() - end_time) with torch.no_grad(): inputs = Variable(inputs) positions = Variable(positions) targets = Variable(targets).cuda() outputs = model(inputs, positions) if not opt.no_softmax_in_test: outputs = F.softmax(outputs, dim=1) total_output_buffer.append(outputs) total_target_buffer.append(targets) for j in range(outputs.size(0)): if not (i == 0 and j == 0) and targets[j].item() != previous_video_id: calculate_video_results(output_buffer, previous_video_id, test_results, class_names) output_buffer = [] output_buffer.append(outputs[j].data.cpu()) previous_video_id = targets[j].item() if (i % 100) == 0: with open( os.path.join(opt.result_path, '{}.json'.format(opt.test_subset)), 'w') as f: json.dump(test_results, f) batch_time.update(time.time() - end_time) end_time = time.time() print('[{}/{}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'.format( i + 1, len(data_loader), batch_time=batch_time, data_time=data_time)) total_output = torch.cat(total_output_buffer, dim=0) total_target = torch.cat(total_target_buffer, dim=0) acc = calculate_accuracy(total_output, total_target) precision = calculate_precision(total_output, total_target) recall = calculate_recall(total_output, total_target) print("Overall:\t acc: {}, precision: {}, recall: {}".format( acc, precision, recall)) with open(os.path.join(opt.result_path, '{}.json'.format(opt.test_subset)), 'w') as f: json.dump(test_results, f)
text = batch['text'] outputs = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids) logits = outputs.logits preds = torch.max(logits, dim=2)[1] tp, fp, fn, tn = model.evaluate(preds.tolist(), labels.tolist(), attention_mask.tolist()) TP += tp FP += fp FN += fn TN += tn pr = utils.calculate_precision(TP, FP) re = utils.calculate_recall(TP, FN) F1 = utils.calculate_F1(TP, FP, FN) acc = utils.calculate_accuracy(TP, FP, FN, TN) balanced_acc = utils.calculate_balanced_accuracy(TP, FP, FN, TN) print(f"Validation: pr {pr:.4f} - re {re:.4f} - F1 {F1:.4f} - acc {acc:.4f} - balanced acc {balanced_acc:.4f}") scheduler.step(balanced_acc) if balanced_acc > best_acc: best_acc = balanced_acc count = 0 else: count += 1 print(count) if count == patience_limit:
def test(self, annotation_path='', video_path=''): opt = self.opt if annotation_path != '': opt.annotation_path = annotation_path if opt.root_path != '': opt.annotation_path = os.path.join(opt.root_path, opt.annotation_path) # if video_path != '': # opt.video_path = video_path # if opt.root_path != '': # opt.video_path = os.path.join(opt.root_path, opt.video_path) if not os.path.exists(opt.result_path): os.makedirs(opt.result_path) with open(os.path.join(opt.result_path, 'opts.json'), 'w') as opt_file: json.dump(vars(opt), opt_file) if opt.no_mean_norm and not opt.std_norm: norm_method = Normalize([0, 0, 0], [1, 1, 1]) elif not opt.std_norm: norm_method = Normalize(opt.mean, [1, 1, 1]) else: norm_method = Normalize(opt.mean, opt.std) # original spatial_transform = Compose([ #Scale(opt.sample_size), Scale(112), CenterCrop(112), ToTensor(opt.norm_value), norm_method ]) temporal_transform = TemporalCenterCrop(opt.sample_duration) target_transform = ClassLabel() test_data = get_test_set(opt, spatial_transform, temporal_transform, target_transform) test_loader = torch.utils.data.DataLoader(test_data, batch_size=opt.batch_size, shuffle=False, num_workers=opt.n_threads, pin_memory=True) test_logger = Logger(os.path.join(opt.result_path, 'test.log'), ['top1', 'precision', 'recall']) if opt.resume_path: print('loading checkpoint {}'.format(opt.resume_path)) checkpoint = torch.load(opt.resume_path) assert opt.arch == checkpoint['arch'] opt.begin_epoch = checkpoint['epoch'] self.model.load_state_dict(checkpoint['state_dict']) recorder = [] self.model.eval() batch_time = AverageMeter() top1 = AverageMeter() precisions = AverageMeter() recalls = AverageMeter() y_true = [] y_pred = [] end_time = time.time() for i, (inputs, targets) in enumerate(test_loader): if not opt.no_cuda: targets = targets.cuda(non_blocking=True) #inputs = Variable(torch.squeeze(inputs), volatile=True) with torch.no_grad(): inputs = Variable(inputs) targets = Variable(targets) outputs = self.model(inputs) if not opt.no_softmax_in_test: outputs = F.softmax(outputs, dim=1) recorder.append(outputs.data.cpu().numpy().copy()) y_true.extend(targets.cpu().numpy().tolist()) y_pred.extend(outputs.argmax(1).cpu().numpy().tolist()) _cls = outputs.argmax(1).cpu().numpy().tolist()[0] prec1 = self.calculate_accuracy(outputs, targets, topk=(1, )) precision = calculate_precision(outputs, targets) recall = calculate_recall(outputs, targets) top1.update(prec1[0], inputs.size(0)) precisions.update(precision, inputs.size(0)) recalls.update(recall, inputs.size(0)) batch_time.update(time.time() - end_time) end_time = time.time() test_logger.log({ 'top1': top1.avg, 'precision': precisions.avg, 'recall': recalls.avg }) print('-----Evaluation is finished------') print('Overall Prec@1 {:.05f}%'.format(top1.avg * 100)) return y_pred, y_true, test_data
if __name__ == "__main__": orig_settings = termios.tcgetattr(sys.stdin) tty.setcbreak(sys.stdin) # load and init data data = pd.read_csv("data/ex2data1.txt", ",") X = data.iloc[:, :-1].to_numpy() y = data.iloc[:, -1].to_numpy() X = utils.map_feature(X[:, 0], X[:, 1], 2) weights = zeros(X.shape[1]) print( 'Menu:\n1) Press G for gradient descent.\n2) Press O for optimized algorithm.\n3) Press Q for quit.' ) key = sys.stdin.read(1)[0] if key == 'g' or key == 'G': print("Run gradient descent") weights = gradient_descent(weights, X, y, 500000, 0.00101, 0.003) utils.calculate_precision(weights, X, y) elif key == 'o' or key == 'O': print("Run optimized algorithm") weights, _, _ = opt.fmin_tnc(func=cost_function_regularized, x0=weights, args=(X, y, 0.1), messages=0) utils.calculate_precision(weights, X, y) elif key == 'q' or key == 'Q': exit()