def validate(data_loader, model, criterion, device, epoch, args): losses = AverageMeter() accuracy = AverageMeter() model.eval() with torch.no_grad(): for idx, (input_seq, target) in tqdm(enumerate(data_loader), total=len(data_loader)): input_seq = input_seq.to(device) target = target.to(device) B = input_seq.size(0) output, _ = model(input_seq) [_, N, D] = output.size() output = output.view(B * N, D) target = target.repeat(1, N).view(-1) loss = criterion(output, target) acc = calc_accuracy(output, target) losses.update(loss.item(), B) accuracy.update(acc.item(), B) print('Loss {loss.avg:.4f}\t' 'Acc: {acc.avg:.4f} \t'.format(loss=losses, acc=accuracy)) args.writer_val.add_scalar('global/loss', losses.avg, epoch) args.writer_val.add_scalar('global/accuracy', accuracy.avg, epoch) return losses.avg, accuracy.avg
def eval_model(embedding_dir, label_dir, sense, model_dir, target_languages, device): result_dict = defaultdict(float) net = relation_classifier(1024 * 5, output_dim).to(device) net.load_state_dict(torch.load(model_dir, map_location=device)) for test_set in target_languages: if test_set == "ted": languages = ["tr", "de", "pl", "ru", "pt", "en", "lt"] for lang in languages: ted_test_arg1, ted_test_arg2, ted_test_target = read_laser_embeddings(embedding_dir, label_dir, lang, device, sense=sense, set_type="test") test_pred = net(ted_test_arg1, ted_test_arg2) # predict acc, f1, prec, rec, base_acc, base_f1 = calc_accuracy(ted_test_target, test_pred, verbose=False) result_dict[lang] = (f1) else: test_arg1, test_arg2, test_target = read_laser_embeddings(embedding_dir, label_dir, test_set, device, sense=sense, set_type="test") test_pred = net(test_arg1, test_arg2) # predict acc, f1, prec, rec, base_acc, base_f1 = calc_accuracy(test_target, test_pred, verbose=False) result_dict[test_set] = (f1) return result_dict
def train(embeddings_dir, label_dir, out_dir, training_langs, sense_list, epoch_num, batch_size, early_stopping_threshold, device, verbose = False): for sense in sense_list: print("~~~ Training the \" %s vs Others \" classifier" % (sense)) best_score = -1 score_list = [] out_file = out_dir + "/%s_%s_model" % ("_".join(training_langs), sense) if not os.path.exists(out_dir): os.makedirs(out_dir) # Prepare Data dev_arg1, dev_arg2, dev_label = prepare_input(embeddings_dir, label_dir, training_langs, sense, device, type="dev") # model input_dim = dev_arg1.shape[1]*5 # *5 due to the dissent model network = relation_classifier(input_dim).to(device) optimizer = optim.Adagrad(network.parameters(), lr=0.01) loss_fn = nn.CrossEntropyLoss() dev_label = dev_label.long() if verbose: print(network, "\n\n") # training loop for epoch in range(epoch_num): train_arg1, train_arg2, train_label = prepare_input(embeddings_dir, label_dir, training_langs, sense, device, type="training") train_label = train_label.long() for batch in range(int(len(train_label)/batch_size)+1): arg1 = train_arg1[batch*batch_size:min((batch+1)*batch_size, len(train_arg1))] arg2 = train_arg1[batch*batch_size:min((batch+1)*batch_size, len(train_arg2))] batch_labels = train_label[batch*batch_size:min((batch+1)*batch_size, len(train_label))] optimizer.zero_grad() y_pred = network(arg1, arg2) output = loss_fn(y_pred, batch_labels) output.backward() optimizer.step() ## compute validation loss valid_predictions = network(dev_arg1, dev_arg2) acc, f1, prec, rec, base_acc, base_f1 = calc_accuracy(dev_label, valid_predictions, verbose=False) if verbose: print("Epoch {} Validation acc: {:.7f} f1: {:.7f}".format(epoch, acc, f1), sep=" ", flush=True) elif epoch % 10 == 0: print("Epoch {} Validation acc: {:.7f} f1: {:.7f}".format(epoch, acc, f1), sep=" ", flush=True) ## early stopping if f1 > best_score: torch.save(network.state_dict(), out_file) if verbose: print("model saved at Epoch", epoch) best_score = f1 score_list = [] else: score_list.append(f1) if len(score_list) > early_stopping_threshold: print("Early Stopping after {} iteration".format( epoch)) print("the highest F-score achieved on the validation data: %.4f" % best_score) print("The model is saved to", out_file) print("--------------------") break
print("Weights Sum:", utils.sum_model_weights(model)[1]) # Get the predicted class probabilities, labels & probabilities of abnormality preds = model.predict(dataset_valid_noshuffle, steps=num_steps_per_epoch_valid) print("Preds Shape:", preds.shape) pred_probs_valid, pred_labels_valid, pred_probs_abnormal_valid = utils.get_predictions( dataset=dataset_valid_noshuffle, model=model, steps=num_steps_per_epoch_valid) print("Pred Labels Shape:", pred_labels_valid.shape) ### Tensorflow no longer required, so come out of the session # Image-wise accuracy & cross-entropy loss accuracy_valid = utils.calc_accuracy(labels_valid_scalar, pred_labels_valid) loss_valid = utils.calc_crossentropy_loss(labels_valid_onehot, pred_probs_valid) print("ACCURACY VALID:", accuracy_valid) print("LOSS VALID:", loss_valid) # Study-wise performence # Breakdowns of numbers of studies, from the orig MURA paper (Table 1, p3) - use these as a check on the numbers of studies we derive from our data num_studies_published_dict = utils.get_num_studies_published() images_accuracy_check, studies_summary_valid = utils.get_study_predictions( images_summary_valid, pred_probs_abnormal_valid) # Check image-wise accuracy to ensure image-wise results not reordered when calculating study-wise results if (images_accuracy_check != accuracy_valid): print("Imagewise Accuracy Check Failed - Have Images Been Reordered?")
def train_one_epoch(data_loader, model, criterion, optimizer, device, epoch, args): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() accuracy = AverageMeter() if args.train_what == 'last': model.eval() model.module.final_bn.train() model.module.final_fc.train() print('[Warning] train model with eval mode, except final layer') else: model.train() end = time.time() tic = time.time() for idx, (input_seq, target) in enumerate(data_loader): data_time.update(time.time() - end) input_seq = input_seq.to(device) target = target.to(device) B = input_seq.size(0) output, _ = model(input_seq) [_, N, D] = output.size() output = output.view(B * N, D) target = target.repeat(1, N).view(-1) loss = criterion(output, target) acc = calc_accuracy(output, target) losses.update(loss.item(), B) accuracy.update(acc.item(), B) optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end) end = time.time() if idx % args.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Loss {loss.val:.4f} ({loss.local_avg:.4f})\t' 'Acc: {acc.val:.4f} ({acc.local_avg:.4f})\t' 'T-data:{dt.val:.2f} T-batch:{bt.val:.2f}\t'.format( epoch, idx, len(data_loader), loss=losses, acc=accuracy, dt=data_time, bt=batch_time)) args.writer_train.add_scalar('local/loss', losses.val, args.iteration) args.writer_train.add_scalar('local/accuracy', accuracy.val, args.iteration) args.iteration += 1 print('Epoch: [{0}]\t' 'T-epoch:{t:.2f}\t'.format(epoch, t=time.time() - tic)) args.writer_train.add_scalar('global/loss', losses.avg, epoch) args.writer_train.add_scalar('global/accuracy', accuracy.avg, epoch) return losses.avg, accuracy.avg