def validate(data_loader, model, criterion, device, epoch, args):
losses = AverageMeter()
accuracy = AverageMeter()
model.eval()
with torch.no_grad():
for idx, (input_seq, target) in tqdm(enumerate(data_loader),
total=len(data_loader)):
input_seq = input_seq.to(device)
target = target.to(device)
B = input_seq.size(0)
output, _ = model(input_seq)
[_, N, D] = output.size()
output = output.view(B * N, D)
target = target.repeat(1, N).view(-1)
loss = criterion(output, target)
acc = calc_accuracy(output, target)
losses.update(loss.item(), B)
accuracy.update(acc.item(), B)
print('Loss {loss.avg:.4f}\t'
'Acc: {acc.avg:.4f} \t'.format(loss=losses, acc=accuracy))
args.writer_val.add_scalar('global/loss', losses.avg, epoch)
args.writer_val.add_scalar('global/accuracy', accuracy.avg, epoch)
return losses.avg, accuracy.avg
def eval_model(embedding_dir, label_dir, sense, model_dir, target_languages, device):
result_dict = defaultdict(float)
net = relation_classifier(1024 * 5, output_dim).to(device)
net.load_state_dict(torch.load(model_dir, map_location=device))
for test_set in target_languages:
if test_set == "ted":
languages = ["tr", "de", "pl", "ru", "pt", "en", "lt"]
for lang in languages:
ted_test_arg1, ted_test_arg2, ted_test_target = read_laser_embeddings(embedding_dir, label_dir, lang, device, sense=sense, set_type="test")
test_pred = net(ted_test_arg1, ted_test_arg2) # predict
acc, f1, prec, rec, base_acc, base_f1 = calc_accuracy(ted_test_target, test_pred, verbose=False)
result_dict[lang] = (f1)
else:
test_arg1, test_arg2, test_target = read_laser_embeddings(embedding_dir, label_dir, test_set, device, sense=sense, set_type="test")
test_pred = net(test_arg1, test_arg2) # predict
acc, f1, prec, rec, base_acc, base_f1 = calc_accuracy(test_target, test_pred, verbose=False)
result_dict[test_set] = (f1)
return result_dict
def train(embeddings_dir, label_dir, out_dir, training_langs, sense_list, epoch_num, batch_size, early_stopping_threshold, device, verbose = False):
for sense in sense_list:
print("~~~ Training the \" %s vs Others \" classifier" % (sense))
best_score = -1
score_list = []
out_file = out_dir + "/%s_%s_model" % ("_".join(training_langs), sense)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# Prepare Data
dev_arg1, dev_arg2, dev_label = prepare_input(embeddings_dir, label_dir, training_langs, sense, device, type="dev")
# model
input_dim = dev_arg1.shape[1]*5 # *5 due to the dissent model
network = relation_classifier(input_dim).to(device)
optimizer = optim.Adagrad(network.parameters(), lr=0.01)
loss_fn = nn.CrossEntropyLoss()
dev_label = dev_label.long()
if verbose:
print(network, "\n\n")
# training loop
for epoch in range(epoch_num):
train_arg1, train_arg2, train_label = prepare_input(embeddings_dir, label_dir, training_langs, sense, device, type="training")
train_label = train_label.long()
for batch in range(int(len(train_label)/batch_size)+1):
arg1 = train_arg1[batch*batch_size:min((batch+1)*batch_size, len(train_arg1))]
arg2 = train_arg1[batch*batch_size:min((batch+1)*batch_size, len(train_arg2))]
batch_labels = train_label[batch*batch_size:min((batch+1)*batch_size, len(train_label))]
optimizer.zero_grad()
y_pred = network(arg1, arg2)
output = loss_fn(y_pred, batch_labels)
output.backward()
optimizer.step()
## compute validation loss
valid_predictions = network(dev_arg1, dev_arg2)
acc, f1, prec, rec, base_acc, base_f1 = calc_accuracy(dev_label, valid_predictions, verbose=False)
if verbose:
print("Epoch {} Validation acc: {:.7f} f1: {:.7f}".format(epoch, acc, f1), sep=" ", flush=True)
elif epoch % 10 == 0:
print("Epoch {} Validation acc: {:.7f} f1: {:.7f}".format(epoch, acc, f1), sep=" ", flush=True)
## early stopping
if f1 > best_score:
torch.save(network.state_dict(), out_file)
if verbose:
print("model saved at Epoch", epoch)
best_score = f1
score_list = []
else:
score_list.append(f1)
if len(score_list) > early_stopping_threshold:
print("Early Stopping after {} iteration".format( epoch))
print("the highest F-score achieved on the validation data: %.4f" % best_score)
print("The model is saved to", out_file)
print("--------------------")
break
print("Weights Sum:", utils.sum_model_weights(model)[1])
# Get the predicted class probabilities, labels & probabilities of abnormality
preds = model.predict(dataset_valid_noshuffle,
steps=num_steps_per_epoch_valid)
print("Preds Shape:", preds.shape)
pred_probs_valid, pred_labels_valid, pred_probs_abnormal_valid = utils.get_predictions(
dataset=dataset_valid_noshuffle,
model=model,
steps=num_steps_per_epoch_valid)
print("Pred Labels Shape:", pred_labels_valid.shape)
### Tensorflow no longer required, so come out of the session
# Image-wise accuracy & cross-entropy loss
accuracy_valid = utils.calc_accuracy(labels_valid_scalar,
pred_labels_valid)
loss_valid = utils.calc_crossentropy_loss(labels_valid_onehot,
pred_probs_valid)
print("ACCURACY VALID:", accuracy_valid)
print("LOSS VALID:", loss_valid)
# Study-wise performence
# Breakdowns of numbers of studies, from the orig MURA paper (Table 1, p3) - use these as a check on the numbers of studies we derive from our data
num_studies_published_dict = utils.get_num_studies_published()
images_accuracy_check, studies_summary_valid = utils.get_study_predictions(
images_summary_valid, pred_probs_abnormal_valid)
# Check image-wise accuracy to ensure image-wise results not reordered when calculating study-wise results
if (images_accuracy_check != accuracy_valid):
print("Imagewise Accuracy Check Failed - Have Images Been Reordered?")
def train_one_epoch(data_loader, model, criterion, optimizer, device, epoch,
args):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracy = AverageMeter()
if args.train_what == 'last':
model.eval()
model.module.final_bn.train()
model.module.final_fc.train()
print('[Warning] train model with eval mode, except final layer')
else:
model.train()
end = time.time()
tic = time.time()
for idx, (input_seq, target) in enumerate(data_loader):
data_time.update(time.time() - end)
input_seq = input_seq.to(device)
target = target.to(device)
B = input_seq.size(0)
output, _ = model(input_seq)
[_, N, D] = output.size()
output = output.view(B * N, D)
target = target.repeat(1, N).view(-1)
loss = criterion(output, target)
acc = calc_accuracy(output, target)
losses.update(loss.item(), B)
accuracy.update(acc.item(), B)
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if idx % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Loss {loss.val:.4f} ({loss.local_avg:.4f})\t'
'Acc: {acc.val:.4f} ({acc.local_avg:.4f})\t'
'T-data:{dt.val:.2f} T-batch:{bt.val:.2f}\t'.format(
epoch,
idx,
len(data_loader),
loss=losses,
acc=accuracy,
dt=data_time,
bt=batch_time))
args.writer_train.add_scalar('local/loss', losses.val,
args.iteration)
args.writer_train.add_scalar('local/accuracy', accuracy.val,
args.iteration)
args.iteration += 1
print('Epoch: [{0}]\t'
'T-epoch:{t:.2f}\t'.format(epoch, t=time.time() - tic))
args.writer_train.add_scalar('global/loss', losses.avg, epoch)
args.writer_train.add_scalar('global/accuracy', accuracy.avg, epoch)
return losses.avg, accuracy.avg