bert_model = BertFinetune(bert, output_type=opt['output_bert_model'])
# Load model
version = opt['version']
folder = './' + version
image_encoder.load_state_dict(
torch.load(os.path.join(folder, 'image_encoder.pth'),
map_location=device))
text_encoder.load_state_dict(
torch.load(os.path.join(folder, 'text_encoder.pth'),
map_location=device))
bert_model.load_state_dict(
torch.load(os.path.join(folder, 'bert_model.pth'),
map_location=device))
image_mha.load_state_dict(
torch.load(os.path.join(folder, 'image_mha.pth'), map_location=device))
image_encoder.eval()
text_encoder.eval()
bert_model.eval()
image_mha.eval()
image_mha = None
# Define dataloader
def my_collate_fn(batch):
return tuple(zip(*batch))
image_folder = opt['feature_folder']
test_image_dataset = FeatureDataset(image_folder, opt['test_file'])
test_image_dataloader = DataLoader(test_image_dataset,
batch_size=64,
shuffle=False,
collate_fn=my_collate_fn)
# forward + backward + optimize
predict_labels, _ = model(input_features) #size 64x11
loss = loss_function(predict_labels, input_val) #size 64x11 vs 64
loss.backward()
optimizer.step()
total_loss += loss.cpu().data.numpy()
num_batch = batch_idx + 1
print("avg training loss:", total_loss / num_batch)
logfile.writelines("avg training loss:" + str(total_loss / num_batch) +
"\n")
logfile.flush()
train_loss.append(total_loss / num_batch)
# validation
with torch.no_grad():
model.eval()
predict_labels, _ = model(valid_features.to(device))
predict_val = torch.argmax(predict_labels, 1).cpu().data
accuracy = np.mean((predict_val == valid_val).numpy())
print("validation accuracy: ", accuracy)
logfile.writelines("validation accuracy: " + str(accuracy) + "\n")
logfile.flush()
valid_acc.append(accuracy)
# saving best acc model as best.pth
if (accuracy > max_accuracy):
max_accuracy = accuracy
torch.save(model, 'best_cnnp2.pth')
logfile.writelines("save as best_cnnp2.pth\n")
logfile.flush()
model.train()