def load_model(config):
device = torch.device("cuda")
n_gpu = torch.cuda.device_count()
infersent = load_infersent()
infersent.to(device)
if not config.data_sign == 'SST-2':
model = NLINet(config)
else:
model = ClassificationNet(config)
model.to(device)
if config.mode == 'score' or config.mode == 'attack':
model_dict_path = os.path.join(config.output_dir,
"{}_{}.bin".format(config.data_sign, config.target_model))
model.load_state_dict(torch.load(model_dict_path))
optimizer = optim.Adam(model.parameters(), lr=config.learning_rate)
if n_gpu > 1:
model = torch.nn.DataParallel(model)
return infersent, model, optimizer, device, n_gpu
if isinstance(m, nn.Linear):
print('initialize one Linear layer')
torch.nn.init.xavier_uniform_(m.weight.data)
"""
Train model on Natural Language Inference task
"""
if params.fineTuneOnPretrainedModel == True:
if not params.pretrainedModelPath:
print(
'\nERROR: If you want to fine tune on pre-trained model, you need to specify the pre-trained model path'
)
nli_net.load_state_dict(torch.load(params.pretrainedModelPath))
print('\nPre-trained model has been loaded from {}'.format(
params.pretrainedModelPath))
if params.freezeEncoder == True:
if params.fineTuneOnPretrainedModel == True:
ct = 0
for child in nli_net.children():
if ct == 0:
for param in child.parameters():
param.requires_grad = False
ct += 1
print('\nFreeze the Infersent encoder part')
else:
print(
"\nERROR: If you want to freeze encoder, you need to use pre-trained encoder"
'encoder_type': params.encoder_type,
'use_cuda': True,
}
# model
encoder_types = [
'InferSent', 'BLSTMprojEncoder', 'BGRUlastEncoder',
'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder',
'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder'
]
assert params.encoder_type in encoder_types, "encoder_type must be in " + \
str(encoder_types)
MODEL_PATH = 'savedir_from_scratch/model.pickle'
nli_net = NLINet(config_nli_model)
nli_net.load_state_dict(torch.load(MODEL_PATH))
print(nli_net)
# loss
weight = torch.FloatTensor(params.n_classes).fill_(1)
loss_fn = nn.CrossEntropyLoss(weight=weight)
loss_fn.size_average = False
# optimizer
optim_fn, optim_params = get_optimizer(params.optimizer)
optimizer = optim_fn(nli_net.parameters(), **optim_params)
# cuda by default
nli_net.cuda()
loss_fn.cuda()
"""
stop_training = adam_stop
adam_stop = True
return eval_acc, adv_models, bias_models, bias_loss_functions, bias_list, bias_optimizers
"""
Train model on Natural Language Inference task
"""
epoch = 1
if params.load_model:
encoder_net.load_state_dict(
torch.load('../encoder_' + params.load_id + 'saved.pt'))
if params.load_classifier:
nli_net.load_state_dict(
torch.load('../classifier_' + params.load_id + 'saved.pt'))
while (not stop_training
or epoch <= params.min_epoch) and epoch <= params.n_epochs:
train_acc = trainepoch(epoch)
eval_acc, adv_models, bias_models, bias_loss_functions, bias_list, bias_optimizers = evaluate(
epoch, adv_models, bias_models, bias_loss_functions, bias_list,
bias_optimizers, optim_params, 'valid')
epoch += 1
print("Evaluating on test-set")
evaluate(0, adv_models, bias_models, bias_loss_functions, bias_list,
bias_optimizers, optim_params, 'test', True)
.format(params.lrshrink,
optimizer.param_groups[0]['lr']))
if optimizer.param_groups[0]['lr'] < params.minlr:
stop_training = True
if 'adam' in params.optimizer:
# early stopping (at 2nd decrease in accuracy)
stop_training = adam_stop
adam_stop = True
return eval_acc
"""
Train model on Natural Language Inference task
"""
epoch = 1
while not stop_training and epoch <= params.n_epochs:
train_acc = trainepoch(epoch)
eval_acc = evaluate(epoch, 'valid')
epoch += 1
# Run best model on test set.
nli_net.load_state_dict(os.path.join(params.outputdir, params.outputmodelname))
print('\nTEST : Epoch {0}'.format(epoch))
evaluate(1e6, 'valid', True)
evaluate(0, 'test', True)
# Save encoder instead of full model
torch.save(nli_net.encoder.state_dict(), os.path.join(params.outputdir, params.outputmodelname + '.encoder.pkl'))
adam_stop = True
return eval_acc
"""
Train model on Natural Language Inference task
"""
epoch = 1
while not stop_training and epoch <= params.n_epochs:
train_acc = trainepoch(epoch)
eval_acc = evaluate(epoch, 'valid')
epoch += 1
# Run best model on test set.
nli_net.load_state_dict(torch.load(
os.path.join(params.outputdir, params.outputmodelname)),
map_location={
'cuda:1': 'cuda:0',
'cuda:2': 'cuda:0'
})
print('\nTEST : Epoch {0}'.format(epoch))
evaluate(1e6, 'valid', True)
evaluate(0, 'test', True)
# Save encoder instead of full model
torch.save(
nli_net.encoder.state_dict(),
os.path.join(params.outputdir, params.outputmodelname + '.encoder.pkl'))
if optimizer.param_groups[0]['lr'] < params.minlr:
stop_training = True
if 'adam' in params.optimizer:
# early stopping (at 2nd decrease in accuracy)
stop_training = adam_stop
adam_stop = True
return eval_acc
"""
Train model on Natural Language Inference task
"""
epoch = 1
while not stop_training and epoch <= params.n_epochs:
train_acc = trainepoch(epoch)
eval_acc = evaluate(epoch, 'valid')
epoch += 1
# Run best model on test set.
nli_net.load_state_dict(os.path.join(params.outputdir, params.outputmodelname))
print('\nTEST : Epoch {0}'.format(epoch))
evaluate(1e6, 'valid', True)
evaluate(0, 'test', True)
# Save encoder instead of full model
torch.save(
nli_net.encoder.state_dict(),
os.path.join(params.outputdir, params.outputmodelname + '.encoder.pkl'))
'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder']
assert params.encoder_type in encoder_types, "encoder_type must be in " + \
str(encoder_types)
nli_net = NLINet(config_nli_model)
if params.pre_trained_model:
print "Pre_trained_model: " + params.pre_trained_model
pre_trained_model = torch.load(params.pre_trained_model)
nli_net_params = nli_net.state_dict()
pre_trained_params = pre_trained_model.state_dict()
assert nli_net_params.keys() == pre_trained_params.keys(), "load model has different parameter state names that NLI_HYPOTHS_NET"
for key, parameters in nli_net_params.items():
if parameters.size() == pre_trained_params[key].size():
nli_net_params[key] = pre_trained_params[key]
nli_net.load_state_dict(nli_net_params)
print(nli_net)
# loss
weight = torch.FloatTensor(params.n_classes).fill_(1)
loss_fn = nn.CrossEntropyLoss(weight=weight)
loss_fn.size_average = False
# optimizer
optim_fn, optim_params = get_optimizer(params.optimizer)
optimizer = optim_fn(nli_net.parameters(), **optim_params)
# cuda by default
nli_net.cuda()
loss_fn.cuda()
if params.task == 'train':
val_acc_best = -1e10
adam_stop = False
stop_training = False
lr = optim_params['lr'] if 'sgd' in params.optimizer else None
# Restore saved model (if one exists).
ckpt_path = os.path.join(params.outputdir, params.outputmodelname)
if os.path.exists(ckpt_path):
print('Loading checkpoint: %s' % ckpt_path)
ckpt = torch.load(ckpt_path)
epoch = ckpt['epoch']
nli_net.load_state_dict(ckpt['model'])
optimizer.load_state_dict(ckpt['optimizer'])
else:
epoch = 1
# Training process
while not stop_training and epoch <= params.n_epochs:
train_acc = trainepoch(epoch)
eval_acc = evaluate(epoch, 'valid')
epoch += 1
# Run best model on test set.
nli_net.load_state_dict(ckpt['model'])