def load_model(config): device = torch.device("cuda") n_gpu = torch.cuda.device_count() infersent = load_infersent() infersent.to(device) if not config.data_sign == 'SST-2': model = NLINet(config) else: model = ClassificationNet(config) model.to(device) if config.mode == 'score' or config.mode == 'attack': model_dict_path = os.path.join(config.output_dir, "{}_{}.bin".format(config.data_sign, config.target_model)) model.load_state_dict(torch.load(model_dict_path)) optimizer = optim.Adam(model.parameters(), lr=config.learning_rate) if n_gpu > 1: model = torch.nn.DataParallel(model) return infersent, model, optimizer, device, n_gpu
'dpout_fc': params.dpout_fc, 'fc_dim': params.fc_dim, 'bsize': params.batch_size, 'n_classes': params.n_classes, 'pool_type': params.pool_type, 'nonlinear_fc': params.nonlinear_fc, 'encoder_type': params.encoder_type, 'use_cuda': True, } # model encoder_types = ['BLSTMEncoder', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder',\ 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder'] assert params.encoder_type in encoder_types, "encoder_type must be in " + str( encoder_types) nli_net = NLINet(config_nli_model) print(nli_net) # loss weight = torch.FloatTensor(params.n_classes).fill_(1) loss_fn = nn.CrossEntropyLoss(weight=weight) loss_fn.size_average = False # optimizer optim_fn, optim_params = get_optimizer(params.optimizer) optimizer = optim_fn(nli_net.parameters(), **optim_params) # cuda by default nli_net.cuda() loss_fn.cuda() #src_embeddings.cuda()
def main(args): GLOVE_PATH = "dataset/GloVe/glove.840B.300d.txt" parser = argparse.ArgumentParser(description='NLI training') # paths parser.add_argument("--nlipath", type=str, default='dataset/SNLI/', help="NLI data path (SNLI or MultiNLI)") parser.add_argument("--outputdir", type=str, default='savedir/', help="Output directory") parser.add_argument("--outputmodelname", type=str, default='model.pickle') # dataset, dimensions, transfer learning parser.add_argument("--dataset", type=str, required=True, help="Semantic similarity dataset") parser.add_argument('--dimension', nargs='+', required=True, help='Dimension(s) on the dataset') parser.add_argument('--transfer', default='DNT', help='Transfer learning approach') parser.add_argument('--save', default='no', help='Save trained model') parser.add_argument( '--load_model', default='no', help='If load model, do not perform training, just evalute') # training parser.add_argument("--n_epochs", type=int, default=10) parser.add_argument("--batch_size", type=int, default=16) parser.add_argument("--dpout_model", type=float, default=0., help="encoder dropout") parser.add_argument("--dpout_fc", type=float, default=0., help="classifier dropout") parser.add_argument("--nonlinear_fc", type=float, default=0, help="use nonlinearity in fc") parser.add_argument("--optimizer", type=str, default="sgd,lr=5", help="adam or sgd,lr=0.1") parser.add_argument("--lrshrink", type=float, default=5, help="shrink factor for sgd") parser.add_argument("--decay", type=float, default=1., help="lr decay") parser.add_argument("--minlr", type=float, default=1e-5, help="minimum lr") parser.add_argument("--max_norm", type=float, default=5., help="max norm (grad clipping)") # model parser.add_argument("--encoder_type", type=str, default='BLSTMEncoder', help="see list of encoders") parser.add_argument("--enc_lstm_dim", type=int, default=2048, help="encoder nhid dimension") parser.add_argument("--n_enc_layers", type=int, default=1, help="encoder num layers") parser.add_argument("--fc_dim", type=int, default=512, help="nhid of fc layers") parser.add_argument("--n_classes", type=int, default=3, help="entailment/neutral/contradiction") parser.add_argument("--pool_type", type=str, default='max', help="max or mean") # gpu parser.add_argument("--gpu_id", type=int, default=0, help="GPU ID") parser.add_argument("--seed", type=int, default=1236, help="seed") params, _ = parser.parse_known_args(args) # set gpu device torch.cuda.set_device(params.gpu_id) # print parameters passed, and all parameters #print('\ntogrep : {0}\n'.format(sys.argv[1:])) #print(params) def trainepoch(epoch): print('TRAINING : Epoch ' + str(epoch)) nli_net.train() logs = [] last_time = time.time() #correct = 0. # shuffle the data permutation = np.random.permutation(len(train['s1'])) s1 = train['s1'][permutation] s2 = train['s2'][permutation] targets = [x[permutation] for x in train['labels']] optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] * params.decay if epoch>1\ and 'sgd' in params.optimizer else optimizer.param_groups[0]['lr'] #print('Learning rate : {0}'.format(optimizer.param_groups[0]['lr'])) for stidx in range(0, len(s1), params.batch_size): tgt_batches = [] # prepare batch s1_batch, s1_len = get_batch(s1[stidx:stidx + params.batch_size], word_vec) s2_batch, s2_len = get_batch(s2[stidx:stidx + params.batch_size], word_vec) s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable( s2_batch.cuda()) for i, _ in enumerate(MTL_index): tgt_batches.append( Variable( torch.FloatTensor( targets[i][stidx:stidx + params.batch_size])).cuda()) #for dim in [1,2,3,4]: # model forward outputs = nli_net((s1_batch, s1_len), (s2_batch, s2_len)) # loss if params.transfer == 'DNT': #print(outputs[0]) #print((tgt_batches[0] - 1)/(params.n_classes-1)) losses = [ nli_net.loss_fn(outputs[i], (tgt_batches[i] - 1) / (params.n_classes - 1)) for i, _ in enumerate(MTL_index) ] elif params.transfer == 'NT': losses = [ nli_net.loss_fn(outputs[i], tgt_batches[i]) for i, _ in enumerate(MTL_index) ] #if 'kl' in MTL_index: # output1 = torch.log(output1) loss = np.sum(losses) #loss = loss1 + loss2 + loss3 + loss4# + loss5 + loss6 + loss7 + loss8 #ADDED #optimizer.zero_grad() #loss1.backward(retain_graph=True) #loss2.backward(retain_graph=True) #loss3.backward(retain_graph=True) #loss4.backward(retain_graph=True) #optimizer.step() #END ADDED """ if dim == 1: loss = nli_net.loss_fn(output1, tgt_batch1) elif dim == 2: loss = nli_net.loss_fn(output2, tgt_batch2) elif dim == 3: loss = nli_net.loss_fn(output3, tgt_batch3) elif dim == 4: loss = nli_net.loss_fn(output4, tgt_batch4) """ # backward optimizer.zero_grad() loss.backward() # optimizer step optimizer.step() def evaluate(epoch, eval_type='valid', flag='', correlation=spearmanr, transfer='NT'): nli_net.eval() #correct = 0. preds = [] r = np.arange(1, 1 + nli_net.n_classes) global val_acc_best, lr, stop_training, adam_stop if eval_type == 'valid': print('VALIDATION : Epoch {0}'.format(epoch)) s1 = valid['s1'] s2 = valid['s2'] targets = valid['scores'] elif eval_type == 'test': print('TEST : Epoch {0}'.format(epoch)) s1 = test['s1'] s2 = test['s2'] targets = test['scores'] elif eval_type == 'train': print('EVAL ON TRAIN : Epoch {0}'.format(epoch)) s1 = train['s1'] s2 = train['s2'] targets = train['scores'] else: raise ValueError('Wrong eval_type.') probas = [[] for _ in MTL_index] correct = 0. for i in range(0, len(s1), params.batch_size): # prepare batch s1_batch, s1_len = get_batch(s1[i:i + params.batch_size], word_vec) s2_batch, s2_len = get_batch(s2[i:i + params.batch_size], word_vec) s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable( s2_batch.cuda()) # model forward outputs = nli_net((s1_batch, s1_len), (s2_batch, s2_len)) for i, _ in enumerate(MTL_index): if len(probas[i]) == 0: probas[i] = outputs[i].data.cpu().numpy() else: probas[i] = np.concatenate( (probas[i], outputs[i].data.cpu().numpy()), axis=0) """ if 2 in MTL_index: if 'e' in MTL_index: tgt_batch2 = Variable(torch.LongTensor(target2[i:i + params.batch_size])).cuda() pred2 = output2.data.max(1)[1] correct += pred2.long().eq(tgt_batch2.data.long()).cpu().sum() else: if len(probas2) == 0: probas2 = output2.data.cpu().numpy() else: probas2 = np.concatenate((probas2, output2.data.cpu().numpy()), axis=0) """ if transfer == 'NT': ret = [ correlation(np.dot(x, r), y)[0] for x, y in zip(probas, targets) ] elif transfer == 'DNT': ret = [correlation(x, y)[0] for x, y in zip(probas, targets)] else: raise ValueError('Wrong transfer.') """ if 2 in MTL_index: if 'e' in MTL_index: ret.append(round(100 * correct/len(s1), 2)) else: yhat2 = np.dot(probas2, r) p2 = spearmanr(yhat2, target2)[0] ret.append(p2) else: ret.append(0) """ return ret """ SEED """ np.random.seed(params.seed) torch.manual_seed(params.seed) torch.cuda.manual_seed(params.seed) """ DATA """ #for i in range(1,9): # print(i) # print('----------') dataset_path = { 'stsbenchmark': '../stsbenchmark/', 'sts12': '../SemEval12/', 'sick': '../SICK/', 'activities': '../human_activity_phrase_data/', 'sag': '../ShortAnswerGrading_v2.0/data/processed/', 'typed': '../SemEval13/typed/' } #MTL_index = [1,2,3,4, 'mse'] #'e' MTL_index = [int(x) for x in params.dimension] train, valid, test = get_sts(dataset_path[params.dataset], MTL_index, params.transfer, params.n_classes) word_vec = build_vocab( train['s1'] + train['s2'] + valid['s1'] + valid['s2'] + test['s1'] + test['s2'], GLOVE_PATH) for split in ['s1', 's2']: for data_type in ['train', 'valid', 'test']: eval(data_type)[split] = np.array( [[word for word in sent.split() if word in word_vec] for sent in eval(data_type)[split]]) #eval(data_type)[split] = np.array([['<s>'] + # [word for word in sent.split() if word in word_vec or word[:2] == 'dc'] + # ['</s>'] for sent in eval(data_type)[split]]) params.word_emb_dim = 300 """ MODEL """ # model config config_nli_model = { 'n_words': len(word_vec), 'word_emb_dim': params.word_emb_dim, 'enc_lstm_dim': params.enc_lstm_dim, 'n_enc_layers': params.n_enc_layers, 'dpout_model': params.dpout_model, 'dpout_fc': params.dpout_fc, 'fc_dim': params.fc_dim, 'bsize': params.batch_size, 'n_classes': params.n_classes, 'pool_type': params.pool_type, 'nonlinear_fc': params.nonlinear_fc, 'encoder_type': params.encoder_type, 'use_cuda': True, 'MTL_index': MTL_index, 'transfer': params.transfer } # model encoder_types = [ 'BLSTMEncoder', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder' ] assert params.encoder_type in encoder_types, "encoder_type must be in " + \ str(encoder_types) perfs_all = [] for rd in range(1): print("Round", rd) if params.load_model == 'no': nli_net = NLINet(config_nli_model) nli_net.encoder = torch.load('encoder/infersent.allnli.pickle', map_location={ 'cuda:1': 'cuda:0', 'cuda:2': 'cuda:0' }) else: nli_net = torch.load(params.load_model) print(nli_net) # optimizer optim_fn, optim_params = get_optimizer(params.optimizer) optimizer = optim_fn(nli_net.parameters(), **optim_params) # cuda by default nli_net.cuda() """ TRAIN """ val_acc_best = -1e10 adam_stop = False stop_training = False lr = optim_params['lr'] if 'sgd' in params.optimizer else None last_result = 0 last_test_result = 0 drop_count = 0 """ Train model on Natural Language Inference task """ correlation = spearmanr if params.dataset == 'activities' else pearsonr epoch = 0 perfs_valid = evaluate(epoch, 'valid', 'begin', correlation, params.transfer) perfs_test = evaluate(epoch, 'test', 'begin', correlation, params.transfer) print(perfs_valid, perfs_test) epoch += 1 if params.load_model == 'no': while not stop_training and epoch <= params.n_epochs: trainepoch(epoch) perfs_valid = evaluate(epoch, 'valid', '', correlation, params.transfer) perfs_test = evaluate(epoch, 'test', '', correlation, params.transfer) print(perfs_valid, perfs_test) epoch += 1 #perfs_all.append(perfs) if params.save != 'no': torch.save(nli_net, params.save)
'n_classes': params.n_classes, 'pool_type': params.pool_type, 'nonlinear_fc': params.nonlinear_fc, 'encoder_type': params.encoder_type, 'use_cuda': True, } # model encoder_types = [ 'InferSent', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder' ] assert params.encoder_type in encoder_types, "encoder_type must be in " + \ str(encoder_types) nli_net = NLINet(config_nli_model) print(nli_net) # loss weight = torch.FloatTensor(params.n_classes).fill_(1) loss_fn = nn.CrossEntropyLoss(weight=weight) loss_fn.size_average = False # optimizer optim_fn, optim_params = get_optimizer(params.optimizer) optimizer = optim_fn(nli_net.parameters(), **optim_params) # cuda by default nli_net.cuda() loss_fn.cuda() """
'pool_type': params.pool_type, 'nonlinear_fc': params.nonlinear_fc, 'encoder_type': params.encoder_type, 'use_cuda': True, } # model encoder_types = [ 'InferSent', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder', 'LSTM_sememe', 'BILSTM_sememe' ] assert params.encoder_type in encoder_types, "encoder_type must be in " + \ str(encoder_types) nli_net = NLINet(config_nli_model) #nli_net.emb_sememe.weight.data.copy_(emb_s) print(nli_net) # loss weight = torch.FloatTensor(params.n_classes).fill_(1) loss_fn = nn.CrossEntropyLoss(weight=weight, reduction='sum') # optimizer optim_fn, optim_params = get_optimizer(params.optimizer) optimizer = optim_fn(nli_net.parameters(), **optim_params) # cuda by default nli_net.cuda() loss_fn.cuda() """
'n_classes': params.n_classes, 'pool_type': params.pool_type, 'nonlinear_fc': params.nonlinear_fc, 'encoder_type': params.encoder_type, 'use_cuda': True, } # model encoder_types = [ 'InferSent', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder' ] assert params.encoder_type in encoder_types, "encoder_type must be in " + \ str(encoder_types) nli_net = NLINet(config_nli_model, transformer_opt) print(nli_net) #nli_net.load_state_dict(torch.load(os.path.join(params.outputdir, params.outputmodelname))) #print("Model Loaded") # loss weight = torch.FloatTensor(params.n_classes).fill_(1) loss_fn = nn.CrossEntropyLoss(weight=weight) loss_fn.size_average = False # optimizer optim_fn, optim_params = get_optimizer(params.optimizer) optimizer = optim_fn(nli_net.parameters(), **optim_params) # cuda by default nli_net.cuda()
'nonlinear_fc': params.nonlinear_fc, 'encoder_type': params.encoder_type, 'use_cuda': True, } # model encoder_types = [ 'InferSent', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder' ] assert params.encoder_type in encoder_types, "encoder_type must be in " + \ str(encoder_types) MODEL_PATH = 'savedir_from_scratch/model.pickle' nli_net = NLINet(config_nli_model) nli_net.load_state_dict(torch.load(MODEL_PATH)) print(nli_net) # loss weight = torch.FloatTensor(params.n_classes).fill_(1) loss_fn = nn.CrossEntropyLoss(weight=weight) loss_fn.size_average = False # optimizer optim_fn, optim_params = get_optimizer(params.optimizer) optimizer = optim_fn(nli_net.parameters(), **optim_params) # cuda by default nli_net.cuda() loss_fn.cuda()
'n_words': len(word_vec), 'word_emb_dim': params.word_emb_dim, 'enc_lstm_dim': params.enc_lstm_dim, 'n_enc_layers': params.n_enc_layers, 'dpout_model': params.dpout_model, 'dpout_fc': params.dpout_fc, 'fc_dim': params.fc_dim, 'bsize': params.batch_size, 'n_classes': params.n_classes, 'pool_type': params.pool_type, 'nonlinear_fc': params.nonlinear_fc, 'encoder_type': params.encoder_type, 'use_cuda': True, 'version': params.version, } nli_net = NLINet(config_nli_model) print(nli_net) config_debias_model = { 'n_words': len(word_vec), 'word_emb_dim': params.word_emb_dim, 'enc_lstm_dim': params.enc_lstm_dim, 'n_enc_layers': params.n_enc_layers, 'dpout_model': params.dpout_model, 'dpout_fc': params.dpout_fc, 'fc_dim': params.fc_dim, 'bsize': params.batch_size, 'n_classes': params.n_classes, 'pool_type': params.pool_type, 'nonlinear_fc': params.nonlinear_fc, 'encoder_type': params.encoder_type,
'fc_dim': params.fc_dim, 'bsize': params.batch_size, 'n_classes': params.n_classes, 'pool_type': params.pool_type, 'nonlinear_fc': params.nonlinear_fc, 'use_cuda': True, } # Model encoders available (currently set to LSTMEncoder) encoder_types = [ 'InferSent', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder' ] nli_net = NLINet(config_nli_model) print(nli_net) nli_net.cuda() encoder_net = LSTMEncoder(config_nli_model) print(encoder_net) encoder_net.cuda() bias_list = [] # Loss # NLI model loss weight = torch.FloatTensor(params.n_classes).fill_(1) loss_fn_main = nn.CrossEntropyLoss(weight=weight) loss_fn_main.size_average = False
'dpout_fc' : params.dpout_fc , 'fc_dim' : params.fc_dim , 'bsize' : params.batch_size , 'n_classes' : params.n_classes , 'pool_type' : params.pool_type , 'nonlinear_fc' : params.nonlinear_fc , 'encoder_type' : params.encoder_type , 'encoder' : encoder , 'use_cuda' : params.use_cuda , } nli_net = NLINet(config_nli_model) print("================== Classifier =====================") print(nli_net) print("================== Encoder =====================") print(nli_net.encoder) # loss weight = torch.FloatTensor(params.n_classes).fill_(1) loss_fn = nn.CrossEntropyLoss(weight=weight) loss_fn.size_average = False # optimizer optim_fn, optim_params = get_optimizer(params.optimizer) optimizer = optim_fn(nli_net.parameters(), **optim_params) # cuda by default
'bsize' : params.batch_size , 'n_classes' : params.n_classes , 'pool_type' : params.pool_type , 'nonlinear_fc' : params.nonlinear_fc , 'encoder_type' : params.encoder_type , 'use_cuda' : True , } # model encoder_types = ['InferSent', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder'] assert params.encoder_type in encoder_types, "encoder_type must be in " + \ str(encoder_types) nli_net = NLINet(config_nli_model) print(nli_net) # loss weight = torch.FloatTensor(params.n_classes).fill_(1) loss_fn = nn.CrossEntropyLoss(weight=weight) loss_fn.size_average = False # optimizer optim_fn, optim_params = get_optimizer(params.optimizer) optimizer = optim_fn(nli_net.parameters(), **optim_params) # cuda by default nli_net.cuda() loss_fn.cuda()
'n_classes': params.n_classes, 'pool_type': params.pool_type, 'nonlinear_fc': params.nonlinear_fc, 'encoder_type': params.encoder_type, 'use_cuda': True, } # model encoder_types = [ 'InferSent', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder' ] assert params.encoder_type in encoder_types, "encoder_type must be in " + \ str(encoder_types) nli_net = NLINet(config_nli_model) print(nli_net) # loss weight = torch.FloatTensor(params.n_classes).fill_(1) loss_fn = nn.CrossEntropyLoss(weight=weight) loss_fn.size_average = False # optimizer optim_fn, optim_params = get_optimizer(params.optimizer) optimizer = optim_fn(nli_net.parameters(), **optim_params) # cuda by default nli_net.cuda() loss_fn.cuda() """
'bsize' : params.batch_size , 'n_classes' : params.n_classes , 'pool_type' : params.pool_type , 'nonlinear_fc' : params.nonlinear_fc , 'encoder_type' : params.encoder_type , 'use_cuda' : True , } # model encoder_types = ['BLSTMEncoder', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder'] assert params.encoder_type in encoder_types, "encoder_type must be in " + \ str(encoder_types) nli_net = NLINet(config_nli_model) if params.pre_trained_model: print "Pre_trained_model: " + params.pre_trained_model pre_trained_model = torch.load(params.pre_trained_model) nli_net_params = nli_net.state_dict() pre_trained_params = pre_trained_model.state_dict() assert nli_net_params.keys() == pre_trained_params.keys(), "load model has different parameter state names that NLI_HYPOTHS_NET" for key, parameters in nli_net_params.items(): if parameters.size() == pre_trained_params[key].size(): nli_net_params[key] = pre_trained_params[key] nli_net.load_state_dict(nli_net_params) print(nli_net)
'pool_type': params.pool_type, 'nonlinear_fc': params.nonlinear_fc, 'encoder_type': params.encoder_type, 'use_cuda': True, } # model encoder_types = [ 'BLSTMEncoder', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder', 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder' ] assert params.encoder_type in encoder_types, "encoder_type must be in " + \ str(encoder_types) nli_net = NLINet(config_nli_model) print(nli_net) # loss if params.loss == 'svm': loss_fn = MultiClassHingeLoss() else: loss_fn = nn.CrossEntropyLoss() # cuda by default nli_net.cuda() loss_fn.cuda() # optimizer optimizer = get_optimizer(params, parameters=nli_net.parameters())