def __init__(self, opt): """ 初始化模型和数据预处理,并token化 :param opt: argparse的参数 """ self.opt = opt #是否是bert类模型,使用bert类模型初始化, 非BERT类使用GloVe if 'bert' in opt.model_name: #初始化tokenizer tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name, cache_dir=opt.pretrained_bert_cache_dir) # 加载BERT模型 bert = BertModel.from_pretrained( opt.pretrained_bert_name, cache_dir=opt.pretrained_bert_cache_dir) # 然后把BERT模型和opt参数传入自定义模型,进行进一步处理 self.model = opt.model_class(bert, opt).to(opt.device) else: # 自定义tokenizer,生成id2word,word2idx tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) #返回所有单词的词嵌入 [word_nums, embedding_dimesion] embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) # 加载模型 self.model = opt.model_class(embedding_matrix, opt).to(opt.device) # 加载训练集 self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer, recreate_caches=opt.recreate_caches) self.testset = ABSADataset(opt.dataset_file['test'], tokenizer, recreate_caches=opt.recreate_caches) #如果valset_ratio为0,测试集代替验证集 assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: self.valset = self.testset # 检查cuda的内存 if opt.device.type == 'cuda': logger.info('cuda 可用内存: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) self.trainset = IOBDataset(opt.dataset_file['train'], tokenizer) if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt self.tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) print('loading model {0} ...'.format(opt.model_name)) self.model.load_state_dict(torch.load(opt.state_dict_path)) self.model = self.model.to(opt.device) # switch model to evaluation mode self.model.eval() torch.autograd.set_grad_enabled(False)
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: self.tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) print('loading model {0} ...'.format(opt.model_name)) # remember removed map_location='cpu' when using on server w GPU self.model.load_state_dict(torch.load(opt.state_dict_path)) # switch model to evaluation mode self.model.eval() torch.autograd.set_grad_enabled(False)
def __init__(self): opt = module_opt() bert = BertModel.from_pretrained('bert-base-uncased') self.tokenizer = Tokenizer4Bert(80, 'bert-base-uncased') model = LCF_BERT(bert, opt).to(opt.device) print('loading sa module ...') model.load_state_dict( torch.load('state_dict/lcf_bert_movie_val_acc0.8203', map_location=torch.device('cpu'))) model.eval() torch.autograd.set_grad_enabled(False) self.model = model self.opt = opt
def __init__(self, arguments): # 项目的超参 parser = argparse.ArgumentParser() parser.add_argument("-e", "--EPOCHS", default=5, type=int, help="train epochs") parser.add_argument("-b", "--BATCH", default=2, type=int, help="batch size") self.args = parser.parse_args() self.arguments = arguments self.dataset = Dataset(epochs=self.args.EPOCHS, batch=self.args.BATCH, val_batch=self.args.BATCH) if 'bert' in self.arguments.model_name: self.tokenizer = Tokenizer4Bert( max_seq_len=self.arguments.max_seq_len, pretrained_bert_name=os.path.join( os.getcwd(), self.arguments.pretrained_bert_name)) bert = BertModel.from_pretrained(pretrained_model_name_or_path=self .arguments.pretrained_bert_name) self.model = self.arguments.model_class(bert, self.arguments).to( self.arguments.device) else: self.tokenizer = Util.bulid_tokenizer( fnames=[ self.arguments.dataset_file['train'], self.arguments.dataset_file['test'] ], max_seq_len=self.arguments.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(self.arguments.dataset)) embedding_matrix = Util.build_embedding_matrix( word2idx=self.tokenizer.word2idx, embed_dim=self.arguments.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(self.arguments.embed_dim), self.arguments.dataset)) self.model = self.arguments.model_class( embedding_matrix, self.arguments).to(self.arguments.device) if self.arguments.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated( device=self.arguments.device.index))) Util.print_args(model=self.model, logger=logger, args=self.arguments)
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='temp_data/' + '{0}_tokenizer.dat'.format(opt.dataset), step=4 if opt.tabsa else 3) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='temp_data/' + '{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) if opt.tabsa: if opt.tabsa_with_absa: self.trainset = TABSADataset(opt.dataset_file['train'], tokenizer, True) self.testset = TABSADataset(opt.dataset_file['test'], tokenizer, True) else: self.trainset = TABSADataset(opt.dataset_file['train'], tokenizer, False) self.testset = TABSADataset(opt.dataset_file['test'], tokenizer, False) else: self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer) self.testset = ABSADataset(opt.dataset_file['test'], tokenizer) assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def initialize(): opt = get_parameters() opt.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) model = model_classes[opt.model_name](bert, opt).to(opt.device) print('loading model {0} ...'.format(opt.model_name)) torch.autograd.set_grad_enabled(False) model.load_state_dict(torch.load(state_dict_paths[opt.model_name])) model.eval() torch.autograd.set_grad_enabled(False) return opt, tokenizer, model
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: self.tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = BERT_SSC(bert, opt).to(opt.device) logger.info('loading model {0} ... done'.format(opt.model_name)) # remember removed map_location='cpu' when using on server w GPU self.model.load_state_dict(torch.load(opt.state_dict_path)) # switch model to evaluation mode self.model.eval() torch.autograd.set_grad_enabled(False) else: logger.info('Now, we only support bert-based model') raise ValueError("Now, we only support bert-based model")
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) # else: # tokenizer = build_tokenizer( # fnames=[opt.dataset_file['train'], opt.dataset_file['test']], # max_seq_len=opt.max_seq_len, # dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) # embedding_matrix = build_embedding_matrix( # word2idx=tokenizer.word2idx, # embed_dim=opt.embed_dim, # dat_fname='{0}_{1}_embedding_matrix.dat'.format(str(opt.embed_dim), opt.dataset)) # self.model = opt.model_class(embedding_matrix, opt).to(opt.device) if 'pair' in opt.model_name: if not self.opt.do_eval: self.trainset = ABSADataset_sentence_pair( opt.dataset_file['train'], tokenizer) self.testset = ABSADataset_sentence_pair(opt.dataset_file['test'], tokenizer) elif 'SA' in opt.model_name: if not self.opt.do_eval: self.trainset = SADataset(opt.dataset_file['train'], tokenizer) self.testset = SADataset(opt.dataset_file['test'], tokenizer) else: if not self.opt.do_eval: self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer) self.testset = ABSADataset(opt.dataset_file['test'], tokenizer) assert 0 <= opt.valset_ratio < 1 if not self.opt.do_eval: if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, data): self.net = None self.data = data self.args = args self.idx2label = dict( (i, args.labels[i]) for i in range(len(args.labels))) self.tokenizer = Tokenizer4Bert(max_seq_len=self.args.max_seq_len, pretrained_bert_name=os.path.join( os.getcwd(), self.args.pretrained_bert_name)) bert = BertModel.from_pretrained( os.path.join(os.getcwd(), self.args.pretrained_bert_name)) model = self.args.model_classes[args.model_name](bert, self.args).to( self.args.device) if self.args.topics is not None: self.net_0 = Util.load_model(model=model, output_dir=os.path.join( os.getcwd(), args.best_model_path, self.args.topics[0])) self.net_0.eval() self.net_1 = Util.load_model(model=model, output_dir=os.path.join( os.getcwd(), args.best_model_path, self.args.topics[1])) self.net_1.eval() self.net_2 = Util.load_model(model=model, output_dir=os.path.join( os.getcwd(), args.best_model_path, self.args.topics[2])) self.net_2.eval() self.net_3 = Util.load_model(model=model, output_dir=os.path.join( os.getcwd(), args.best_model_path, self.args.topics[3])) self.net_3.eval() self.net_4 = Util.load_model(model=model, output_dir=os.path.join( os.getcwd(), args.best_model_path, self.args.topics[4])) self.net_4.eval() else: self.net = Util.load_model(model=model, output_dir=os.path.join( os.getcwd(), args.best_model_path))
def __init__(self, arguments): # 项目的超参 parser = argparse.ArgumentParser() parser.add_argument("-e", "--EPOCHS", default=5, type=int, help="train epochs") parser.add_argument("-b", "--BATCH", default=2, type=int, help="batch size") self.args = parser.parse_args() self.arguments = arguments self.dataset = Dataset(epochs=self.args.EPOCHS, batch=self.args.BATCH, val_batch=self.args.BATCH) if 'bert' in self.arguments.model_name: self.tokenizer = Tokenizer4Bert(max_seq_len=self.arguments.max_seq_len, pretrained_bert_name=os.path.join(os.getcwd(), self.arguments.pretrained_bert_name)) bert = BertModel.from_pretrained(pretrained_model_name_or_path=self.arguments.pretrained_bert_name) self.model = self.arguments.model_class(bert, self.arguments).to(self.arguments.device) else: self.tokenizer = Util.bulid_tokenizer( fnames=[self.arguments.dataset_file['train'], self.arguments.dataset_file['test']], max_seq_len=self.arguments.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(self.arguments.dataset) ) embedding_matrix = Util.build_embedding_matrix( word2idx=self.tokenizer.word2idx, embed_dim=self.arguments.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format(str(self.arguments.embed_dim), self.arguments.dataset) ) self.model = self.arguments.model_class(embedding_matrix, self.arguments).to(self.arguments.device) if self.arguments.device.type == 'cuda': logger.info( 'cuda memory allocated: {}'.format(torch.cuda.memory_allocated(device=self.arguments.device.index))) Util.print_args(model=self.model, logger=logger, args=self.arguments) target_text, stance, _, _ = self.dataset.get_all_data() target = np.asarray([i['TARGET'].lower() for i in target_text]) text = np.asarray([i['TEXT'].lower() for i in target_text]) stance = np.asarray([i['STANCE'] for i in stance]) self.target_set = set() for tar in target: self.target_set.add(tar) text = PreProcessing(text).get_file_text() trainset = ABSADataset(data_type=None, fname=(target, text, stance), tokenizer=self.tokenizer) valset_len = int(len(trainset) * self.arguments.valset_ratio) self.trainset, self.valset = random_split(trainset, (len(trainset) - valset_len, valset_len))
def __init__(self, opt): self.opt = opt print("loading {0} tokenizer...".format(opt.dataset)) self.bert_tokenizer = Tokenizer4Bert('bert-base-chinese') self.model_list = [] for i, model_name in enumerate(opt.model_name_list): print('loading model {0}... '.format(model_name)) bert = BertModel.from_pretrained('bert-base-chinese') model = nn.DataParallel(opt.model_class_list[i](bert, opt).to( opt.device)) model.load_state_dict(torch.load(opt.state_dict_path_list[i])) # switch model to evaluation mode model.eval() self.model_list.append(model) torch.autograd.set_grad_enabled(False)
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: # set bert_based_vocab tokenizer = Tokenizer4Bert( opt.max_seq_len, '/data/kkzhang/aaa/command/bert-base-uncased-vocab.txt') #tokenizer = Tokenizer4Bert(opt.max_seq_len, '/home/kkzhang/bert-large-uncased/bert-large-uncased-vocab.txt') # set bert pre_train model bert = BertModel.from_pretrained( '/data/kkzhang/WordeEmbedding/bert_base/') ##### multi gpu ########## if torch.cuda.device_count() > 1: logging.info('The device has {} gpus!!!!!!!!!!!!!'.format( torch.cuda.device_count())) bert = nn.DataParallel(bert) self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer) self.testset = ABSADataset(opt.dataset_file['test'], tokenizer) assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) self.trainset = ABSADataset('./data/Train_Data.csv', tokenizer) assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) # self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer) # self.testset = ABSADataset(opt.dataset_file['test'], tokenizer) ## using our own dataset data = pd.read_csv('train_data1.csv') # test_data = pd.read_csv('../test_tOlRoBf.csv') train_data, test_data = train_test_split(data, test_size=0.1, random_state=42) self.trainset = ABSADataset(train_data, tokenizer) self.testset = ABSADataset(test_data, tokenizer) assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = nn.DataParallel(opt.model_class(bert, opt)).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) print('pretrainn model load done') self.trainset = CLFDataset(opt.dataset_file['train'], tokenizer, label_dict=opt.label_dict) print('trainset build done') assert 0 <= opt.valset_ratio < 1 self.testset = self.trainset self.valset = self.testset if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) self.testset = self.valset else: self.testset = self.trainset self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) if self.opt.bert_path: bert_path = self.opt.bert_path.replace("\r", "").replace("\n", "") bert = BertModel.from_pretrained(bert_path) self.model = opt.model_class(bert, opt).to(opt.device) self.trainset = PreTrainDataset(opt.dataset, tokenizer, train_or_test='train') np.random.shuffle(self.trainset.data) self.testset = PreTrainDataset(opt.dataset, tokenizer, train_or_test='test') # np.random.shuffle(self.testset.data) if self.opt.cross_val_fold < 0: self.valset = PreTrainDataset(opt.dataset, tokenizer, train_or_test='val') if self.opt.cross_val_fold == 0: self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format(torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt if opt.model_name.lower() in ['vh_bert', 'bert_att', 'my_lcf']: tokenizer = BertTokenizer.from_pretrained(opt.pretrained_bert_name) config = BertConfig.from_pretrained(opt.pretrained_bert_name, output_attentions=True) self.model = opt.model_class(config, ).to(opt.device) elif 'bert' in opt.model_name.lower(): tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) config = BertConfig.from_pretrained(opt.pretrained_bert_name, output_attentions=True) bert = BertModel.from_pretrained(opt.pretrained_bert_name, config=config) self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='./cache/{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='./cache/{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer) self.testset = ABSADataset(opt.dataset_file['test'], tokenizer) assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def get_model(models): opt_list = [] pred_list = [] for model in models: opt = main(model) opt_list.append(opt) tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name, output_hidden_states=True) testset = ABSADataset(opt.dataset_file['test'], tokenizer) for opt in opt_list: if (opt.model_name == "bert_spc" or opt.model_name == "lcf_bert"): bert1 = BertModel.from_pretrained(opt.pretrained_bert_name) pred = Predictor(opt, tokenizer, bert1, testset) else: pred = Predictor(opt, tokenizer, bert, testset) predictions = pred.save_predictions() pred_list.append(predictions) return pred_list
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.ç) if opt.dataset in ['twitter', 'restaurant', 'laptop']: self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer) #返回 torch 的dataset类 self.testset = ABSADataset(opt.dataset_file['test'], tokenizer) else: self.trainset = CovData(opt.dataset_file['train'], tokenizer) #返回 torch 的dataset类 self.testset = CovData(opt.dataset_file['test'], tokenizer) # 定义切分数据集的比例 切分训练集 assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name, output_hidden_states=True) # tokenizer = Tokenizer4Bert(opt.max_seq_len, '/content/drive/My Drive/FYP/pretrained_BERT_further_trained_with_criminal_corpus/vocab.txt') # bert = BertModel.from_pretrained('/content/drive/My Drive/FYP/pretrained_BERT_further_trained_with_criminal_corpus') self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) self.trainset = ABSADataset( opt.dataset_file['train'], './datasets/semeval14/law_train.raw.graph', tokenizer) self.testset = ABSADataset(opt.dataset_file['test'], './datasets/semeval14/law_train.raw.graph', tokenizer) assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: self.tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) else: self.tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=self.tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format(str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt) print('loading model {0} ...'.format(opt.model_name)) self.model.load_state_dict(torch.load(opt.state_dict_path)) self.model = self.model.to(opt.device) # switch model to evaluation mode self.model.eval() torch.autograd.set_grad_enabled(False)
def get_model(models): opt_list = [] models_list = [] for model in models: opt = main(model) opt_list.append(opt) tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name, output_hidden_states=True) for opt in opt_list: if (opt.model_name == "bert_spc" or opt.model_name == "lcf_bert"): bert1 = BertModel.from_pretrained(opt.pretrained_bert_name) pred = Preloader(opt, tokenizer, bert1) models_list.append(pred.get_model()) else: pred = Preloader(opt, tokenizer, bert) models_list.append(pred.get_model()) return models_list, opt_list, tokenizer
def __init__(self, opt): self.opt = opt tokenizer = Tokenizer4Bert(opt.max_length, opt.pretrained_bert_name) bert_model = BertModel.from_pretrained(opt.pretrained_bert_name, output_hidden_states=True) self.pretrained_bert_state_dict = bert_model.state_dict() self.model = opt.model_class(bert_model, opt).to(opt.device) print('loading model {0} ...'.format(opt.model_name)) self.model.load_state_dict(torch.load(opt.state_dict_path)) self.model = self.model.to(opt.device) torch.autograd.set_grad_enabled(False) testset = BertSentenceDataset(opt.dataset_file['test'], tokenizer, target_dim=self.opt.polarities_dim, opt=opt) self.test_dataloader = DataLoader(dataset=testset, batch_size=opt.eval_batch_size, shuffle=False)
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) # freeze pretrained bert params # for param in bert.parameters(): # param.requires_grad = False self.model = opt.model_class(bert, opt) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) trainset = ABSADataset(opt.dataset_file['train'], tokenizer) testset = ABSADataset(opt.dataset_file['test'], tokenizer) self.train_data_loader = DataLoader(dataset=trainset, batch_size=opt.batch_size, shuffle=True) self.test_data_loader = DataLoader(dataset=testset, batch_size=opt.batch_size, shuffle=False) if opt.device.type == 'cuda': self.model = nn.DataParallel(self.model).cuda() print("cuda memory allocated:", torch.cuda.memory_allocated(device=opt.device.index)) else: self.model = self.model.to(opt.device) self._print_args()
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name + '/vocab.txt') bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, fname=opt.embed_fname, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.train_dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer) self.testset = ABSADataset(opt.dataset_file['test'], tokenizer) assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0 and (not opt.val_test): print('Splitting trainset in train and val') valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: print('Setting testset as valset through valsetratio = 0') self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: # opt.learning_rate = 2e-5 tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) else: # opt.learning_rate = 0.001 tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) trainset = ABSADataset(opt.dataset_file['train'], tokenizer) testset = ABSADataset(opt.dataset_file['test'], tokenizer) self.train_data_loader = DataLoader(dataset=trainset, batch_size=opt.batch_size, shuffle=True) self.test_data_loader = DataLoader(dataset=testset, batch_size=opt.batch_size, shuffle=False) if opt.device.type == 'cuda': logging.info("cuda memory allocated:{}".format( torch.cuda.memory_allocated(device=opt.device.index))) self._log_write_args()
model_classes = { 'bert_spc': BERT_SPC, 'aen_bert': AEN_BERT, 'lcf_bert': LCF_BERT } # set your trained models here state_dict_paths = { 'lcf_bert': 'state_dict/lcf_bert_laptop_val_acc0.2492', 'bert_spc': 'state_dict/bert_spc_laptop_val_acc0.268', 'aen_bert': 'state_dict/aen_bert_laptop_val_acc0.2006' } opt = get_parameters() opt.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) model = model_classes[opt.model_name](bert, opt).to(opt.device) print('loading model {0} ...'.format(opt.model_name)) model.load_state_dict(torch.load(state_dict_paths[opt.model_name])) model.eval() torch.autograd.set_grad_enabled(False) # input: This little place has a cute interior decor and affordable city prices. # text_left = This little place has a cute # aspect = interior decor # text_right = and affordable city prices. text_bert_indices, bert_segments_ids, text_raw_bert_indices, aspect_bert_indices = \ prepare_data('This little place has a cute', 'interior decor', 'and affordable city prices.', tokenizer)
def __int__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.datasets_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer) self.testset = ABSADataset(opt.dataset_file['test'], tokenizer) assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args() def _print_args(self): n_trainable_params, n_nontrainable_params = 0, 0 for p in self.model.parameters(): n_params = torch.prod(torch.tensor(p.shape)) if p.requires_grad: n_trainable_params += n_params else: n_nontrainable_params += n_params logger.info( 'n_trainable_params: {0}, n_nontrainable_params: {1}'.format( n_trainable_params, n_nontrainable_params)) logger.info('> training arguments:') for arg in vars(self.opt): logger.info('>>> {0}: {1}'.format(arg, getattr(self.opt, arg))) def _reset_params(self): for child in self.model.children(): if type(child) != BertModel: # skip bert params for p in child.parameters(): if p.requires_grad: if len(p.shape) > 1: self.opt.initializer(p) else: stdv = 1. / math.sqrt(p.shape[0]) torch.nn.init.uniform_(p, a=-stdv, b=stdv) def _train(self, criterion, optimizer, train_data_loader, val_data_loader): max_val_acc = 0 max_val_f1 = 0 global_step = 0 path = None for epoch in range(self.opt.num_epoch): logger.info('>' * 100) logger.info('epoch: {}'.format(epoch)) n_correct, n_total, loss_total = 0, 0, 0 # switch model to training mode self.model.train() for i_batch, sample_batched in enumerate(train_data_loader): global_step += 1 # clear gradient accumulators optimizer.zero_grad() inputs = [ sample_batched[col].to(self.opt.device) for col in self.opt.inputs_cols ] outputs = self.model(inputs) targets = sample_batched['polarity'].to(self.opt.device) loss = criterion(outputs, targets) loss.backward() optimizer.step() n_correct += (torch.argmax(outputs, -1) == targets).sum().item() n_total += len(outputs) loss_total += loss.item() * len(outputs) if global_step % self.opt.log_step == 0: train_acc = n_correct / n_total train_loss = loss_total / n_total logger.info('loss: {:.4f}, acc: {:.4f}'.format( train_loss, train_acc)) val_acc, val_f1 = self._evaluate_acc_f1(val_data_loader) logger.info('> val_acc: {:.4f}, val_f1: {:.4f}'.format( val_acc, val_f1)) if val_acc > max_val_acc: max_val_acc = val_acc if not os.path.exists('state_dict'): os.mkdir('state_dict') path = 'state_dict/{0}_{1}_val_acc{2}'.format( self.opt.model_name, self.opt.dataset, round(val_acc, 4)) torch.save(self.model.state_dict(), path) logger.info('>> saved: {}'.format(path)) if val_f1 > max_val_f1: max_val_f1 = val_f1 return path def _evaluate_acc_f1(self, data_loader): n_correct, n_total = 0, 0 t_targets_all, t_outputs_all = None, None # switch model to evaluation mode self.model.eval() with torch.no_grad(): for t_batch, t_sample_batched in enumerate(data_loader): t_inputs = [ t_sample_batched[col].to(self.opt.device) for col in self.opt.inputs_cols ] t_targets = t_sample_batched['polarity'].to( self.opt.device) t_outputs = self.model(t_inputs) n_correct += (torch.argmax(t_outputs, -1) == t_targets).sum().item() n_total += len(t_outputs) if t_targets_all is None: t_targets_all = t_targets t_outputs_all = t_outputs else: t_targets_all = torch.cat((t_targets_all, t_targets), dim=0) t_outputs_all = torch.cat((t_outputs_all, t_outputs), dim=0) acc = n_correct / n_total f1 = metrics.f1_score(t_targets_all.cpu(), torch.argmax(t_outputs_all, -1).cpu(), labels=[0, 1, 2], average='macro') return acc, f1 def run(self): # Loss and Optimizer criterion = nn.CrossEntropyLoss() _params = filter(lambda p: p.requires_grad, self.model.parameters()) optimizer = self.opt.optimizer(_params, lr=self.opt.learning_rate, weight_decay=self.opt.l2reg) train_data_loader = DataLoader(dataset=self.trainset, batch_size=self.opt.batch_size, shuffle=True) test_data_loader = DataLoader(dataset=self.testset, batch_size=self.opt.batch_size, shuffle=False) val_data_loader = DataLoader(dataset=self.valset, batch_size=self.opt.batch_size, shuffle=False) self._reset_params() best_model_path = self._train(criterion, optimizer, train_data_loader, val_data_loader) self.model.load_state_dict(torch.load(best_model_path)) self.model.eval() test_acc, test_f1 = self._evaluate_acc_f1(test_data_loader) logger.info('>> test_acc: {:.4f}, test_f1: {:.4f}'.format( test_acc, test_f1))