def __init__(self, large, temp_dir, finetune=False): super(Bert, self).__init__() if(large): self.model = BertModel.from_pretrained('bert-large-uncased', cache_dir=temp_dir) else: self.model = BertModel.from_pretrained('bert-base-uncased', cache_dir=temp_dir) self.finetune = finetune
def __init__(self, model_path, large, temp_dir, finetune=False): super(Bert, self).__init__() if(large): self.model = BertModel.from_pretrained(model_path) else: self.model = BertModel.from_pretrained(model_path) self.finetune = finetune
def __init__(self, temp_dir, cased=False, finetune=False): super(Bert, self).__init__() if (cased): self.model = BertModel.from_pretrained('BETO', cache_dir=temp_dir) else: self.model = BertModel.from_pretrained('BETO', cache_dir=temp_dir) self.finetune = finetune
def __init__(self, large, temp_dir, finetune=False): super(Bert, self).__init__() if (large): self.model = BertModel.from_pretrained(BERT_PATH, cache_dir=temp_dir) else: self.model = BertModel.from_pretrained(BERT_PATH, cache_dir=temp_dir) self.finetune = finetune
def __init__(self, device, config, labels=None): super().__init__() if config.model == "BertCased": self.bert = BertModel.from_pretrained('bert-base-cased', output_hidden_states=True) else: self.bert = BertModel.from_pretrained('bert-base-uncased', output_hidden_states=True) self.fc = nn.Linear(768*12, labels).to(device) self.device = device
def __init__(self, large, temp_dir, finetune=False): super(Bert, self).__init__() if (large): self.model = BertModel.from_pretrained('bert-large-uncased', cache_dir=temp_dir) else: self.model = BertModel.from_pretrained( "/content/PhoBERT_base_transformers", cache_dir=temp_dir) self.finetune = finetune
def __init__(self, large, temp_dir, finetune=False): super(Bert, self).__init__() if (large): self.model = BertModel.from_pretrained('bert-large-uncased', cache_dir=temp_dir) else: self.model = BertModel.from_pretrained( '/home/ybai/projects/PreSumm/PreSumm/temp/', cache_dir=temp_dir) self.finetune = finetune
def __init__(self, large, temp_dir, finetune=False): super(Bert, self).__init__() if(large): # self.model = BertModel.from_pretrained('bert-large-uncased', cache_dir=temp_dir) self.model = BertModel.from_pretrained(temp_dir, cache_dir='~/.cache/torch/pytorch_transformers') else: # self.model = BertModel.from_pretrained('bert-base-uncased', cache_dir=temp_dir) self.model = BertModel.from_pretrained(temp_dir, cache_dir='~/.cache/torch/pytorch_transformers') self.finetune = finetune
def __init__(self, bert_model, temp_dir, finetune=False): super(Bert, self).__init__() if (bert_model == 'bert-base-multilingual-cased'): self.model = BertModel.from_pretrained( 'bert-base-multilingual-cased', cache_dir=temp_dir) else: self.model = BertModel.from_pretrained(bert_model, cache_dir=temp_dir) self.model.resize_token_embeddings(31756) self.word_embeddings = nn.Embedding(31756, 768, padding_idx=0) self.finetune = finetune
def __init__(self, large, temp_dir, finetune=False): super(Bert, self).__init__() if (large): self.model = BertModel.from_pretrained('bert-large-uncased', cache_dir=temp_dir) else: self.model = BertModel.from_pretrained( 'bert-base-chinese', cache_dir= r'C:\Users\Administrator\PycharmProjects\one\bertsum-chinese-LAI\temp' ) self.finetune = finetune #ture
def __init__(self, large, temp_dir, finetune=False): super(Bert, self).__init__() if (large): #self.model = BertModel.from_pretrained('bert-base-multilingual-cased', cache_dir=temp_dir) config = BertConfig.from_json_file('bert-large/config.json') self.model = BertModel.from_pretrained('bert-large', cache_dir=None, config=config) else: self.model = BertModel.from_pretrained( 'bert-base-multilingual-cased', cache_dir=temp_dir) self.finetune = finetune
def __init__(self, device='cpu', language='en', labels=None): super().__init__() if language == 'en': self.bert = BertModel.from_pretrained('bert-base-cased') else: self.bert = BertModel.from_pretrained('bert-base-chinese') self.fc = nn.Linear(1536, 4) self.device = device self.lstm = nn.LSTM(input_size=768, hidden_size=768, num_layers=1, dropout=0, bidirectional=True)
def __init__(self, data, args, num_episodes=None): self.data = data self.args = args self.num_episodes = num_episodes self.all_classes = np.unique(self.data['label']) self.num_classes = len(self.all_classes) if self.num_classes < self.args.way: raise ValueError("Total number of classes is less than #way.") self.idx_list = [] for y in self.all_classes: self.idx_list.append( np.squeeze(np.argwhere(self.data['label'] == y))) self.count = 0 self.done_queue = Queue() self.num_cores = cpu_count() if args.n_workers is 0 else args.n_workers # print("{}, Initializing parallel data loader with {} processes".format( # datetime.datetime.now().strftime('%02y/%02m/%02d %H:%M:%S'), # self.num_cores), flush=True) if self.args.bert_path is not None: # use bert online print("{}, Loading pretrained bert from {}".format( datetime.datetime.now().strftime('%02y/%02m/%02d %H:%M:%S'), self.args.bert_path), flush=True) if self.args.cuda != -1: self.model = BertModel.from_pretrained( 'bert-base-uncased', cache_dir=self.args.bert_path).cuda(self.args.cuda) else: self.model = BertModel.from_pretrained( 'bert-base-uncased', cache_dir=self.args.bert_path) self.model.eval() self.p_list = [] for i in range(self.num_cores): self.p_list.append( Process(target=self.worker, args=(self.done_queue,))) for i in range(self.num_cores): self.p_list[i].start()
def main(raw_args=None): parser = argparse.ArgumentParser() parser.add_argument("--model_name", type=str, required=True, help="model name e.g. bert-base-uncased") parser.add_argument("--cache_dir", type=str, default=None, required=False, help="Directory containing pytorch model") parser.add_argument("--pytorch_model_path", type=str, required=True, help="/path/to/<pytorch-model-name>.bin") parser.add_argument("--tf_cache_dir", type=str, required=True, help="Directory in which to save tensorflow model") args = parser.parse_args(raw_args) model = BertModel.from_pretrained( pretrained_model_name_or_path=args.model_name, state_dict=torch.load(args.pytorch_model_path), cache_dir=args.cache_dir ) convert_pytorch_checkpoint_to_tf( model=model, ckpt_dir=args.tf_cache_dir, model_name=args.model_name )
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer) self.testset = ABSADataset(opt.dataset_file['test'], tokenizer) assert 0 <= opt.valset_ratio < 1 if opt.valset_ratio > 0: valset_len = int(len(self.trainset) * opt.valset_ratio) self.trainset, self.valset = random_split( self.trainset, (len(self.trainset) - valset_len, valset_len)) else: self.valset = self.testset if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) # bert = BertModel.from_pretrained(opt.pretrained_bert_name) config = BertConfig.from_pretrained(opt.pretrained_bert_name, output_attentions=True) bert = BertModel.from_pretrained(opt.pretrained_bert_name, config=config) self.pretrained_bert_state_dict = bert.state_dict() self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer) self.testset = ABSADataset(opt.dataset_file['test'], tokenizer) if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) self._print_args()
def __init__(self, num_labels=2): super(BertForSequenceClassification, self).__init__() self.num_labels = num_labels self.bert = BertModel.from_pretrained('bert-base-uncased') self.dropout = nn.Dropout(dropout_prob) self.classifier = nn.Linear(hidden_size, num_labels) nn.init.xavier_normal_(self.classifier.weight)
def __init__( self, bert_model_or_path: str = "bert-base-uncased", layers: str = "-1,-2,-3,-4", pooling_operation: str = "first", use_scalar_mix: bool = False, ): """ Bidirectional transformer embeddings of words, as proposed in Devlin et al., 2018. :param bert_model_or_path: name of BERT model ('') or directory path containing custom model, configuration file and vocab file (names of three files should be - config.json, pytorch_model.bin/model.chkpt, vocab.txt) :param layers: string indicating which layers to take for embedding :param pooling_operation: how to get from token piece embeddings to token embedding. Either pool them and take the average ('mean') or use first word piece embedding as token embedding ('first) """ super().__init__() self.tokenizer = BertTokenizer.from_pretrained(bert_model_or_path) self.model = BertModel.from_pretrained( pretrained_model_name_or_path=bert_model_or_path, output_hidden_states=True) self.layer_indexes = [int(x) for x in layers.split(",")] self.pooling_operation = pooling_operation self.use_scalar_mix = use_scalar_mix self.name = str(bert_model_or_path) self.static_embeddings = True self.model.to(flair.device) self.model.eval()
def __init__(self, config: BertEncoderConfig = BertEncoderConfig()): assert isinstance(config, BertEncoderConfig) super().__init__(config) self.config = config self.model = BertModel.from_pretrained(self.config.mode, output_hidden_states=True) self.freeze_layers()
def __init__(self, rnn_type, bert_model_path, bert_embedding_dim, hidden_dim, n_layers, bidirectional, batch_first, dropout, num_classes): super(BertRNN, self).__init__() self.rnn_type = rnn_type.lower() self.bidirectional = bidirectional self.hidden_dim = hidden_dim self.n_layers = n_layers self.bert = BertModel.from_pretrained(bert_model_path) for param in self.bert.parameters(): param.requires_grad = True if rnn_type == 'lstm': self.rnn = nn.LSTM(bert_embedding_dim, hidden_size=hidden_dim, num_layers=n_layers, bidirectional=bidirectional, batch_first=batch_first, dropout=dropout) elif rnn_type == 'gru': self.rnn = nn.GRU(bert_embedding_dim, hidden_size=hidden_dim, num_layers=n_layers, bidirectional=bidirectional, batch_first=batch_first, dropout=dropout) else: self.rnn = nn.RNN(bert_embedding_dim, hidden_size=hidden_dim, num_layers=n_layers, bidirectional=bidirectional, batch_first=batch_first, dropout=dropout) self.dropout = nn.Dropout(dropout) self.fc_rnn = nn.Linear(hidden_dim * 2, num_classes)
def get_bert(bert_type='bert'): tokenizer, model = None, None if (bert_type == 'bert'): ######## bert ########### tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertModel.from_pretrained('bert-base-uncased') ######################## if (bert_type == 'biobert'): #### Bio BERT ######### model = bm.from_pretrained('biobert_v1.1_pubmed') tokenizer = BertTokenizer(vocab_file="biobert_v1.1_pubmed/vocab.txt", do_lower_case=True) #### Bio BERT ######### if (bert_type == 'scibert'): #### sci bert ######### config = AutoConfig.from_pretrained('allenai/scibert_scivocab_uncased', output_hidden_states=False) tokenizer = AutoTokenizer.from_pretrained( 'allenai/scibert_scivocab_uncased') model = AutoModel.from_pretrained('allenai/scibert_scivocab_uncased', config=config) ####################### return tokenizer, model
def __init__(self, opt, model_path: str): super(ModelTrained, self).__init__() self.opt = opt self.model_path = model_path bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) self.model.load_state_dict(torch.load(model_path))
def load_bert_to_cache(cache: Dict): bert_model_name = 'bert-base-multilingual-cased' if 'bert_tokenizer' not in cache: cache['bert_tokenizer'] = BertTokenizer.from_pretrained( bert_model_name) if 'bert_model' not in cache: cache['bert_model'] = BertModel.from_pretrained(bert_model_name)
def __init__(self, bert_model_path, large, temp_dir, finetune=False): super(Bert, self).__init__() self.model_name = 'bert-large-uncased' if large else 'bert-base-uncased' self.model = BertModel.from_pretrained(path.join( bert_model_path, self.model_name), cache_dir=temp_dir) self.finetune = finetune
def __init__(self): model_classes = { # 'bert_spc': BERT_SPC, # 'aen_bert': AEN_BERT, 'lcf_bert': LCF_BERT } # set your trained models here state_dict_paths = { 'lcf_bert': 'models/lcf_bert_combined_val_acc0.7563', # 'lcf_bert': 'models/lcf_bert_twitter_val_acc0.7283', # 'lcf_bert': 'models/lcf_bert_twitter_val_acc0.7225', # 'bert_spc': 'state_dict/bert_spc_laptop_val_acc0.268', # 'aen_bert': 'state_dict/aen_bert_laptop_val_acc0.2006' } self.opt = get_parameters() self.opt.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # self.opt.device = torch.device("cpu") print("TargetSentimentAnalyzer using device:",self.opt.device) self.tokenizer = Tokenizer4Bert(self.opt.max_seq_len, self.opt.pretrained_bert_name) self.bert = BertModel.from_pretrained(self.opt.pretrained_bert_name) self.model = model_classes[self.opt.model_name](self.bert, self.opt).to(self.opt.device) print('loading model {0} ...'.format(self.opt.model_name)) self.model.load_state_dict(torch.load(state_dict_paths[self.opt.model_name], map_location=torch.device('cpu'))) # if torch.cuda.is_available(): # else: # self.model.load_state_dict(torch.load(state_dict_paths[self.opt.model_name])) self.model.eval() torch.autograd.set_grad_enabled(False)
def __init__(self, server): self.server = server self.dimensions = 768 transformer_type = 'bert-base-uncased' self.tokenizer = BertTokenizer.from_pretrained(transformer_type) self.model = BertModel.from_pretrained(transformer_type) self.model.eval()
def main(): training_dictionaries = load_training_dictionaries() bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') bert_model = BertModel.from_pretrained('bert-base-uncased') extractor_model = ExtractorModel(bert_tokenizer, bert_model) # train_extractor( # extractor_model, # data=training_dictionaries # ) abstractor_model = AbstractorModelRNN(bert_tokenizer, bert_model) # train_abstractor( # abstractor_model, # data=training_dictionaries # ) rl_model = RLModel( extractor_model, abstractor_model, ) train_system( rl_model=rl_model, data=training_dictionaries )
def __init__(self, args, temp_dir, finetune=False): super(Bert, self).__init__() if args.pretrained_model_type in ['bert-base-uncased', 'bert-base-multilingual-uncased']: self.model = BertModel.from_pretrained(args.pretrained_model_type, cache_dir=temp_dir) if args.pretrained_model_type in ['rubert-deeppavlov']: name = args.pretrained_model_type config = BertConfig.from_json_file(mapper(name, 'config')) self.model = BertModel.from_pretrained(mapper(name, 'model'), config=config) if not self.model: raise NotImplementedError("self.model") bert_data = BertData(args) self.model.resize_token_embeddings(len(bert_data.tokenizer)) self.finetune = finetune
def __init__(self, args, dictionary, left_pad=False): super().__init__(dictionary) self.dropout = args.dropout from pytorch_transformers import RobertaModel, BertModel from pytorch_transformers.file_utils import PYTORCH_TRANSFORMERS_CACHE from pytorch_transformers import RobertaConfig, RobertaTokenizer, BertConfig, BertTokenizer if args.pretrained_bert_model.startswith('roberta'): self.embed = RobertaModel.from_pretrained( args.pretrained_bert_model, cache_dir=PYTORCH_TRANSFORMERS_CACHE / 'distributed_{}'.format(args.distributed_rank)) # self.context = RobertaModel.from_pretrained(args.pretrained_bert_model, # cache_dir=PYTORCH_TRANSFORMERS_CACHE / 'distributed_{}'.format(args.distributed_rank)) self.config = RobertaConfig.from_pretrained( args.pretrained_bert_model) self.tokenizer = RobertaTokenizer.from_pretrained('roberta-base') else: self.embed = BertModel.from_pretrained( args.pretrained_bert_model, cache_dir=PYTORCH_TRANSFORMERS_CACHE / 'distributed_{}'.format(args.distributed_rank)) # self.context = BertModel.from_pretrained(args.pretrained_bert_model, # cache_dir=PYTORCH_TRANSFORMERS_CACHE / 'distributed_{}'.format(args.distributed_rank)) self.config = BertConfig.from_pretrained( args.pretrained_bert_model) self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') self.padding_idx = self.tokenizer.convert_tokens_to_ids( self.tokenizer.pad_token)
def __init__(self, opt): self.opt = opt if 'bert' in opt.model_name: tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name) bert = BertModel.from_pretrained(opt.pretrained_bert_name) self.model = opt.model_class(bert, opt).to(opt.device) else: tokenizer = build_tokenizer( fnames=[opt.dataset_file['train'], opt.dataset_file['test1']], max_seq_len=opt.max_seq_len, dat_fname='{0}_tokenizer.dat'.format(opt.dataset)) embedding_matrix = build_embedding_matrix( word2idx=tokenizer.word2idx, embed_dim=opt.embed_dim, dat_fname='{0}_{1}_embedding_matrix.dat'.format( str(opt.embed_dim), opt.dataset)) self.model = opt.model_class(embedding_matrix, opt).to(opt.device) self.testset = ABSADataset(opt.dataset_file['test1'], tokenizer) if opt.device.type == 'cuda': logger.info('cuda memory allocated: {}'.format( torch.cuda.memory_allocated(device=opt.device.index))) model_path = 'state_dict/bert_spc_law_val_acc0.5314.hdf5' # provide best model path self.model.load_state_dict(torch.load(model_path))