def initialize(self, ctx): properties = ctx.system_properties MODEL_DIR = properties.get("model_dir") self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda. is_available() else "cpu") self.labelencoder = preprocessing.LabelEncoder() self.labelencoder.classes_ = np.load( os.path.join(MODEL_DIR, 'classes.npy')) config = BertConfig(os.path.join(MODEL_DIR, 'bert_config.json')) self.model = BertForSequenceClassification( config, num_labels=len(self.labelencoder.classes_)) self.model.load_state_dict( torch.load(os.path.join(MODEL_DIR, 'pytorch_model.bin'), map_location="cpu")) self.model.to(self.device) self.model.eval() self.tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") self.softmax = torch.nn.Softmax(dim=-1) # self.batch_size = batch_size logger.debug( 'Transformer model from path {0} loaded successfully'.format( MODEL_DIR)) self.manifest = ctx.manifest self.initialized = True
def get_bert_inference_model(bert_model, batch_size=16, max_seq_length=100): tokenizer = BertTokenizer.from_pretrained(os.path.join( bert_model, "vocab.txt"), do_lower_case=True) transform = BertClassificationTransform(tokenizer=tokenizer, is_test=True, max_len=max_seq_length) state_save_path = os.path.join(bert_model, 'model.state') if os.path.exists(state_save_path): state = torch.load(state_save_path, map_location="cpu") model = BertForSequenceClassification.from_pretrained( bert_model, num_labels=2, state_dict=state['model_state']) else: previous_model_file = os.path.join(bert_model, "pytorch_model.bin") model_state_dict = torch.load(previous_model_file, map_location="cpu") model = BertForSequenceClassification.from_pretrained( bert_model, state_dict=model_state_dict, num_labels=2) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) inference_model = BertInferenceModel(model, transform, device, batch_size=batch_size) return inference_model
def create_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForSequenceClassification(config=config, num_labels=self.num_labels) model.eval() loss = model(input_ids, token_type_ids, input_mask, sequence_labels) logits = model(input_ids, token_type_ids, input_mask) outputs = { "loss": loss, "logits": logits, } return outputs
def load_pretrained_model_tokenizer(model_type="BertForSequenceClassification", device="cuda", config=None): bert_model = config['bert_model'] # Load pre-trained model (weights) if model_type == "BertForSequenceClassification": model = BertForSequenceClassification.from_pretrained(bert_model, num_labels=2) # Load pre-trained model tokenizer (vocabulary) elif model_type == "BertForNextSentencePrediction": model = BertForNextSentencePrediction.from_pretrained(bert_model) elif model_type == "specific_shared": model = SpecificShared(config) elif model_type == "siamese_bert": model = SiameseBert(config) elif model_type == "n_bert": model = nBert(config) elif model_type == "bert_sts": model = BertSts(config) elif model_type == "bert_fine_tune": model = BertFineTune(config) else: print("[Error]: unsupported model type") return None, None tokenizer = BertTokenizer.from_pretrained(bert_model) model.to(device) print("Initialized model and tokenizer") return model, tokenizer
def load_pretrained_model_tokenizer(model_type="BertForSequenceClassification", base_model=None, base_tokenizer=None, device="cuda", chinese=False, num_labels=2): # Load pre-trained model (weights) if base_model is None: if chinese: base_model = "bert-base-chinese" else: base_model = "bert-base-uncased" if model_type == "BertForSequenceClassification": model = BertForSequenceClassification.from_pretrained( base_model, num_labels=num_labels) # Load pre-trained model tokenizer (vocabulary) elif model_type == "BertForNextSentencePrediction": model = BertForNextSentencePrediction.from_pretrained(base_model) elif model_type == "BertForTokenClassification": model = BertForTokenClassification.from_pretrained( base_model, num_labels=num_labels) elif model_type == "BertMSE": model = BertMSE() else: print("[Error]: unsupported model type") return None, None if base_tokenizer is None: # Download from huggingface tokenizer = BertTokenizer.from_pretrained(base_model) else: # Load local file tokenizer = BertTokenizer.from_pretrained(base_tokenizer) model.to(device) return model, tokenizer
def __init__(self): credentialFilePath = 'twitter_credentials.json' # TODO add that file to .gitignore with open(credentialFilePath, "r") as file: creds = json.load(file) self.twitter = Twython(creds['CONSUMER_KEY'], creds['CONSUMER_SECRET'], creds['ACCESS_TOKEN'], creds['ACCESS_SECRET']) verification = self.twitter.verify_credentials() print('Login Verification:' + verification['name']) model_state_dict = torch.load(output_model_file) self.loaded_model = BertForSequenceClassification.from_pretrained( modelfilePath, state_dict=model_state_dict, num_labels=2) self.loaded_model.cuda() device_name = tf.test.gpu_device_name() if device_name != '/device:GPU:0': raise SystemError('GPU device not found') print('Found GPU at: {}'.format(device_name)) self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") torch.cuda.get_device_name(0) if self.loaded_model is None: raise ModelError( "Sequence Classification Model was not properly loaded!!") print("Model loaded") if TwitterAnalyzer.__instance != None: raise Exception("This class is a singleton!") else: TwitterAnalyzer.__instance = self
def __init__(self, model_type, db, freeze_bert=False): super(CodahClassifier, self).__init__() # instantiating BERT model object size = model_type.split('-')[1] if size == 'large': output_dim = 1024 else: output_dim = 768 self.bert_layer = BertForSequenceClassification.from_pretrained( model_type, num_labels=1) # freeze bert layers if freeze_bert: for p in self.bert_layer.parameters(): p.requires_grad = False self.asn = DrawModel(db) for p in self.asn.parameters(): p.requires_grad = False self.obj_cls = nn.Linear(db.cfg.output_cls_size, 1) self.att_layer = nn.Linear(db.cfg.num_scales * 2, 1) self.att_cls = nn.Linear(db.cfg.grid_size[0]**2, 1) self.pos_cls = nn.Linear(db.cfg.grid_size[0]**2, 1) self.relu = nn.ReLU()
def test(): # 配置文件 cf = Config('./config.yaml') # 有GPU用GPU device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 测试数据 test_data = NewsDataset("./data/cnews_final_test.txt", cf.max_seq_len) test_dataloader = DataLoader(test_data, batch_size=cf.batch_size, shuffle=True) # 模型 config = BertConfig("./output/pytorch_bert_config.json") model = BertForSequenceClassification(config, num_labels=cf.num_labels) model.load_state_dict(torch.load("./output/pytorch_model.bin")) # 把模型放到指定设备 model.to(device) # 让模型并行化运算 if torch.cuda.device_count() > 1: model = torch.nn.DataParallel(model) # 训练 start_time = time.time() data_len = len(test_dataloader) model.eval() y_pred = np.array([]) y_test = np.array([]) # for step,batch in enumerate(tqdm(test_dataloader,"batch",total=len(test_dataloader))): for step, batch in enumerate(test_dataloader): label_id = batch['label_id'].squeeze(1).to(device) word_ids = batch['word_ids'].to(device) segment_ids = batch['segment_ids'].to(device) word_mask = batch['word_mask'].to(device) loss = model(word_ids, segment_ids, word_mask, label_id) with torch.no_grad(): pred = get_model_labels(model, word_ids, segment_ids, word_mask) y_pred = np.hstack((y_pred, pred)) y_test = np.hstack((y_test, label_id.to("cpu").numpy())) # 评估 print("Precision, Recall and F1-Score...") print( metrics.classification_report(y_test, y_pred, target_names=get_labels('./data/label'))) # 混淆矩阵 print("Confusion Matrix...") cm = metrics.confusion_matrix(y_test, y_pred) print(cm)
def __init__(self, vocab: Vocabulary, pretrained_bert_model_file: str, num_labels: int) -> None: super().__init__(vocab) self.bert_sc_model = HuggingFaceBertSC.from_pretrained( pretrained_bert_model_file, num_labels=num_labels) self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss()
def __init__(self): log.info('Instantiating model') model_type = 'bert-base-cased' self.model = BertForSequenceClassification.from_pretrained( model_type, cache_dir=None, num_labels=1) log.info('Instantiating tokenizer') self.tokenizer = BertTokenizer.from_pretrained(model_type)
def __init__(self,model="",num_labels =2): print('kaishi') self.num_labels =num_labels self.tokenizer = BertTokenizer.from_pretrained(model) self.model = BertForSequenceClassification.from_pretrained(model,num_labels = num_labels) # model = BertForSequenceClassification.from_pretrained('bert-base-chinese') # model = BertForNextSentencePrediction.from_pretrained('bert-base-chinese') self.model.eval()
def __init__(self, pretrain_path, dropout=0.1): super(PairModel, self).__init__() self.bert = BertForSequenceClassification.from_pretrained( pretrain_path, cache_dir=None, num_labels=1) self.head = nn.Sequential( OrderedDict([ ('dropout', nn.Dropout(dropout)), ('clf', nn.Linear(self.bert.config.hidden_size, 1)), ]))
def from_scratch(cls, bert_model, verbose=True): tokenizer = BertTokenizer.from_pretrained(bert_model) bert = BertForSequenceClassification.from_pretrained(bert_model, num_labels=2) model = cls(tokenizer, bert) if verbose: print('Intialized the model and the tokenizer from scratch') return model
def finetune_bert(args, model_save_directory, bert_file_path): train_data, valid_data = data_utils.make_datasets(args) model = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=4) args.save_path = model_save_directory + "bert_model.pt" ACC, _ = train_utils.train_model(train_data, valid_data, model, args, 0, "bert")
def __init__(self, n_epochs=4, batch_size=10, lr=2e-5): self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True) self.MAX_LEN = 128 self.batch_size = batch_size self.n_epochs = n_epochs self.device = torch.device('cuda:0') self.model = BertForSequenceClassification.from_pretrained( "bert-base-uncased", num_labels=2) self.lr = lr
def from_model_path(cls, output_model_path, verbose=True): tokenizer = BertTokenizer.from_pretrained(output_model_path) bert = BertForSequenceClassification.from_pretrained(output_model_path, num_labels=2) model = cls(tokenizer, bert) if verbose: print('Restored the model and the tokenizer from {}'.format( output_model_path)) return model
def load_pytorch_model(path, *args, **kwargs): state_dict = torch.load(path) new_state_dict = OrderedDict() for k, v in state_dict.items(): name = k[7:] # remove `module.` new_state_dict[name] = v #model = SimpleLSTM(*args, **kwargs) model = BertForSequenceClassification.from_pretrained(BERT_MODEL_PATH, num_labels=1) model.load_state_dict(new_state_dict) return model
def __init__(self, device, n_ctx_embs, ctx_emb_dim, pretrain): """ The value of the parameters should also be specified in the BCN model config. """ self.device = device self.n_ctx_embs = n_ctx_embs self.ctx_emb_dim = ctx_emb_dim self.tokenizer = BertTokenizer.from_pretrained(pretrain) self.model = BertForSequenceClassification.from_pretrained(pretrain) self.model.eval() self.model.to(self.device)
def _load_model(self) -> None: self.device = torch.device("cuda" if torch.cuda.is_available() and not self.no_cuda else "cpu") self.n_gpu = torch.cuda.device_count() # Load a trained model and vocabulary that you have fine-tuned self.model = BertForSequenceClassification.from_pretrained( self.model_dir, num_labels=self.num_labels) self.tokenizer = BertTokenizer.from_pretrained( self.model_dir, do_lower_case=self.do_lower_case) self.model.to(self.device)
def __init__(self): self.args = GlobalVariable.get_value('BERT_ARGS') self.model = BertForSequenceClassification.from_pretrained(self.args.get('bert_model'), cache_dir=PYTORCH_PRETRAINED_BERT_CACHE / 'distributed_{}'.format( self.args.get('local_rank'))) self.processors = {'SentencePro': sentencePro} self.processor = self.processors['SentencePro']() self.label_list = self.processor.get_labels() self.device = torch.device("cuda" if torch.cuda.is_available() and not self.args.get('no_cuda') else "cpu") self.tokenizer = BertTokenizer.from_pretrained(self.args.get('bert_model')) self.model.load_state_dict(GlobalVariable.get_value('NEW_STATE_DICT'))
def __init__(self, output_dir="./models/"): # get device self.__device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") n_gpu = torch.cuda.device_count() # get model from weight self.__model = BertForSequenceClassification.from_pretrained( output_dir, num_labels=28) # create tokenizer self.__tokenizer = BertTokenizer.from_pretrained(output_dir) self.__model.eval()
def load_info(): # Load BertForSequenceClassification, the pretrained BERT model with a single linear classification layer on top. model = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=2) model.cuda() model.load_state_dict(torch.load("model/model.pt")) model.eval() tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True) return model, tokenizer
def __init__(self, dir_path, max_seq_length=30): self.max_seq_length = max_seq_length self.processor = PredicateClassificationProcessor.load(os.path.join(dir_path, PROCESSOR_NAME)) self.tokenizer = BertTokenizer.from_pretrained(dir_path) self.classifier = BertForSequenceClassification.from_pretrained(dir_path, len(self.processor.labels)).to(self.device) self.classifier.eval() self.id2label = {i: label for i, label in enumerate(self.processor.labels)} global debug_message debug_message = False
def init_bert(pre_trained='bert-base-cased', num_labels=2): # 分词器 tokenizer = BertTokenizer.from_pretrained(pre_trained) # 获得hidden states model = BertForSequenceClassification.from_pretrained( pre_trained, num_labels=num_labels) model.eval() # print('成功加载bert模型') return tokenizer, model
def _create_net_and_optim(self, net_cfg, optim_cfg, num_train_optimization_steps): net = BertForSequenceClassification.from_pretrained(net_cfg.bert_pretrain, net_cfg.num_labels) net.to(device=self._device) param_optimizer = filter(lambda p: p.requires_grad, net.parameters()) if num_train_optimization_steps != None: optim = BertAdam(param_optimizer, t_total=num_train_optimization_steps, **optim_cfg.kwargs) else: optim = None return net, optim
def __init__(self, bert_model_dir, class_size, feature_lens, seed=None, hidden_size=75, dropout_rate_fc=0.2, num_layers=2, input_dropout=.3, lstm_dropout=0.5, fine_tune_embeddings=True): self.fine_tune_embeddings = fine_tune_embeddings if seed is None: seed = torch.initial_seed() & ((1 << 63) - 1) self.logger.info("Using seed {}".format(seed)) torch.manual_seed(seed) super(RelationExtractorBertBiLstmNetworkNoPos, self).__init__() # Use random weights if vocab size if passed in else load pretrained weights self.text_column_index = feature_lens.argmax(axis=0) self.max_sequence_len = feature_lens[self.text_column_index] self.logger.info( "The text feature is index {}, the feature lengths are {}".format( self.text_column_index, feature_lens)) bidirectional = True num_directions = 2 if bidirectional else 1 self.embeddings = BertForSequenceClassification.from_pretrained( bert_model_dir, num_labels=class_size).bert.embeddings.word_embeddings self.lstm = nn.Sequential( nn.Dropout(p=input_dropout), nn.LSTM(self.embeddings.embedding_dim, hidden_size=hidden_size, num_layers=num_layers, batch_first=True, bidirectional=bidirectional, dropout=lstm_dropout)) # self.fc_input_size = (hidden_size * num_directions) * self.max_sequence_len self._class_size = class_size self.fc = nn.Sequential(nn.Dropout(dropout_rate_fc), nn.Linear(self.fc_input_size, class_size))
def create_bert_for_sequence_classification_attn(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels): # Disable attention dropout drop = config.attention_probs_dropout_prob config.attention_probs_dropout_prob = 0.0 model = BertForSequenceClassification(config=config, num_labels=self.num_labels) config.attention_probs_dropout_prob = drop loss, _ = model(input_ids, token_type_ids, input_mask, sequence_labels, return_att=True) logits, attn = model(input_ids, token_type_ids, input_mask, return_att=True) outputs = { "loss": loss, "logits": logits, "attn": attn, } return outputs
def main_bert(args, DEVICE): train_dataloader, valid_dataloader, test_dataloader, num_train_examples = load_data_bert( args, DEVICE) model = BertForSequenceClassification.from_pretrained( 'bert-base-uncased', cache_dir='../result/', num_labels=2) # Train if no pre-trained model is given if args.new_train: train_bert(args, [ train_dataloader, valid_dataloader, test_dataloader, num_train_examples ], model, DEVICE) interpret_bert(args, test_dataloader, DEVICE)
def build_model(self, param): num_classes = param['num_classes'] bert_base = param['bert_base'] bert_model = param['bert_model'] model = BertForSequenceClassification.from_pretrained( bert_model, num_labels=num_classes) # model = BertForSequenceClassification.from_pretrained(bert_base, num_labels=num_classes) if torch.cuda.is_available(): model.cuda() # model.summary() self.model = model self.model_param = param
def configure_model(): model = BertForSequenceClassification.from_pretrained(BERT_MODEL_PATH,cache_dir=None,num_labels=1) model.zero_grad() model = model.to(device) param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] lr = 3e-5 epsilon=1 lr_d = {} weight_d = {} for n, p in param_optimizer: if any(nd in n for nd in no_decay): weight_d[n] = 0.0 else: weight_d[n] = 0.01 for n, p in param_optimizer[:5]: lr_d[n] = lr*(epsilon**(11)) for n, p in param_optimizer: if 'bert.encoder.layer.' in n: for i in range(0, 12): if 'bert.encoder.layer.'+str(i)+'.' in n: lr_d[n] = lr*(epsilon**(11-i)) break for n, p in param_optimizer[-4:]: lr_d[n] = lr comb_dict = {} for n, p in param_optimizer: para = (weight_d[n], lr_d[n]) if para in comb_dict: comb_dict[para].append(p) else: comb_dict[para] = [p] optimizer_grouped_parameters = [] for i, j in comb_dict.items(): optimizer_grouped_parameters.append({'params':j, 'weight_decay' : i[0], 'lr' : i[1]}) train = train_dataset num_train_optimization_steps = int(EPOCHS*len(train)/batch_size/accumulation_steps) optimizer = BertAdam(optimizer_grouped_parameters, lr=lr, warmup=0.05, t_total=num_train_optimization_steps) model, optimizer = amp.initialize(model, optimizer, opt_level="O1",verbosity=0) model=model.train() return model, optimizer, train