def get_kobert_model(model_file, vocab_file, ctx="cpu"): bertmodel = BertModel(config=BertConfig.from_dict(bert_config)) bertmodel.load_state_dict(torch.load(model_file)) device = torch.device(ctx) bertmodel.to(device) bertmodel.eval() vocab_b_obj = nlp.vocab.BERTVocab.from_json( open(vocab_file, 'rt').read()) return bertmodel, vocab_b_obj
def load_model(model_type): checkpoint = torch.load(f'checkpoints/{model_type}.pt', map_location='cpu') config = BertConfig.from_json_file('models/config.json') model = Summarizer(args=None, device="cpu", load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) return model
def __init__(self, model_name, epochs=1, batch_size=64, base_batch_size=32, part=1., half=2, seed=1234, debug_mode=False): self.device = torch.device('cuda') self.input_dir = "../input" self.work_dir = "../working/" self.debug_mode = debug_mode self.model_name = model_name self.half = half self.seed = seed self.identity_list = [ 'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish', 'muslim', 'black', 'white', 'psychiatric_or_mental_illness' ] self.toxicity_type_list = [ 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat' ] self.stopwords = '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n“”’\'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—' self.seed_everything() self.max_len = 220 self.epochs = epochs self.base_batch_size = base_batch_size self.batch_size = batch_size self.split_ratio = 0.95 self.sample_num = 1804874 if not self.debug_mode: self.train_df = pd.read_csv( os.path.join( "../input/jigsaw-unintended-bias-in-toxicity-classification/train.csv" )).sample(int(self.sample_num * part), random_state=1234).fillna(0.) self.test_df = pd.read_csv( os.path.join( "../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv" )) else: self.train_df = pd.read_csv( os.path.join( "../input/jigsaw-unintended-bias-in-toxicity-classification/train.csv" )).head(1000).fillna(0.) self.test_df = pd.read_csv( os.path.join( "../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv" )).head(1000) self.train_len = int(len(self.train_df) * self.split_ratio) self.test_len = len(self.test_df) self.bert_config = BertConfig( '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/' + 'bert_config.json') self.bert_model_path = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
def test(): # 配置文件 cf = Config('./config.yaml') # 有GPU用GPU device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 测试数据 test_data = NewsDataset("./data/cnews_final_test.txt", cf.max_seq_len) test_dataloader = DataLoader(test_data, batch_size=cf.batch_size, shuffle=True) # 模型 config = BertConfig("./output/pytorch_bert_config.json") model = BertForSequenceClassification(config, num_labels=cf.num_labels) model.load_state_dict(torch.load("./output/pytorch_model.bin")) # 把模型放到指定设备 model.to(device) # 让模型并行化运算 if torch.cuda.device_count() > 1: model = torch.nn.DataParallel(model) # 训练 start_time = time.time() data_len = len(test_dataloader) model.eval() y_pred = np.array([]) y_test = np.array([]) # for step,batch in enumerate(tqdm(test_dataloader,"batch",total=len(test_dataloader))): for step, batch in enumerate(test_dataloader): label_id = batch['label_id'].squeeze(1).to(device) word_ids = batch['word_ids'].to(device) segment_ids = batch['segment_ids'].to(device) word_mask = batch['word_mask'].to(device) loss = model(word_ids, segment_ids, word_mask, label_id) with torch.no_grad(): pred = get_model_labels(model, word_ids, segment_ids, word_mask) y_pred = np.hstack((y_pred, pred)) y_test = np.hstack((y_test, label_id.to("cpu").numpy())) # 评估 print("Precision, Recall and F1-Score...") print( metrics.classification_report(y_test, y_pred, target_names=get_labels('./data/label'))) # 混淆矩阵 print("Confusion Matrix...") cm = metrics.confusion_matrix(y_test, y_pred) print(cm)
def get_kobert_model(model_file, vocab_file, ctx="cpu"): bertmodel = BertModel(config=BertConfig.from_dict(bert_config)) bertmodel.load_state_dict(torch.load(model_file)) device = torch.device(ctx) bertmodel.to(device) bertmodel.eval() vocab_b_obj = nlp.vocab.BERTVocab.from_sentencepiece(vocab_file, padding_token='[PAD]') return bertmodel, vocab_b_obj
def __init__(self): # configuration self.ROOT_FOLDER = os.path.dirname(__file__) print('ROOT_FOLDER', self.ROOT_FOLDER) DIRECTORIES = { 'ml_hate_speech_path': os.path.join(self.ROOT_FOLDER, 'models/ml_hate_speech_classifier') } # Load a trained model that you have fine-tuned self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.bert_model = 'bert-base-multilingual-uncased' self.model_file = os.path.join( DIRECTORIES['ml_hate_speech_path'], 'pytorch_model_epoch_20_seqlen_256.bin') # self.model_file = 'pytorch_model_epoch_20_seqlen_256.bin' print('model_file', self.model_file) print('model_dir', os.listdir(os.path.join(self.ROOT_FOLDER, 'models'))) print('model_dir_s', os.listdir(DIRECTORIES['ml_hate_speech_path'])) print(os.path.isfile(self.model_file)) if not os.path.isfile(self.model_file): print('Please Download the model ...') exit(0) # # model_download_file = "sh " + self.ROOT_FOLDER+'/model_download.sh' # os.system('ls /usr/src/app/project/') # print('model_download_file ',model_download_file, os.path.isfile(model_download_file)) # os.system(model_download_file) # print(self.model_file) if torch.cuda.is_available(): model_state_dict = torch.load(self.model_file) else: print('Loading model ...', self.model_file) model_state_dict = torch.load(self.model_file, map_location='cpu') tokenizer_file = DIRECTORIES[ 'ml_hate_speech_path'] + '/' + self.bert_model + '/vocab.txt' config_file = DIRECTORIES[ 'ml_hate_speech_path'] + '/' + self.bert_model + '/bert_config.json' bert_model_file = DIRECTORIES[ 'ml_hate_speech_path'] + '/' + self.bert_model + '/' #+'-pytorch_model.bin' self.tokenizer = BertTokenizer.from_pretrained(tokenizer_file) config = BertConfig.from_json_file(config_file) self.model = BertForSequenceClassification.from_pretrained( bert_model_file, state_dict=model_state_dict, num_labels=2) # self.model = BertForSequenceClassification.from_pretrained(self.bert_model, state_dict=model_state_dict, # num_labels=2) self.model.to(self.device)
def evaluate_config(): # args = parser.parse_args() data_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/temp' bert_model_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/bert-base-chinese-pytorch' model_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/experiments/base_model' seed = 23 # help="random seed for initialization" restore_file = 'best' # help="name of the file in `model_dir` containing weights to load" multi_gpu = False # action='store_true', help="Whether to use multiple GPUs if available" fp16 = False # action='store_true', help="Whether to use 16-bit float precision instead of 32-bit" # Load the parameters from json file json_path = os.path.join(model_dir, 'params.json') assert os.path.isfile( json_path), "No json configuration file found at {}".format(json_path) params = Params(json_path) # Use GPUs if available params.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') params.n_gpu = torch.cuda.device_count() params.multi_gpu = multi_gpu # Set the random seed for reproducible experiments random.seed(seed) torch.manual_seed(seed) if params.n_gpu > 0: torch.cuda.manual_seed_all(seed) # set random seed for all GPUs params.seed = seed # Set the logger set_logger(os.path.join(model_dir, 'evaluate.log')) # Create the input data pipeline logging.info("Loading the dataset...") # Initialize the DataLoader data_loader = DataLoader(data_dir, bert_model_dir, params, token_pad_idx=0) logging.info("- done.") # Define the model config_path = os.path.join(bert_model_dir, 'bert_config.json') config = BertConfig.from_json_file(config_path) model = BertForTokenClassification(config, num_labels=len(params.tag2idx)) model.to(params.device) # Reload weights from the saved file load_checkpoint(os.path.join(model_dir, restore_file + '.pth.tar'), model) if fp16: model.half() if params.n_gpu > 1 and multi_gpu: model = torch.nn.DataParallel(model) return data_loader, model, params
def __init__(self, args, device, load_pretrained_bert=False, bert_config=None): super(Summarizer, self).__init__() self.args = args self.device = device self.bert = Bert(args, load_pretrained_bert=True, bert_config=BertConfig.from_json_file( args.bert_config_path)) self.bert_config = BertConfig.from_json_file(args.bert_config_path) if (args.encoder == 'classifier'): self.encoder = Classifier(self.bert_config.hidden_size) elif (args.encoder == 'transformer'): self.encoder = TransformerInterEncoder( self.bert_config.hidden_size, args.ff_size, args.heads, args.dropout, args.inter_layers) elif (args.encoder == 'rnn'): self.encoder = RNNEncoder(bidirectional=True, num_layers=1, input_size=self.bert_config.hidden_size, hidden_size=args.rnn_size, dropout=args.dropout) elif (args.encoder == 'baseline'): bert_config = BertConfig(self.bert_config.vocab_size, hidden_size=args.hidden_size, num_hidden_layers=6, num_attention_heads=8, intermediate_size=args.ff_size) self.bert.model = BertModel(bert_config) self.encoder = Classifier(self.bert_config.hidden_size) if args.param_init != 0.0: for p in self.encoder.parameters(): p.data.uniform_(-args.param_init, args.param_init) if args.param_init_glorot: for p in self.encoder.parameters(): if p.dim() > 1: xavier_uniform_(p) self.to(device)
def __init__(self, vocab_size=None, device='cpu', training=False): super().__init__() bert_vocab_size = 30522 config = BertConfig(bert_vocab_size, max_position_embeddings=512) self.bert = BertModel(config).from_pretrained('bert-base-cased').to( device) self.classifier = nn.Linear(768, vocab_size) self.device = device self.training = training self.bert.eval()
def load_model(): config = BertConfig.from_json_file(MODEL_CONFIG_PATH) model = BertForTokenClassification(config, num_labels=len(LABELS)) state_dict = torch.load(MODEL_PATH) model.load_state_dict(state_dict) model.eval() return ( model, BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True), )
def __init__(self, args, device, load_pretrained_bert=False, bert_config=None): super(Summarizer, self).__init__() self.args = args self.device = device self.bert = Bert(args.temp_dir, load_pretrained_bert, bert_config) if (args.freeze_initial > 0): for param in self.bert.model.encoder.layer[ 0:args.freeze_initial].parameters(): param.requires_grad = False print("*" * 80) print("*" * 80) print("Initial Layers of BERT is frozen, ie first ", args.freeze_initial, "Layers") print(self.bert.model.encoder.layer[0:args.freeze_initial]) print("*" * 80) print("*" * 80) if (args.encoder == 'classifier'): self.encoder = Classifier(self.bert.model.config.hidden_size) elif (args.encoder == 'multi_layer_classifier'): self.encoder = MultiLayerClassifier( self.bert.model.config.hidden_size, 32) elif (args.encoder == 'transformer'): self.encoder = TransformerInterEncoder( self.bert.model.config.hidden_size, args.ff_size, args.heads, args.dropout, args.inter_layers) elif (args.encoder == 'rnn'): self.encoder = RNNEncoder( bidirectional=True, num_layers=1, input_size=self.bert.model.config.hidden_size, hidden_size=args.rnn_size, dropout=args.dropout) elif (args.encoder == 'baseline'): bert_config = BertConfig(self.bert.model.config.vocab_size, hidden_size=args.hidden_size, num_hidden_layers=6, num_attention_heads=8, intermediate_size=args.ff_size) self.bert.model = BertModel(bert_config) self.encoder = Classifier(self.bert.model.config.hidden_size) if args.param_init != 0.0: for p in self.encoder.parameters(): p.data.uniform_(-args.param_init, args.param_init) if args.param_init_glorot: for p in self.encoder.parameters(): if p.dim() > 1: xavier_uniform_(p) self.to(device)
def load_BFTC_from_TF_ckpt(bert_config, ckpt_path, num_labels): """ Helper function for loading model - workaround to prevent error """ config = BertConfig.from_json_file(bert_config) model = BertForPreTraining(config) load_tf_weights_in_bert(model, ckpt_path) state_dict=model.state_dict() model = BertForTokenClassification(config, num_labels=num_labels) # Load from a PyTorch state_dict old_keys = [] new_keys = [] for key in state_dict.keys(): new_key = None if 'gamma' in key: new_key = key.replace('gamma', 'weight') if 'beta' in key: new_key = key.replace('beta', 'bias') if new_key: old_keys.append(key) new_keys.append(new_key) for old_key, new_key in zip(old_keys, new_keys): state_dict[new_key] = state_dict.pop(old_key) missing_keys = [] unexpected_keys = [] error_msgs = [] # copy state_dict so _load_from_state_dict can modify it metadata = getattr(state_dict, '_metadata', None) state_dict = state_dict.copy() if metadata is not None: state_dict._metadata = metadata def load(module, prefix=''): local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) module._load_from_state_dict( state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs) for name, child in module._modules.items(): if child is not None: load(child, prefix + name + '.') start_prefix = '' if not hasattr(model, 'bert') and any(s.startswith('bert.') for s in state_dict.keys()): start_prefix = 'bert.' load(model, prefix=start_prefix) if len(missing_keys) > 0: print("Weights of {} not initialized from pretrained model: {}".format( model.__class__.__name__, missing_keys)) if len(unexpected_keys) > 0: print("Weights from pretrained model not used in {}: {}".format( model.__class__.__name__, unexpected_keys)) if len(error_msgs) > 0: raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( model.__class__.__name__, "\n\t".join(error_msgs))) return model
def __init__(self, data_dir, model_name, epochs=4, batch_size=64, base_batch_size=32, part=1., seed=1234, debug_mode=False): self.device = torch.device('cuda') self.data_dir = data_dir self.debug_mode = debug_mode self.model_name = model_name self.seed = seed self.identity_list = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish', 'muslim', 'black', 'white', 'psychiatric_or_mental_illness'] self.toxicity_type_list = ['severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat'] if part == 1.: self.weight_dict = {"severe_toxicity": 1000, "obscene": 235, "identity_attack": 236, "insult": 22, "threat": 646, "male": 45, "female": 35, "homosexual_gay_or_lesbian": 176, "christian": 50, "jewish": 249, "muslim": 91, "black": 130, "white": 75, "psychiatric_or_mental_illness": 442, "pp": 101, "np": 13, "pn": 20, "nn": 1, "pp_male": 431, "np_male": 50, "pn_male": 17, "nn_male": 1, "pp_female": 384, "np_female": 39, "pn_female": 17, "nn_female": 1, "pp_homosexual_gay_or_lesbian": 900, "np_homosexual_gay_or_lesbian": 219, "pn_homosexual_gay_or_lesbian": 17, "nn_homosexual_gay_or_lesbian": 1, "pp_christian": 859, "np_christian": 54, "pn_christian": 17, "nn_christian": 1, "pp_jewish": 2365, "np_jewish": 278, "pn_jewish": 17, "nn_jewish": 1, "pp_muslim": 606, "np_muslim": 108, "pn_muslim": 17, "nn_muslim": 1, "pp_black": 586, "np_black": 167, "pn_black": 17, "nn_black": 1, "pp_white": 387, "np_white": 94, "pn_white": 17, "nn_white": 1, "pp_psychiatric_or_mental_illness": 2874, "np_psychiatric_or_mental_illness": 523, "pn_psychiatric_or_mental_illness": 17, "nn_psychiatric_or_mental_illness": 1} else: self.weight_dict = {"severe_toxicity": 1000, "obscene": 196, "identity_attack": 278, "insult": 22, "threat": 609, "male": 45, "female": 33, "homosexual_gay_or_lesbian": 198, "christian": 48, "jewish": 243, "muslim": 133, "black": 131, "white": 90, "psychiatric_or_mental_illness": 369, "pp": 107, "np": 13, "pn": 19, "nn": 1, "pp_male": 434, "np_male": 51, "pn_male": 17, "nn_male": 1, "pp_female": 324, "np_female": 37, "pn_female": 17, "nn_female": 1, "pp_homosexual_gay_or_lesbian": 1055, "np_homosexual_gay_or_lesbian": 244, "pn_homosexual_gay_or_lesbian": 17, "nn_homosexual_gay_or_lesbian": 1, "pp_christian": 986, "np_christian": 50, "pn_christian": 17, "nn_christian": 1, "pp_jewish": 2680, "np_jewish": 268, "pn_jewish": 16, "nn_jewish": 1, "pp_muslim": 772, "np_muslim": 161, "pn_muslim": 17, "nn_muslim": 1, "pp_black": 633, "np_black": 165, "pn_black": 17, "nn_black": 1, "pp_white": 465, "np_white": 111, "pn_white": 17, "nn_white": 1, "pp_psychiatric_or_mental_illness": 2748, "np_psychiatric_or_mental_illness": 427, "pn_psychiatric_or_mental_illness": 16, "nn_psychiatric_or_mental_illness": 1} self.stopwords = '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n“”’\'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—' self.seed_everything() self.max_len = 220 self.epochs = epochs self.base_batch_size = base_batch_size self.batch_size = batch_size self.split_ratio = 0.95 self.sample_num = 1804874 if not self.debug_mode: self.train_df = pd.read_csv(os.path.join(self.data_dir, "train.csv")).sample(int(self.sample_num * part), random_state=1234).fillna(0.) self.test_df = pd.read_csv(os.path.join(self.data_dir, "test.csv")) else: self.train_df = pd.read_csv(os.path.join(self.data_dir, "train.csv")).head(1000).fillna(0.) self.test_df = pd.read_csv(os.path.join(self.data_dir, "test.csv")).head(1000) self.train_len = int(len(self.train_df) * self.split_ratio) self.evaluator = self.init_evaluator() self.bert_config = BertConfig(os.path.join(self.data_dir, "uncased_L-12_H-768_A-12/bert_config.json")) self.bert_model_path = os.path.join(self.data_dir, "uncased_L-12_H-768_A-12/")
def __init__(self, config, num_classes, vocab=None): super(KobertCRF, self).__init__() self.bert_config = con.BERT_CONFIG self.bert_config['output_attentions'] = True self.bert = BertModel(config=BertConfig.from_dict(self.bert_config)) self.vocab = vocab self.dropout = nn.Dropout(config["dropout"]) self.position_wise_ff = nn.Linear(config["hidden_size"], num_classes) self.crf = CRF(num_tags=num_classes, batch_first=True) with open(con.NER_UTIL_PATH["token_to_index"], 'rb') as f: self.token_to_index = pickle.load(f)
def _convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): # adapated from https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/pytorch_pretrained_bert/convert_tf_checkpoint_to_pytorch.py#L30 # Initialise PyTorch model config = BertConfig.from_json_file(bert_config_file) model = BertForPreTraining(config) # Load weights from tf checkpoint _load_tf_weights_in_bert(model, tf_checkpoint_path) # Save pytorch-model torch.save(model.state_dict(), pytorch_dump_path)
def test_convert_onnx(self): model = BertModel(BertConfig.from_json_file(BERT_CONFIG_PATH)) model.train(False) output = torch.onnx.export( model, self.org_dummy_input, self.model_onnx_path, verbose=True, operator_export_type=OPERATOR_EXPORT_TYPE, input_names=['input_ids', 'token_type_ids', 'attention_mask']) print("Export of torch_model.onnx complete!")
def load_biobert_model(biobert_pth, device): """Read saved state dict for biobert model on disk Args: biobert_pth: str, folder path where model state dictionary and config file are saved """ bert_config_file = os.path.join(biobert_pth, 'bert_config.json') config = BertConfig.from_json_file(bert_config_file) print("Building PyTorch model from configuration: {}".format(str(config))) model = BertForPreTraining(config) model.load_state_dict(torch.load(os.path.join(biobert_pth, 'biobert_statedict.pkl'), map_location=device)) return model
def __init__(self, config, num_classes, vocab=None) -> None: super(KobertCRF, self).__init__() if vocab is None: # pretraining model 사용 self.bert, self.vocab = get_pytorch_kobert_model() else: # finetuning model 사용 self.bert = BertModel(config=BertConfig.from_dict(bert_config)) self.vocab = vocab self.dropout = nn.Dropout(config.dropout) self.position_wise_ff = nn.Linear(config.hidden_size, num_classes) self.crf = CRF(num_tags=num_classes, batch_first=True)
def test_config_to_json_file(self): config_first = BertConfig(vocab_size_or_config_json_file=99, hidden_size=37) json_file_path = "/tmp/config.json" config_first.to_json_file(json_file_path) config_second = BertConfig.from_json_file(json_file_path) os.remove(json_file_path) self.assertEqual(config_second.to_dict(), config_first.to_dict())
def __init__(self, weight_path, num_labels=2, vocab="base-cased"): super(BertClassification, self).__init__() self.num_labels = num_labels self.vocab = vocab if self.vocab == "base-cased": self.bert = BertModel.from_pretrained(weight_path) self.config = BertConfig(vocab_size_or_config_json_file=28996, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) elif self.vocab == "base-uncased": self.bert = BertModel.from_pretrained(weight_path) self.config = BertConfig(vocab_size_or_config_json_file=30522, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) elif self.vocab == "finance-cased": self.bert = BertModel.from_pretrained(weight_path) self.config = BertConfig(vocab_size_or_config_json_file=28573, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) elif self.vocab == "finance-uncased": self.bert = BertModel.from_pretrained(weight_path) self.config = BertConfig(vocab_size_or_config_json_file=30873, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) self.dropout = nn.Dropout(self.config.hidden_dropout_prob) self.classifier = nn.Linear(self.config.hidden_size, num_labels) nn.init.xavier_normal(self.classifier.weight)
def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): # Initialise PyTorch model config = BertConfig.from_json_file(bert_config_file) print("Building PyTorch model from configuration: {}".format(str(config))) model = BertForPreTraining(config) # Load weights from tf checkpoint load_tf_weights_in_bert(model, tf_checkpoint_path) # Save pytorch-model print("Save PyTorch model to {}".format(pytorch_dump_path)) torch.save(model.state_dict(), pytorch_dump_path)
def __init__(self, config, num_classes, vocab=None) -> None: super(KobertCRFViz, self).__init__() # attention weight는 transformers 패키지에서만 지원됨 from transformers import BertModel, BertConfig # 모델 로딩 전에 True 값으로 설정해야함 bert_config['output_attentions'] = True self.bert = BertModel(config=BertConfig.from_dict(bert_config)) self.vocab = vocab self.dropout = nn.Dropout(config.dropout) self.position_wise_ff = nn.Linear(config.hidden_size, num_classes) self.crf = CRF(num_tags=num_classes, batch_first=True)
def get_kobert_model(ctx="cpu"): model_file = './kobert_model/pytorch_kobert_2439f391a6.params' vocab_file = './kobert_model/kobertvocab_f38b8a4d6d.json' bertmodel = BertModel(config=BertConfig.from_dict(bert_config)) bertmodel.load_state_dict(torch.load(model_file)) device = torch.device(ctx) bertmodel.to(device) bertmodel.eval() #print(vocab_file) #./kobertvocab_f38b8a4d6d.json vocab_b_obj = nlp.vocab.BERTVocab.from_json( open(vocab_file, 'rt').read()) #print(vocab_b_obj) return bertmodel, vocab_b_obj
def __init__(self, args, w=None, b=None): # 初始化模型,建立encoder和decoder super(Summarizer, self).__init__() if args.mode == "train": self.encoder = BertModel.from_pretrained('bert-base-cased', cache_dir="./temp") self.w = w self.b = b elif args.mode == "test": config = BertConfig.from_json_file(args.predict_config) self.encoder = BertModel(config) self.args = args # we choose same hiedden_size with bert embedding self.decoder = nn.GRU(input_size=768, hidden_size=768, num_layers=1)
def __init__(self, args, tagset_size, embedding_dim, hidden_dim, rnn_layers, dropout_ratio, dropout1, use_cuda=False): super(BERT_LSTM_CRF, self).__init__() self.embedding_dim = embedding_dim self.hidden_dim = hidden_dim self.word_embeds = BertModel(config=BertConfig.from_json_file(args.bert_config_json)) # print(self.word_embeds) self.word_embeds.load_state_dict(torch.load('./ckpts/9134_bert_weight.bin')) self.lstm = nn.LSTM(embedding_dim, hidden_dim, num_layers=rnn_layers, bidirectional=True, dropout=dropout_ratio, batch_first=True) self.rnn_layers = rnn_layers self.dropout1 = nn.Dropout(p=dropout1) self.crf = CRF(target_size=tagset_size, average_batch=True, use_cuda=use_cuda) self.liner = nn.Linear(hidden_dim*2, tagset_size+2) self.tagset_size = tagset_size
def __init__(self, config, num_classes, vocab=None) -> None: super(KobertCRF, self).__init__() self.name = self.__class__.__name__ if vocab is None: self.bert, self.vocab = get_kobert_model() else: self.bert = BertModel(config=BertConfig.from_dict(bert_config)) #self.bert = get_bert_multi_model() self.vocab = vocab self.dropout = nn.Dropout(config.dropout) self.position_wise_ff = nn.Linear(config.hidden_size, num_classes) self.crf = CRF(num_tags=num_classes, batch_first=True)
def __init__(self, config): super().__init__(config) self.encoder = config.encoder if config.train: bert = BertModel.from_pretrained('bert-base-chinese') else: bert = BertModel(BertConfig(21128)) bert.cuda() if self.encoder == 'bert': self.bert = bert else: for p in bert.parameters(): p.requires_grad = False BERTEncoder.bert = bert
def __init__(self, args, Y): super(Bert_seq_cls, self).__init__() print("loading pretrained bert from {}".format(args.bert_dir)) config_file = os.path.join(args.bert_dir, 'bert_config.json') self.config = BertConfig.from_json_file(config_file) print("Model config {}".format(self.config)) self.bert = BertModel.from_pretrained(args.bert_dir) self.dim_reduction = nn.Linear(self.config.hidden_size, args.num_filter_maps) self.dropout = nn.Dropout(self.config.hidden_dropout_prob) self.classifier = nn.Linear(args.num_filter_maps, Y) self.apply(self.init_bert_weights)
def __init__(self, vocab: Vocabulary, pretrained_archive_path: str, null_score_difference_threshold: float, n_best_size: int = 20, max_answer_length: int = 30) -> None: super().__init__(vocab) config = BertConfig( os.path.join(pretrained_archive_path, 'bert_config.json')) self.bert_qa_model = HuggingFaceBertQA(config) self._pretrained_archive_path = pretrained_archive_path self._null_score_difference_threshold = null_score_difference_threshold self._n_best_size = n_best_size self._max_answer_length = max_answer_length
def __init__(self, vocab, hidden_size, enc_num_layer): super(BertNoEmbed, self).__init__() self.encoder = BertModelNoEmbed( config=BertConfig(vocab_size_or_config_json_file=len(vocab), hidden_size=hidden_size, num_hidden_layers=enc_num_layer, num_attention_heads=8, intermediate_size=3072, type_vocab_size=1, hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1)) self.hidden_size = hidden_size self.adaptive_softmax = nn.AdaptiveLogSoftmaxWithLoss(hidden_size, len(vocab), cutoffs=[1000])