def get_kobert_model(model_file, vocab_file, ctx="cpu"):
bertmodel = BertModel(config=BertConfig.from_dict(bert_config))
bertmodel.load_state_dict(torch.load(model_file))
device = torch.device(ctx)
bertmodel.to(device)
bertmodel.eval()
vocab_b_obj = nlp.vocab.BERTVocab.from_json(
open(vocab_file, 'rt').read())
return bertmodel, vocab_b_obj
def load_model(model_type):
checkpoint = torch.load(f'checkpoints/{model_type}.pt', map_location='cpu')
config = BertConfig.from_json_file('models/config.json')
model = Summarizer(args=None,
device="cpu",
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
return model
def __init__(self,
model_name,
epochs=1,
batch_size=64,
base_batch_size=32,
part=1.,
half=2,
seed=1234,
debug_mode=False):
self.device = torch.device('cuda')
self.input_dir = "../input"
self.work_dir = "../working/"
self.debug_mode = debug_mode
self.model_name = model_name
self.half = half
self.seed = seed
self.identity_list = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian',
'jewish', 'muslim', 'black', 'white',
'psychiatric_or_mental_illness'
]
self.toxicity_type_list = [
'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat'
]
self.stopwords = '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n“”’\'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
self.seed_everything()
self.max_len = 220
self.epochs = epochs
self.base_batch_size = base_batch_size
self.batch_size = batch_size
self.split_ratio = 0.95
self.sample_num = 1804874
if not self.debug_mode:
self.train_df = pd.read_csv(
os.path.join(
"../input/jigsaw-unintended-bias-in-toxicity-classification/train.csv"
)).sample(int(self.sample_num * part),
random_state=1234).fillna(0.)
self.test_df = pd.read_csv(
os.path.join(
"../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv"
))
else:
self.train_df = pd.read_csv(
os.path.join(
"../input/jigsaw-unintended-bias-in-toxicity-classification/train.csv"
)).head(1000).fillna(0.)
self.test_df = pd.read_csv(
os.path.join(
"../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv"
)).head(1000)
self.train_len = int(len(self.train_df) * self.split_ratio)
self.test_len = len(self.test_df)
self.bert_config = BertConfig(
'../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
+ 'bert_config.json')
self.bert_model_path = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
def test():
# 配置文件
cf = Config('./config.yaml')
# 有GPU用GPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# 测试数据
test_data = NewsDataset("./data/cnews_final_test.txt", cf.max_seq_len)
test_dataloader = DataLoader(test_data,
batch_size=cf.batch_size,
shuffle=True)
# 模型
config = BertConfig("./output/pytorch_bert_config.json")
model = BertForSequenceClassification(config, num_labels=cf.num_labels)
model.load_state_dict(torch.load("./output/pytorch_model.bin"))
# 把模型放到指定设备
model.to(device)
# 让模型并行化运算
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
# 训练
start_time = time.time()
data_len = len(test_dataloader)
model.eval()
y_pred = np.array([])
y_test = np.array([])
# for step,batch in enumerate(tqdm(test_dataloader,"batch",total=len(test_dataloader))):
for step, batch in enumerate(test_dataloader):
label_id = batch['label_id'].squeeze(1).to(device)
word_ids = batch['word_ids'].to(device)
segment_ids = batch['segment_ids'].to(device)
word_mask = batch['word_mask'].to(device)
loss = model(word_ids, segment_ids, word_mask, label_id)
with torch.no_grad():
pred = get_model_labels(model, word_ids, segment_ids, word_mask)
y_pred = np.hstack((y_pred, pred))
y_test = np.hstack((y_test, label_id.to("cpu").numpy()))
# 评估
print("Precision, Recall and F1-Score...")
print(
metrics.classification_report(y_test,
y_pred,
target_names=get_labels('./data/label')))
# 混淆矩阵
print("Confusion Matrix...")
cm = metrics.confusion_matrix(y_test, y_pred)
print(cm)
def get_kobert_model(model_file, vocab_file, ctx="cpu"):
bertmodel = BertModel(config=BertConfig.from_dict(bert_config))
bertmodel.load_state_dict(torch.load(model_file))
device = torch.device(ctx)
bertmodel.to(device)
bertmodel.eval()
vocab_b_obj = nlp.vocab.BERTVocab.from_sentencepiece(vocab_file,
padding_token='[PAD]')
return bertmodel, vocab_b_obj
def __init__(self):
# configuration
self.ROOT_FOLDER = os.path.dirname(__file__)
print('ROOT_FOLDER', self.ROOT_FOLDER)
DIRECTORIES = {
'ml_hate_speech_path':
os.path.join(self.ROOT_FOLDER, 'models/ml_hate_speech_classifier')
}
# Load a trained model that you have fine-tuned
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
self.bert_model = 'bert-base-multilingual-uncased'
self.model_file = os.path.join(
DIRECTORIES['ml_hate_speech_path'],
'pytorch_model_epoch_20_seqlen_256.bin')
# self.model_file = 'pytorch_model_epoch_20_seqlen_256.bin'
print('model_file', self.model_file)
print('model_dir', os.listdir(os.path.join(self.ROOT_FOLDER,
'models')))
print('model_dir_s', os.listdir(DIRECTORIES['ml_hate_speech_path']))
print(os.path.isfile(self.model_file))
if not os.path.isfile(self.model_file):
print('Please Download the model ...')
exit(0)
#
# model_download_file = "sh " + self.ROOT_FOLDER+'/model_download.sh'
# os.system('ls /usr/src/app/project/')
# print('model_download_file ',model_download_file, os.path.isfile(model_download_file))
# os.system(model_download_file)
# print(self.model_file)
if torch.cuda.is_available():
model_state_dict = torch.load(self.model_file)
else:
print('Loading model ...', self.model_file)
model_state_dict = torch.load(self.model_file, map_location='cpu')
tokenizer_file = DIRECTORIES[
'ml_hate_speech_path'] + '/' + self.bert_model + '/vocab.txt'
config_file = DIRECTORIES[
'ml_hate_speech_path'] + '/' + self.bert_model + '/bert_config.json'
bert_model_file = DIRECTORIES[
'ml_hate_speech_path'] + '/' + self.bert_model + '/' #+'-pytorch_model.bin'
self.tokenizer = BertTokenizer.from_pretrained(tokenizer_file)
config = BertConfig.from_json_file(config_file)
self.model = BertForSequenceClassification.from_pretrained(
bert_model_file, state_dict=model_state_dict, num_labels=2)
# self.model = BertForSequenceClassification.from_pretrained(self.bert_model, state_dict=model_state_dict,
# num_labels=2)
self.model.to(self.device)
def evaluate_config():
# args = parser.parse_args()
data_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/temp'
bert_model_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/bert-base-chinese-pytorch'
model_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/experiments/base_model'
seed = 23 # help="random seed for initialization"
restore_file = 'best' # help="name of the file in `model_dir` containing weights to load"
multi_gpu = False # action='store_true', help="Whether to use multiple GPUs if available"
fp16 = False # action='store_true', help="Whether to use 16-bit float precision instead of 32-bit"
# Load the parameters from json file
json_path = os.path.join(model_dir, 'params.json')
assert os.path.isfile(
json_path), "No json configuration file found at {}".format(json_path)
params = Params(json_path)
# Use GPUs if available
params.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
params.n_gpu = torch.cuda.device_count()
params.multi_gpu = multi_gpu
# Set the random seed for reproducible experiments
random.seed(seed)
torch.manual_seed(seed)
if params.n_gpu > 0:
torch.cuda.manual_seed_all(seed) # set random seed for all GPUs
params.seed = seed
# Set the logger
set_logger(os.path.join(model_dir, 'evaluate.log'))
# Create the input data pipeline
logging.info("Loading the dataset...")
# Initialize the DataLoader
data_loader = DataLoader(data_dir, bert_model_dir, params, token_pad_idx=0)
logging.info("- done.")
# Define the model
config_path = os.path.join(bert_model_dir, 'bert_config.json')
config = BertConfig.from_json_file(config_path)
model = BertForTokenClassification(config, num_labels=len(params.tag2idx))
model.to(params.device)
# Reload weights from the saved file
load_checkpoint(os.path.join(model_dir, restore_file + '.pth.tar'), model)
if fp16:
model.half()
if params.n_gpu > 1 and multi_gpu:
model = torch.nn.DataParallel(model)
return data_loader, model, params
def __init__(self,
args,
device,
load_pretrained_bert=False,
bert_config=None):
super(Summarizer, self).__init__()
self.args = args
self.device = device
self.bert = Bert(args,
load_pretrained_bert=True,
bert_config=BertConfig.from_json_file(
args.bert_config_path))
self.bert_config = BertConfig.from_json_file(args.bert_config_path)
if (args.encoder == 'classifier'):
self.encoder = Classifier(self.bert_config.hidden_size)
elif (args.encoder == 'transformer'):
self.encoder = TransformerInterEncoder(
self.bert_config.hidden_size, args.ff_size, args.heads,
args.dropout, args.inter_layers)
elif (args.encoder == 'rnn'):
self.encoder = RNNEncoder(bidirectional=True,
num_layers=1,
input_size=self.bert_config.hidden_size,
hidden_size=args.rnn_size,
dropout=args.dropout)
elif (args.encoder == 'baseline'):
bert_config = BertConfig(self.bert_config.vocab_size,
hidden_size=args.hidden_size,
num_hidden_layers=6,
num_attention_heads=8,
intermediate_size=args.ff_size)
self.bert.model = BertModel(bert_config)
self.encoder = Classifier(self.bert_config.hidden_size)
if args.param_init != 0.0:
for p in self.encoder.parameters():
p.data.uniform_(-args.param_init, args.param_init)
if args.param_init_glorot:
for p in self.encoder.parameters():
if p.dim() > 1:
xavier_uniform_(p)
self.to(device)
def __init__(self, vocab_size=None, device='cpu', training=False):
super().__init__()
bert_vocab_size = 30522
config = BertConfig(bert_vocab_size, max_position_embeddings=512)
self.bert = BertModel(config).from_pretrained('bert-base-cased').to(
device)
self.classifier = nn.Linear(768, vocab_size)
self.device = device
self.training = training
self.bert.eval()
def load_model():
config = BertConfig.from_json_file(MODEL_CONFIG_PATH)
model = BertForTokenClassification(config, num_labels=len(LABELS))
state_dict = torch.load(MODEL_PATH)
model.load_state_dict(state_dict)
model.eval()
return (
model,
BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True),
)
def __init__(self,
args,
device,
load_pretrained_bert=False,
bert_config=None):
super(Summarizer, self).__init__()
self.args = args
self.device = device
self.bert = Bert(args.temp_dir, load_pretrained_bert, bert_config)
if (args.freeze_initial > 0):
for param in self.bert.model.encoder.layer[
0:args.freeze_initial].parameters():
param.requires_grad = False
print("*" * 80)
print("*" * 80)
print("Initial Layers of BERT is frozen, ie first ",
args.freeze_initial, "Layers")
print(self.bert.model.encoder.layer[0:args.freeze_initial])
print("*" * 80)
print("*" * 80)
if (args.encoder == 'classifier'):
self.encoder = Classifier(self.bert.model.config.hidden_size)
elif (args.encoder == 'multi_layer_classifier'):
self.encoder = MultiLayerClassifier(
self.bert.model.config.hidden_size, 32)
elif (args.encoder == 'transformer'):
self.encoder = TransformerInterEncoder(
self.bert.model.config.hidden_size, args.ff_size, args.heads,
args.dropout, args.inter_layers)
elif (args.encoder == 'rnn'):
self.encoder = RNNEncoder(
bidirectional=True,
num_layers=1,
input_size=self.bert.model.config.hidden_size,
hidden_size=args.rnn_size,
dropout=args.dropout)
elif (args.encoder == 'baseline'):
bert_config = BertConfig(self.bert.model.config.vocab_size,
hidden_size=args.hidden_size,
num_hidden_layers=6,
num_attention_heads=8,
intermediate_size=args.ff_size)
self.bert.model = BertModel(bert_config)
self.encoder = Classifier(self.bert.model.config.hidden_size)
if args.param_init != 0.0:
for p in self.encoder.parameters():
p.data.uniform_(-args.param_init, args.param_init)
if args.param_init_glorot:
for p in self.encoder.parameters():
if p.dim() > 1:
xavier_uniform_(p)
self.to(device)
def load_BFTC_from_TF_ckpt(bert_config, ckpt_path, num_labels):
"""
Helper function for loading model - workaround to prevent error
"""
config = BertConfig.from_json_file(bert_config)
model = BertForPreTraining(config)
load_tf_weights_in_bert(model, ckpt_path)
state_dict=model.state_dict()
model = BertForTokenClassification(config, num_labels=num_labels)
# Load from a PyTorch state_dict
old_keys = []
new_keys = []
for key in state_dict.keys():
new_key = None
if 'gamma' in key:
new_key = key.replace('gamma', 'weight')
if 'beta' in key:
new_key = key.replace('beta', 'bias')
if new_key:
old_keys.append(key)
new_keys.append(new_key)
for old_key, new_key in zip(old_keys, new_keys):
state_dict[new_key] = state_dict.pop(old_key)
missing_keys = []
unexpected_keys = []
error_msgs = []
# copy state_dict so _load_from_state_dict can modify it
metadata = getattr(state_dict, '_metadata', None)
state_dict = state_dict.copy()
if metadata is not None:
state_dict._metadata = metadata
def load(module, prefix=''):
local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
module._load_from_state_dict(
state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs)
for name, child in module._modules.items():
if child is not None:
load(child, prefix + name + '.')
start_prefix = ''
if not hasattr(model, 'bert') and any(s.startswith('bert.') for s in state_dict.keys()):
start_prefix = 'bert.'
load(model, prefix=start_prefix)
if len(missing_keys) > 0:
print("Weights of {} not initialized from pretrained model: {}".format(
model.__class__.__name__, missing_keys))
if len(unexpected_keys) > 0:
print("Weights from pretrained model not used in {}: {}".format(
model.__class__.__name__, unexpected_keys))
if len(error_msgs) > 0:
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
model.__class__.__name__, "\n\t".join(error_msgs)))
return model
def __init__(self, data_dir, model_name, epochs=4, batch_size=64, base_batch_size=32, part=1., seed=1234, debug_mode=False):
self.device = torch.device('cuda')
self.data_dir = data_dir
self.debug_mode = debug_mode
self.model_name = model_name
self.seed = seed
self.identity_list = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish', 'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
self.toxicity_type_list = ['severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
if part == 1.:
self.weight_dict = {"severe_toxicity": 1000, "obscene": 235, "identity_attack": 236, "insult": 22,
"threat": 646, "male": 45, "female": 35, "homosexual_gay_or_lesbian": 176, "christian": 50,
"jewish": 249, "muslim": 91, "black": 130, "white": 75, "psychiatric_or_mental_illness": 442,
"pp": 101, "np": 13, "pn": 20, "nn": 1,
"pp_male": 431, "np_male": 50, "pn_male": 17, "nn_male": 1,
"pp_female": 384, "np_female": 39, "pn_female": 17, "nn_female": 1,
"pp_homosexual_gay_or_lesbian": 900, "np_homosexual_gay_or_lesbian": 219, "pn_homosexual_gay_or_lesbian": 17, "nn_homosexual_gay_or_lesbian": 1,
"pp_christian": 859, "np_christian": 54, "pn_christian": 17, "nn_christian": 1,
"pp_jewish": 2365, "np_jewish": 278, "pn_jewish": 17, "nn_jewish": 1,
"pp_muslim": 606, "np_muslim": 108, "pn_muslim": 17, "nn_muslim": 1,
"pp_black": 586, "np_black": 167, "pn_black": 17, "nn_black": 1,
"pp_white": 387, "np_white": 94, "pn_white": 17, "nn_white": 1,
"pp_psychiatric_or_mental_illness": 2874, "np_psychiatric_or_mental_illness": 523, "pn_psychiatric_or_mental_illness": 17, "nn_psychiatric_or_mental_illness": 1}
else:
self.weight_dict = {"severe_toxicity": 1000, "obscene": 196, "identity_attack": 278, "insult": 22,
"threat": 609, "male": 45, "female": 33, "homosexual_gay_or_lesbian": 198, "christian": 48,
"jewish": 243, "muslim": 133, "black": 131, "white": 90, "psychiatric_or_mental_illness": 369,
"pp": 107, "np": 13, "pn": 19, "nn": 1,
"pp_male": 434, "np_male": 51, "pn_male": 17, "nn_male": 1,
"pp_female": 324, "np_female": 37, "pn_female": 17, "nn_female": 1,
"pp_homosexual_gay_or_lesbian": 1055, "np_homosexual_gay_or_lesbian": 244, "pn_homosexual_gay_or_lesbian": 17, "nn_homosexual_gay_or_lesbian": 1,
"pp_christian": 986, "np_christian": 50, "pn_christian": 17, "nn_christian": 1,
"pp_jewish": 2680, "np_jewish": 268, "pn_jewish": 16, "nn_jewish": 1,
"pp_muslim": 772, "np_muslim": 161, "pn_muslim": 17, "nn_muslim": 1,
"pp_black": 633, "np_black": 165, "pn_black": 17, "nn_black": 1,
"pp_white": 465, "np_white": 111, "pn_white": 17, "nn_white": 1,
"pp_psychiatric_or_mental_illness": 2748, "np_psychiatric_or_mental_illness": 427, "pn_psychiatric_or_mental_illness": 16, "nn_psychiatric_or_mental_illness": 1}
self.stopwords = '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n“”’\'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
self.seed_everything()
self.max_len = 220
self.epochs = epochs
self.base_batch_size = base_batch_size
self.batch_size = batch_size
self.split_ratio = 0.95
self.sample_num = 1804874
if not self.debug_mode:
self.train_df = pd.read_csv(os.path.join(self.data_dir, "train.csv")).sample(int(self.sample_num * part), random_state=1234).fillna(0.)
self.test_df = pd.read_csv(os.path.join(self.data_dir, "test.csv"))
else:
self.train_df = pd.read_csv(os.path.join(self.data_dir, "train.csv")).head(1000).fillna(0.)
self.test_df = pd.read_csv(os.path.join(self.data_dir, "test.csv")).head(1000)
self.train_len = int(len(self.train_df) * self.split_ratio)
self.evaluator = self.init_evaluator()
self.bert_config = BertConfig(os.path.join(self.data_dir, "uncased_L-12_H-768_A-12/bert_config.json"))
self.bert_model_path = os.path.join(self.data_dir, "uncased_L-12_H-768_A-12/")
def __init__(self, config, num_classes, vocab=None):
super(KobertCRF, self).__init__()
self.bert_config = con.BERT_CONFIG
self.bert_config['output_attentions'] = True
self.bert = BertModel(config=BertConfig.from_dict(self.bert_config))
self.vocab = vocab
self.dropout = nn.Dropout(config["dropout"])
self.position_wise_ff = nn.Linear(config["hidden_size"], num_classes)
self.crf = CRF(num_tags=num_classes, batch_first=True)
with open(con.NER_UTIL_PATH["token_to_index"], 'rb') as f:
self.token_to_index = pickle.load(f)
def _convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path):
# adapated from https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/pytorch_pretrained_bert/convert_tf_checkpoint_to_pytorch.py#L30
# Initialise PyTorch model
config = BertConfig.from_json_file(bert_config_file)
model = BertForPreTraining(config)
# Load weights from tf checkpoint
_load_tf_weights_in_bert(model, tf_checkpoint_path)
# Save pytorch-model
torch.save(model.state_dict(), pytorch_dump_path)
def test_convert_onnx(self):
model = BertModel(BertConfig.from_json_file(BERT_CONFIG_PATH))
model.train(False)
output = torch.onnx.export(
model,
self.org_dummy_input,
self.model_onnx_path,
verbose=True,
operator_export_type=OPERATOR_EXPORT_TYPE,
input_names=['input_ids', 'token_type_ids', 'attention_mask'])
print("Export of torch_model.onnx complete!")
def load_biobert_model(biobert_pth, device):
"""Read saved state dict for biobert model on disk
Args:
biobert_pth: str, folder path where model state dictionary and config file are saved
"""
bert_config_file = os.path.join(biobert_pth, 'bert_config.json')
config = BertConfig.from_json_file(bert_config_file)
print("Building PyTorch model from configuration: {}".format(str(config)))
model = BertForPreTraining(config)
model.load_state_dict(torch.load(os.path.join(biobert_pth, 'biobert_statedict.pkl'), map_location=device))
return model
def __init__(self, config, num_classes, vocab=None) -> None:
super(KobertCRF, self).__init__()
if vocab is None: # pretraining model 사용
self.bert, self.vocab = get_pytorch_kobert_model()
else: # finetuning model 사용
self.bert = BertModel(config=BertConfig.from_dict(bert_config))
self.vocab = vocab
self.dropout = nn.Dropout(config.dropout)
self.position_wise_ff = nn.Linear(config.hidden_size, num_classes)
self.crf = CRF(num_tags=num_classes, batch_first=True)
def test_config_to_json_file(self):
config_first = BertConfig(vocab_size_or_config_json_file=99, hidden_size=37)
json_file_path = "/tmp/config.json"
config_first.to_json_file(json_file_path)
config_second = BertConfig.from_json_file(json_file_path)
os.remove(json_file_path)
self.assertEqual(config_second.to_dict(), config_first.to_dict())
def __init__(self, weight_path, num_labels=2, vocab="base-cased"):
super(BertClassification, self).__init__()
self.num_labels = num_labels
self.vocab = vocab
if self.vocab == "base-cased":
self.bert = BertModel.from_pretrained(weight_path)
self.config = BertConfig(vocab_size_or_config_json_file=28996,
hidden_size=768,
num_hidden_layers=12,
num_attention_heads=12,
intermediate_size=3072)
elif self.vocab == "base-uncased":
self.bert = BertModel.from_pretrained(weight_path)
self.config = BertConfig(vocab_size_or_config_json_file=30522,
hidden_size=768,
num_hidden_layers=12,
num_attention_heads=12,
intermediate_size=3072)
elif self.vocab == "finance-cased":
self.bert = BertModel.from_pretrained(weight_path)
self.config = BertConfig(vocab_size_or_config_json_file=28573,
hidden_size=768,
num_hidden_layers=12,
num_attention_heads=12,
intermediate_size=3072)
elif self.vocab == "finance-uncased":
self.bert = BertModel.from_pretrained(weight_path)
self.config = BertConfig(vocab_size_or_config_json_file=30873,
hidden_size=768,
num_hidden_layers=12,
num_attention_heads=12,
intermediate_size=3072)
self.dropout = nn.Dropout(self.config.hidden_dropout_prob)
self.classifier = nn.Linear(self.config.hidden_size, num_labels)
nn.init.xavier_normal(self.classifier.weight)
def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file,
pytorch_dump_path):
# Initialise PyTorch model
config = BertConfig.from_json_file(bert_config_file)
print("Building PyTorch model from configuration: {}".format(str(config)))
model = BertForPreTraining(config)
# Load weights from tf checkpoint
load_tf_weights_in_bert(model, tf_checkpoint_path)
# Save pytorch-model
print("Save PyTorch model to {}".format(pytorch_dump_path))
torch.save(model.state_dict(), pytorch_dump_path)
def __init__(self, config, num_classes, vocab=None) -> None:
super(KobertCRFViz, self).__init__()
# attention weight는 transformers 패키지에서만 지원됨
from transformers import BertModel, BertConfig
# 모델 로딩 전에 True 값으로 설정해야함
bert_config['output_attentions'] = True
self.bert = BertModel(config=BertConfig.from_dict(bert_config))
self.vocab = vocab
self.dropout = nn.Dropout(config.dropout)
self.position_wise_ff = nn.Linear(config.hidden_size, num_classes)
self.crf = CRF(num_tags=num_classes, batch_first=True)
def get_kobert_model(ctx="cpu"):
model_file = './kobert_model/pytorch_kobert_2439f391a6.params'
vocab_file = './kobert_model/kobertvocab_f38b8a4d6d.json'
bertmodel = BertModel(config=BertConfig.from_dict(bert_config))
bertmodel.load_state_dict(torch.load(model_file))
device = torch.device(ctx)
bertmodel.to(device)
bertmodel.eval()
#print(vocab_file) #./kobertvocab_f38b8a4d6d.json
vocab_b_obj = nlp.vocab.BERTVocab.from_json(
open(vocab_file, 'rt').read())
#print(vocab_b_obj)
return bertmodel, vocab_b_obj
def __init__(self, args, w=None, b=None):
# 初始化模型，建立encoder和decoder
super(Summarizer, self).__init__()
if args.mode == "train":
self.encoder = BertModel.from_pretrained('bert-base-cased',
cache_dir="./temp")
self.w = w
self.b = b
elif args.mode == "test":
config = BertConfig.from_json_file(args.predict_config)
self.encoder = BertModel(config)
self.args = args
# we choose same hiedden_size with bert embedding
self.decoder = nn.GRU(input_size=768, hidden_size=768, num_layers=1)
def __init__(self, args, tagset_size, embedding_dim, hidden_dim, rnn_layers, dropout_ratio, dropout1, use_cuda=False):
super(BERT_LSTM_CRF, self).__init__()
self.embedding_dim = embedding_dim
self.hidden_dim = hidden_dim
self.word_embeds = BertModel(config=BertConfig.from_json_file(args.bert_config_json))
# print(self.word_embeds)
self.word_embeds.load_state_dict(torch.load('./ckpts/9134_bert_weight.bin'))
self.lstm = nn.LSTM(embedding_dim, hidden_dim,
num_layers=rnn_layers, bidirectional=True, dropout=dropout_ratio, batch_first=True)
self.rnn_layers = rnn_layers
self.dropout1 = nn.Dropout(p=dropout1)
self.crf = CRF(target_size=tagset_size, average_batch=True, use_cuda=use_cuda)
self.liner = nn.Linear(hidden_dim*2, tagset_size+2)
self.tagset_size = tagset_size
def __init__(self, config, num_classes, vocab=None) -> None:
super(KobertCRF, self).__init__()
self.name = self.__class__.__name__
if vocab is None:
self.bert, self.vocab = get_kobert_model()
else:
self.bert = BertModel(config=BertConfig.from_dict(bert_config))
#self.bert = get_bert_multi_model()
self.vocab = vocab
self.dropout = nn.Dropout(config.dropout)
self.position_wise_ff = nn.Linear(config.hidden_size, num_classes)
self.crf = CRF(num_tags=num_classes, batch_first=True)
def __init__(self, config):
super().__init__(config)
self.encoder = config.encoder
if config.train:
bert = BertModel.from_pretrained('bert-base-chinese')
else:
bert = BertModel(BertConfig(21128))
bert.cuda()
if self.encoder == 'bert':
self.bert = bert
else:
for p in bert.parameters():
p.requires_grad = False
BERTEncoder.bert = bert
def __init__(self, args, Y):
super(Bert_seq_cls, self).__init__()
print("loading pretrained bert from {}".format(args.bert_dir))
config_file = os.path.join(args.bert_dir, 'bert_config.json')
self.config = BertConfig.from_json_file(config_file)
print("Model config {}".format(self.config))
self.bert = BertModel.from_pretrained(args.bert_dir)
self.dim_reduction = nn.Linear(self.config.hidden_size,
args.num_filter_maps)
self.dropout = nn.Dropout(self.config.hidden_dropout_prob)
self.classifier = nn.Linear(args.num_filter_maps, Y)
self.apply(self.init_bert_weights)
def __init__(self,
vocab: Vocabulary,
pretrained_archive_path: str,
null_score_difference_threshold: float,
n_best_size: int = 20,
max_answer_length: int = 30) -> None:
super().__init__(vocab)
config = BertConfig(
os.path.join(pretrained_archive_path, 'bert_config.json'))
self.bert_qa_model = HuggingFaceBertQA(config)
self._pretrained_archive_path = pretrained_archive_path
self._null_score_difference_threshold = null_score_difference_threshold
self._n_best_size = n_best_size
self._max_answer_length = max_answer_length
def __init__(self, vocab, hidden_size, enc_num_layer):
super(BertNoEmbed, self).__init__()
self.encoder = BertModelNoEmbed(
config=BertConfig(vocab_size_or_config_json_file=len(vocab),
hidden_size=hidden_size,
num_hidden_layers=enc_num_layer,
num_attention_heads=8,
intermediate_size=3072,
type_vocab_size=1,
hidden_dropout_prob=0.1,
attention_probs_dropout_prob=0.1))
self.hidden_size = hidden_size
self.adaptive_softmax = nn.AdaptiveLogSoftmaxWithLoss(hidden_size,
len(vocab),
cutoffs=[1000])
