def test(args, device_id, pt):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if pt != '':
test_from = pt
else:
test_from = args.best_model
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if k in model_flags:
setattr(args, k, opt[k])
# print(args)
config = BertConfig.from_json_file(args.bert_config_path)
model = Summarizer(args,
device,
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
model.eval()
logger.info("Test dataset......")
test_dataset = torch.load(args.bert_data_path + 'test.data')
trainer = build_trainer(args, device_id, model, None)
trainer.test(model, test_dataset, device)
logger.info("Valid dataset......")
test_dataset = torch.load(args.bert_data_path + 'valid.data')
trainer = build_trainer(args, device_id, model, None)
trainer.test(model, test_dataset, device)
def validate(args, device_id, pt, epoch):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
config = BertConfig.from_json_file(args.bert_config_name)
model = Summarizer(args,
device,
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
model.eval()
valid_dataset = torch.load(args.bert_data_path + 'valid.data')
trainer = build_trainer(args, device_id, model, None)
stats = trainer.validate(valid_dataset, epoch)
return stats.xent()
def __init__(self, args):
super(BertBiLstmCrf, self).__init__(args)
self.args = args
self.vector_path = args.vector_path # 预训练词向量的路径 txt
self.embedding_dim = args.embedding_dim
self.hidden_dim = args.hidden_dim # 隐藏层
self.tag_num = args.tag_num
self.batch_size = args.batch_size
self.bidirectional = True # BiLstm
self.num_layers = args.num_layers
self.pad_index = args.pad_index
self.dropout = args.dropout # 训练时网络中连接 dropout 的概率
self.save_path = args.save_path
embedding_dimension = args.bert_embedding_dim
self.embedding = BertModel(
config=BertConfig.from_json_file(args.bert_config_json)).to(DEVICE)
self.embedding.load_state_dict(torch.load(args.bert_weight))
self.drop = nn.Dropout(0.5)
self.lstm = nn.LSTM(embedding_dimension,
self.hidden_dim,
bidirectional=self.bidirectional,
num_layers=self.num_layers,
dropout=self.dropout).to(DEVICE)
# hidden 除以 2 是因为，双向lstm输出的时候会翻倍，不除以二改成下面的 linear层中 hidden*2 也行
self.linear1 = nn.Linear(self.hidden_dim * 2,
self.hidden_dim).to(DEVICE)
self.lin_drop = nn.Dropout(0.5)
self.linear2 = nn.Linear(self.hidden_dim, self.tag_num + 2).to(
DEVICE) # 隐藏层到 label 的线性转换，即维度变换。
self.crf_layer = CRF(self.tag_num).to(DEVICE)
def convert_tf_checkpoint_to_pytorch():
# gave error originally. Solution found at: https://github.com/tensorflow/models/issues/2676
tf_path = 'weights/biobert_v1.1_pubmed/model.ckpt-1000000'
init_vars = tf.train.list_variables(tf_path)
excluded = ['BERTAdam', '_power', 'global_step']
init_vars = list(filter(lambda x: all([True if e not in x[0] else False for e in excluded]), init_vars))
print(init_vars)
names = []
arrays = []
for name, shape in init_vars:
print("Loading TF weights {} with shape {}".format(name,shape))
array = tf.train.load_variable(tf_path,name)
names.append(name)
arrays.append(array)
config = BertConfig.from_json_file('weights/biobert_v1.1_pubmed/bert_config.json')
print('Building Pytorch model from configuration {}'.format(str(config)))
model = BertForPreTraining(config)
for name, array in zip(names,arrays):
name = name.split('/')
# adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v
# which are not required for using pretrained model
if any(n in ["adam_v", "adam_m", "global_step"] for n in name):
print("Skipping {}".format("/".join(name)))
continue
pointer = model
for m_name in name:
if re.fullmatch(r'[A-Za-z]+_\d+', m_name):
l = re.split(r'_(\d+)', m_name)
else:
l = [m_name]
if l[0] == 'kernel' or l[0] == 'gamma':
pointer = getattr(pointer, 'weight')
elif l[0] == 'output_bias' or l[0] == 'beta':
pointer = getattr(pointer, 'bias')
elif l[0] == 'output_weights':
pointer = getattr(pointer, 'weight')
else:
pointer = getattr(pointer, l[0])
if len(l) >= 2:
num = int(l[1])
pointer = pointer[num]
if m_name[-11:] == '_embeddings':
pointer = getattr(pointer, 'weight')
elif m_name == 'kernel':
array = np.transpose(array)
try:
assert pointer.shape == array.shape
except AssertionError as e:
e.args += (pointer.shape, array.shape)
raise
print("Initialize PyTorch weight {}".format(name))
pointer.data = torch.from_numpy(array)
# Save pytorch-model
print("Save PyTorch model to {}".format('weights/'))
torch.save(model.state_dict(), 'weights/pytorch_weight')
def validate(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == "-1" else "cuda"
if (pt != ""):
test_from = pt
else:
test_from = args.test_from
logger.info("Loading checkpoint from %s" % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint["opt"])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
config = BertConfig.from_json_file(args.bert_config_path)
model = Summarizer(args,
device,
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
model.eval()
valid_iter = data_loader.Dataloader(args,
load_dataset(args,
"valid",
shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=False)
trainer = build_trainer(args, device_id, model, None)
stats = trainer.validate(valid_iter, step)
return stats.xent()
def __init__(self, args):
# 初始化模型，建立encoder和decoder
super(Summarizer, self).__init__()
if args.mode == "train":
self.encoder = BertModel.from_pretrained('bert-base-cased',
cache_dir="./temp")
elif args.mode == "test":
config = BertConfig.from_json_file(args.predict_config)
self.encoder = BertModel(config)
self.args = args
# we choose same hiedden_size with bert embedding
self.decoder = nn.GRU(input_size=768, hidden_size=768, num_layers=1)
# 初始化参数
self.vocab = torch.load(args.vocab_file)
self.vocab_size = len(self.vocab)
self.hidden_size = 768
self.w = torch.randn((self.vocab_size, self.hidden_size),
requires_grad=True)
self.b = torch.randn((self.vocab_size), requires_grad=True)
# self.loss_func = nn.SoftMarginLoss()
self.optimizer = optim.Adagrad([self.w, self.b], lr=0.01)
self.loss_func = nn.BCELoss()
self.loss = 0
self.device = args.device
def summarize(args, cp, data):
# load path to model checkpoint
if cp != '':
checkpoint = cp
else:
checkpoint = args.checkpoint
pt = torch.load(checkpoint, map_location=lambda storage, loc: storage)
# configure pre-trained bert
config = BertConfig.from_json_file(args.bert_path)
# configure the device id
device = "cpu" if args.visible_gpus == "-1" else "cuda"
# load the model using checkpoint
model = Summarizer(args, device, load_pretrained_bert=False, bert_config=config)
model.load_cp(pt)
model.eval()
num_sentence = args.num_sentence
# build the trainer model
bertsum = Trainer(args, device, model)
# run inference on trainer model
bertsum.infer(data, num_sentence)
def test(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
config = BertConfig.from_json_file(args.bert_config_path)
model = Summarizer(args,
device,
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
model.eval()
test_iter = data_loader.Dataloader(args,
load_dataset(args,
'test',
shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=True)
trainer = build_trainer(args, device_id, model, None)
trainer.test(test_iter, step)
def test_config_to_json_file(self):
config_first = BertConfig(vocab_size_or_config_json_file=99, hidden_size=37)
json_file_path = "/tmp/config.json"
config_first.to_json_file(json_file_path)
config_second = BertConfig.from_json_file(json_file_path)
os.remove(json_file_path)
self.assertEqual(config_second.to_dict(), config_first.to_dict())
def main():
DATA_PATH = Path('/home/ky/text_classification/data/')
PATH = Path('/home/ky/text_classification/data/tmp')
CLAS_DATA_PATH = PATH / 'class'
BERT_PRETRAINED_PATH = Path(
'/home/ky/text_classification/chinese_L-12_H-768_A-12/')
PYTORCH_PRETRAINED_BERT_CACHE = BERT_PRETRAINED_PATH / 'cache/'
PYTORCH_PRETRAINED_BERT_CACHE.mkdir(exist_ok=True)
args = {
"train_size": -1,
"val_size": -1,
"full_data_dir": DATA_PATH,
"data_dir": PATH,
"task_name": "toxic_multilabel",
"bert_model": BERT_PRETRAINED_PATH,
"output_dir": CLAS_DATA_PATH / 'output',
"max_seq_length": 512,
"do_train": True,
"do_eval": True,
"do_lower_case": True,
"train_batch_size": 32,
"eval_batch_size": 32,
"learning_rate": 3e-5,
"num_train_epochs": 4.0,
"warmup_proportion": 0.1,
"no_cuda": True,
"local_rank": -1,
"seed": 42,
"gradient_accumulation_steps": 1,
"optimize_on_cpu": False,
"fp16": False,
"loss_scale": 128
}
config_path = '/home/ky/text_classification/BLUC_MODEL/trained_model/config.json'
model_path = '/home/ky/text_classification/BLUC_MODEL/trained_model/pytorch_model.bin'
tag_path = '/home/ky/text_classification/data/tmp/labels.json'
dataset_path = '/home/ky/text_classification/data/dev-20000.tsv'
bert_pretrained_path = '/data/disk1/SegPOSdata/Bert_Pretrained/bert-base-chinese'
config_a = BertConfig.from_json_file(config_path)
device = torch.device('cuda:1')
print(device)
tokenizer = BertTokenizer.from_pretrained(bert_pretrained_path)
dataset = TsvDataset(dataset_path, tokenizer, chunkSize=2**12)
tagDict = TagDictionary.load(tag_path)
model = BertForMultiLabelSequenceClassification.load(
model_path, config_a, 627)
predictor = Predictor(model, tagDict, device, maxInputLen=512)
acc = predictor.validate(dataset, args['eval_batch_size'])
print(len(dataset))
print(acc)
def __init__(self):
# configuration
self.ROOT_FOLDER = os.path.dirname(__file__)
print('ROOT_FOLDER', self.ROOT_FOLDER)
DIRECTORIES = {
'ml_hate_speech_path':
os.path.join(self.ROOT_FOLDER, 'models/ml_hate_speech_classifier')
}
# Load a trained model that you have fine-tuned
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
self.bert_model = 'bert-base-multilingual-uncased'
self.model_file = os.path.join(
DIRECTORIES['ml_hate_speech_path'],
'pytorch_model_epoch_20_seqlen_256.bin')
# self.model_file = 'pytorch_model_epoch_20_seqlen_256.bin'
print('model_file', self.model_file)
print('model_dir', os.listdir(os.path.join(self.ROOT_FOLDER,
'models')))
print('model_dir_s', os.listdir(DIRECTORIES['ml_hate_speech_path']))
print(os.path.isfile(self.model_file))
if not os.path.isfile(self.model_file):
print('Please Download the model ...')
exit(0)
#
# model_download_file = "sh " + self.ROOT_FOLDER+'/model_download.sh'
# os.system('ls /usr/src/app/project/')
# print('model_download_file ',model_download_file, os.path.isfile(model_download_file))
# os.system(model_download_file)
# print(self.model_file)
if torch.cuda.is_available():
model_state_dict = torch.load(self.model_file)
else:
print('Loading model ...', self.model_file)
model_state_dict = torch.load(self.model_file, map_location='cpu')
tokenizer_file = DIRECTORIES[
'ml_hate_speech_path'] + '/' + self.bert_model + '/vocab.txt'
config_file = DIRECTORIES[
'ml_hate_speech_path'] + '/' + self.bert_model + '/bert_config.json'
bert_model_file = DIRECTORIES[
'ml_hate_speech_path'] + '/' + self.bert_model + '/' #+'-pytorch_model.bin'
self.tokenizer = BertTokenizer.from_pretrained(tokenizer_file)
config = BertConfig.from_json_file(config_file)
self.model = BertForSequenceClassification.from_pretrained(
bert_model_file, state_dict=model_state_dict, num_labels=2)
# self.model = BertForSequenceClassification.from_pretrained(self.bert_model, state_dict=model_state_dict,
# num_labels=2)
self.model.to(self.device)
def load_model(model_type):
checkpoint = torch.load(f'checkpoints/{model_type}.pt', map_location='cpu')
config = BertConfig.from_json_file('models/config.json')
model = Summarizer(args=None,
device="cpu",
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
return model
def evaluate_config():
# args = parser.parse_args()
data_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/temp'
bert_model_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/bert-base-chinese-pytorch'
model_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/experiments/base_model'
seed = 23 # help="random seed for initialization"
restore_file = 'best' # help="name of the file in `model_dir` containing weights to load"
multi_gpu = False # action='store_true', help="Whether to use multiple GPUs if available"
fp16 = False # action='store_true', help="Whether to use 16-bit float precision instead of 32-bit"
# Load the parameters from json file
json_path = os.path.join(model_dir, 'params.json')
assert os.path.isfile(
json_path), "No json configuration file found at {}".format(json_path)
params = Params(json_path)
# Use GPUs if available
params.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
params.n_gpu = torch.cuda.device_count()
params.multi_gpu = multi_gpu
# Set the random seed for reproducible experiments
random.seed(seed)
torch.manual_seed(seed)
if params.n_gpu > 0:
torch.cuda.manual_seed_all(seed) # set random seed for all GPUs
params.seed = seed
# Set the logger
set_logger(os.path.join(model_dir, 'evaluate.log'))
# Create the input data pipeline
logging.info("Loading the dataset...")
# Initialize the DataLoader
data_loader = DataLoader(data_dir, bert_model_dir, params, token_pad_idx=0)
logging.info("- done.")
# Define the model
config_path = os.path.join(bert_model_dir, 'bert_config.json')
config = BertConfig.from_json_file(config_path)
model = BertForTokenClassification(config, num_labels=len(params.tag2idx))
model.to(params.device)
# Reload weights from the saved file
load_checkpoint(os.path.join(model_dir, restore_file + '.pth.tar'), model)
if fp16:
model.half()
if params.n_gpu > 1 and multi_gpu:
model = torch.nn.DataParallel(model)
return data_loader, model, params
def load_model():
config = BertConfig.from_json_file(MODEL_CONFIG_PATH)
model = BertForTokenClassification(config, num_labels=len(LABELS))
state_dict = torch.load(MODEL_PATH)
model.load_state_dict(state_dict)
model.eval()
return (
model,
BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True),
)
def __init__(self,
args,
device,
load_pretrained_bert=False,
bert_config=None):
super(Summarizer, self).__init__()
self.args = args
self.device = device
self.bert = Bert(args,
load_pretrained_bert=True,
bert_config=BertConfig.from_json_file(
args.bert_config_path))
self.bert_config = BertConfig.from_json_file(args.bert_config_path)
if (args.encoder == 'classifier'):
self.encoder = Classifier(self.bert_config.hidden_size)
elif (args.encoder == 'transformer'):
self.encoder = TransformerInterEncoder(
self.bert_config.hidden_size, args.ff_size, args.heads,
args.dropout, args.inter_layers)
elif (args.encoder == 'rnn'):
self.encoder = RNNEncoder(bidirectional=True,
num_layers=1,
input_size=self.bert_config.hidden_size,
hidden_size=args.rnn_size,
dropout=args.dropout)
elif (args.encoder == 'baseline'):
bert_config = BertConfig(self.bert_config.vocab_size,
hidden_size=args.hidden_size,
num_hidden_layers=6,
num_attention_heads=8,
intermediate_size=args.ff_size)
self.bert.model = BertModel(bert_config)
self.encoder = Classifier(self.bert_config.hidden_size)
if args.param_init != 0.0:
for p in self.encoder.parameters():
p.data.uniform_(-args.param_init, args.param_init)
if args.param_init_glorot:
for p in self.encoder.parameters():
if p.dim() > 1:
xavier_uniform_(p)
self.to(device)
def load_BFTC_from_TF_ckpt(bert_config, ckpt_path, num_labels):
"""
Helper function for loading model - workaround to prevent error
"""
config = BertConfig.from_json_file(bert_config)
model = BertForPreTraining(config)
load_tf_weights_in_bert(model, ckpt_path)
state_dict=model.state_dict()
model = BertForTokenClassification(config, num_labels=num_labels)
# Load from a PyTorch state_dict
old_keys = []
new_keys = []
for key in state_dict.keys():
new_key = None
if 'gamma' in key:
new_key = key.replace('gamma', 'weight')
if 'beta' in key:
new_key = key.replace('beta', 'bias')
if new_key:
old_keys.append(key)
new_keys.append(new_key)
for old_key, new_key in zip(old_keys, new_keys):
state_dict[new_key] = state_dict.pop(old_key)
missing_keys = []
unexpected_keys = []
error_msgs = []
# copy state_dict so _load_from_state_dict can modify it
metadata = getattr(state_dict, '_metadata', None)
state_dict = state_dict.copy()
if metadata is not None:
state_dict._metadata = metadata
def load(module, prefix=''):
local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
module._load_from_state_dict(
state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs)
for name, child in module._modules.items():
if child is not None:
load(child, prefix + name + '.')
start_prefix = ''
if not hasattr(model, 'bert') and any(s.startswith('bert.') for s in state_dict.keys()):
start_prefix = 'bert.'
load(model, prefix=start_prefix)
if len(missing_keys) > 0:
print("Weights of {} not initialized from pretrained model: {}".format(
model.__class__.__name__, missing_keys))
if len(unexpected_keys) > 0:
print("Weights from pretrained model not used in {}: {}".format(
model.__class__.__name__, unexpected_keys))
if len(error_msgs) > 0:
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
model.__class__.__name__, "\n\t".join(error_msgs)))
return model
def _convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path):
# adapated from https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/pytorch_pretrained_bert/convert_tf_checkpoint_to_pytorch.py#L30
# Initialise PyTorch model
config = BertConfig.from_json_file(bert_config_file)
model = BertForPreTraining(config)
# Load weights from tf checkpoint
_load_tf_weights_in_bert(model, tf_checkpoint_path)
# Save pytorch-model
torch.save(model.state_dict(), pytorch_dump_path)
def test_convert_onnx(self):
model = BertModel(BertConfig.from_json_file(BERT_CONFIG_PATH))
model.train(False)
output = torch.onnx.export(
model,
self.org_dummy_input,
self.model_onnx_path,
verbose=True,
operator_export_type=OPERATOR_EXPORT_TYPE,
input_names=['input_ids', 'token_type_ids', 'attention_mask'])
print("Export of torch_model.onnx complete!")
def load_biobert_model(biobert_pth, device):
"""Read saved state dict for biobert model on disk
Args:
biobert_pth: str, folder path where model state dictionary and config file are saved
"""
bert_config_file = os.path.join(biobert_pth, 'bert_config.json')
config = BertConfig.from_json_file(bert_config_file)
print("Building PyTorch model from configuration: {}".format(str(config)))
model = BertForPreTraining(config)
model.load_state_dict(torch.load(os.path.join(biobert_pth, 'biobert_statedict.pkl'), map_location=device))
return model
def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file,
pytorch_dump_path):
# Initialise PyTorch model
config = BertConfig.from_json_file(bert_config_file)
print("Building PyTorch model from configuration: {}".format(str(config)))
model = BertForPreTraining(config)
# Load weights from tf checkpoint
load_tf_weights_in_bert(model, tf_checkpoint_path)
# Save pytorch-model
print("Save PyTorch model to {}".format(pytorch_dump_path))
torch.save(model.state_dict(), pytorch_dump_path)
def __init__(self, args, w=None, b=None):
# 初始化模型，建立encoder和decoder
super(Summarizer, self).__init__()
if args.mode == "train":
self.encoder = BertModel.from_pretrained('bert-base-cased',
cache_dir="./temp")
self.w = w
self.b = b
elif args.mode == "test":
config = BertConfig.from_json_file(args.predict_config)
self.encoder = BertModel(config)
self.args = args
# we choose same hiedden_size with bert embedding
self.decoder = nn.GRU(input_size=768, hidden_size=768, num_layers=1)
def __init__(self, args, Y):
super(Bert_seq_cls, self).__init__()
print("loading pretrained bert from {}".format(args.bert_dir))
config_file = os.path.join(args.bert_dir, 'bert_config.json')
self.config = BertConfig.from_json_file(config_file)
print("Model config {}".format(self.config))
self.bert = BertModel.from_pretrained(args.bert_dir)
self.dim_reduction = nn.Linear(self.config.hidden_size,
args.num_filter_maps)
self.dropout = nn.Dropout(self.config.hidden_dropout_prob)
self.classifier = nn.Linear(args.num_filter_maps, Y)
self.apply(self.init_bert_weights)
def __init__(self, args, tagset_size, embedding_dim, hidden_dim, rnn_layers, dropout_ratio, dropout1, use_cuda=False):
super(BERT_LSTM_CRF, self).__init__()
self.embedding_dim = embedding_dim
self.hidden_dim = hidden_dim
self.word_embeds = BertModel(config=BertConfig.from_json_file(args.bert_config_json))
# print(self.word_embeds)
self.word_embeds.load_state_dict(torch.load('./ckpts/9134_bert_weight.bin'))
self.lstm = nn.LSTM(embedding_dim, hidden_dim,
num_layers=rnn_layers, bidirectional=True, dropout=dropout_ratio, batch_first=True)
self.rnn_layers = rnn_layers
self.dropout1 = nn.Dropout(p=dropout1)
self.crf = CRF(target_size=tagset_size, average_batch=True, use_cuda=use_cuda)
self.liner = nn.Linear(hidden_dim*2, tagset_size+2)
self.tagset_size = tagset_size
def __init__(self, hidden_size=768, linear_out=2, batch_first=True):
super(Classifier, self).__init__()
self.output_model_file = "lm/pytorch_model.bin"
self.output_config_file = "lm/config.json"
self.tokenizer = BertTokenizer.from_pretrained("lm",
do_lower_case=False)
self.config = BertConfig.from_json_file(self.output_config_file)
self.model = BertForMaskedLM(self.config)
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
self.state_dict = torch.load(self.output_model_file,
map_location=device)
self.model.load_state_dict(self.state_dict)
self.lstm = torch.nn.LSTM(hidden_size, 300)
self.linear = torch.nn.Linear(300, linear_out)
def save_state_dict(args, device_id):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
ckpt = "cnndm_bertsum_classifier_best.pt"
logger.info('Loading checkpoint from %s' % ckpt)
checkpoint = torch.load(ckpt, map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
config = BertConfig.from_json_file(args.bert_config_path)
model = Summarizer(args, device, load_pretrained_bert=False, bert_config = config)
model.load_cp(checkpoint)
model.eval()
# save state_dict
torch.save(model.state_dict(), "weights.pt")
def load_model(args, device, load_bert=False, checkpoint=None):
"""Loads a model given by args.encoder using a checkpoint file if specified"""
config = BertConfig.from_json_file(args.bert_config_path)
if args.encoder == 'bertsum':
logger.info('Loading BertSum baseline model')
model = BertSum(args, device, load_bert=load_bert, bert_config=config)
else:
raise ValueError('unknown encoder %s' % args.encoder)
# Load checkpointed model
if checkpoint:
logger.info('Loading checkpoint from %s' % checkpoint)
checkpoint = torch.load(checkpoint,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
model.load_cp(checkpoint)
# Load optimizer if resuming training, otherwise fresh optimizer
if checkpoint and args.mode == 'train':
logger.info("Restoring checkpointed optimizer state")
#optim = Optimizer.from_opt(model, args, checkpoint)
optim = checkpoint['optim']
saved_optimizer_state_dict = optim.optimizer.state_dict()
optim.set_parameters(list(model.named_parameters()))
optim.optimizer.load_state_dict(saved_optimizer_state_dict)
if args.visible_gpus != '-1':
for state in optim.optimizer.state.values():
for k, v in state.items():
if torch.is_tensor(v):
state[k] = v.cuda()
else:
logger.info("Using fresh optimizer state")
optim = Optimizer.from_opt(model, args)
return model, optim
def embed_data(x_data):
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
output_model_file = "lm/pytorch_model.bin"
output_config_file = "lm/config.json"
tokenizer = BertTokenizer.from_pretrained("lm", do_lower_case=False)
config = BertConfig.from_json_file(output_config_file)
model = BertForMaskedLM(config)
state_dict = torch.load(output_model_file, map_location=device)
model.load_state_dict(state_dict)
entries = []
data_iterator = tqdm(x_data, desc='Loading embeddings')
for entry in data_iterator:
entries.append(get_embeddings(entry, model))
entries = torch.stack(entries)
return entries
def evaluate(args, device_id, checkpoint, step, mode):
""" Evaluates a model in either validation or test mode """
if mode not in ['valid', 'test']:
raise ValueError('mode must be [test, valid], got %s' % mode)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if checkpoint == '': checkpoint = args.test_from
logger.info("Evaluating (%s) from checkpoint %s", mode, checkpoint)
config = BertConfig.from_json_file(args.bert_config_path)
model, _ = load_model(args, device, load_bert=False, checkpoint=checkpoint)
model.eval()
dataset = data.load(args, mode, device)
trainer = build_trainer(args, device_id, model, None)
if mode == 'valid':
stats = trainer.validate(dataset, step)
else:
stats = trainer.test(dataset, step)
return stats
def __init__(self):
super(BioBertModel, self).__init__()
self.config_bert = BertConfig.from_json_file(
BIO_BERT_DIR + 'biobert_v1.1_pubmed/bert_config.json')
self.bert_model = BertModel.from_pretrained(BIO_BERT_DIR +
'biobert_v1.1_pubmed/',
from_tf=True)
self.dropout = nn.Dropout(DROP_PROB)
self.linear_mapping = nn.Linear(BERT_EMBEDDING_DIM, BERT_INTERMED_DIM)
self.relu = nn.ReLU()
self.batch_norm = nn.BatchNorm1d(BERT_INTERMED_DIM)
self.intermed_linear_mapping = nn.Linear(BERT_INTERMED_DIM,
BERT_INTERMED_DIM // 2)
self.intermed_batch_norm = nn.BatchNorm1d(BERT_INTERMED_DIM // 2)
self.intermed_linear_mapping_1 = nn.Linear(BERT_INTERMED_DIM // 2,
BERT_INTERMED_DIM // 4)
self.intermed_batch_norm_1 = nn.BatchNorm1d(BERT_INTERMED_DIM // 4)
self.classification_layer = nn.Linear(BERT_INTERMED_DIM // 2, 1)
self.sigmoid = nn.Sigmoid()
def getTranslator():
# set up model
device = "cpu"
logger.info('Loading checkpoint from %s' % args.test_from)
checkpoint = torch.load(args.test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
config = BertConfig.from_json_file(args.bert_config_path)
model = Summarizer(args,
device,
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
model.eval()
return build_trainer(args, -1, model, None)
