def __init__(self):
# 加载数据
self.sents_src, self.sents_tgt = read_corpus()
# 判断是否有可用GPU
self.device = torch.device(
"cuda:3" if torch.cuda.is_available() else "cpu")
print("device: " + str(self.device))
# 定义模型
self.gpt_model = load_gpt(word2idx)
## 加载预训练的模型参数～
self.gpt_model.load_pretrain_params(model_path)
# 将模型发送到计算设备(GPU或CPU)
self.gpt_model.set_device(self.device)
# 声明需要优化的参数
self.optim_parameters = list(self.gpt_model.parameters())
self.optimizer = torch.optim.Adam(self.optim_parameters,
lr=lr,
weight_decay=1e-3)
# 声明自定义的数据加载器
dataset = SeqDataset(self.sents_src, self.sents_tgt)
self.dataloader = DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_fn)
def __init__(self):
# 判断是否有可用GPU
# self.device = torch.device("cpu")
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("device: " + str(self.device))
# 定义模型
self.model = load_gpt(word2ix, tokenizer=tokenizer)
self.model.load_pretrain_params(model_path)
# 加载已经训练好的模型，继续训练
# 将模型发送到计算设备(GPU或CPU)
self.model.set_device(self.device)
# 声明需要优化的参数
self.optim_parameters = list(self.model.parameters())
self.optimizer = torch.optim.Adam(self.optim_parameters, lr=lr, weight_decay=1e-3)
# 声明自定义的数据加载器
dataset = GPTDataset()
self.dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)
import sys
sys.path.append(
"/Users/xingzhaohu/Downloads/code/python/ml/ml_code/bert/bert_seq2seq")
import torch
from bert_seq2seq.utils import load_gpt
from bert_seq2seq.tokenizer import load_chinese_base_vocab
from transformers import AutoTokenizer
import os
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model_path = "./state_dict/english_gpt_model/english_gpt_story.bin"
if __name__ == "__main__":
tokenizer = AutoTokenizer.from_pretrained(
"pranavpsv/gpt2-genre-story-generator")
word2ix = tokenizer.get_vocab()
model = load_gpt(word2ix, tokenizer=tokenizer)
model.eval()
model.set_device(device)
model.load_pretrain_params(model_path)
print(
model.sample_generate_english("Nice weather today",
out_max_length=300))
import sys
sys.path.append(
"/Users/xingzhaohu/Downloads/code/python/ml/ml_code/bert/bert_seq2seq")
import torch
from bert_seq2seq.utils import load_gpt
from bert_seq2seq.tokenizer import load_chinese_base_vocab
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
vocab_path = "./state_dict/gpt2通用中文模型/vocab.txt"
model_path = "./state_dict/gpt_ancient_trans_model.bin"
if __name__ == "__main__":
word2ix = load_chinese_base_vocab(vocab_path)
model = load_gpt(word2ix)
model.eval()
model.set_device(device)
model.load_all_params(model_path)
print(
model.sample_generate("余忆童稚时，能张目对日。", out_max_length=300,
add_eos=True))