def __init__(self): # 加载数据 self.sents_src, self.sents_tgt = read_corpus() # 判断是否有可用GPU self.device = torch.device( "cuda:3" if torch.cuda.is_available() else "cpu") print("device: " + str(self.device)) # 定义模型 self.gpt_model = load_gpt(word2idx) ## 加载预训练的模型参数~ self.gpt_model.load_pretrain_params(model_path) # 将模型发送到计算设备(GPU或CPU) self.gpt_model.set_device(self.device) # 声明需要优化的参数 self.optim_parameters = list(self.gpt_model.parameters()) self.optimizer = torch.optim.Adam(self.optim_parameters, lr=lr, weight_decay=1e-3) # 声明自定义的数据加载器 dataset = SeqDataset(self.sents_src, self.sents_tgt) self.dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)
def __init__(self): # 判断是否有可用GPU # self.device = torch.device("cpu") self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print("device: " + str(self.device)) # 定义模型 self.model = load_gpt(word2ix, tokenizer=tokenizer) self.model.load_pretrain_params(model_path) # 加载已经训练好的模型,继续训练 # 将模型发送到计算设备(GPU或CPU) self.model.set_device(self.device) # 声明需要优化的参数 self.optim_parameters = list(self.model.parameters()) self.optimizer = torch.optim.Adam(self.optim_parameters, lr=lr, weight_decay=1e-3) # 声明自定义的数据加载器 dataset = GPTDataset() self.dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)
import sys sys.path.append( "/Users/xingzhaohu/Downloads/code/python/ml/ml_code/bert/bert_seq2seq") import torch from bert_seq2seq.utils import load_gpt from bert_seq2seq.tokenizer import load_chinese_base_vocab from transformers import AutoTokenizer import os device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model_path = "./state_dict/english_gpt_model/english_gpt_story.bin" if __name__ == "__main__": tokenizer = AutoTokenizer.from_pretrained( "pranavpsv/gpt2-genre-story-generator") word2ix = tokenizer.get_vocab() model = load_gpt(word2ix, tokenizer=tokenizer) model.eval() model.set_device(device) model.load_pretrain_params(model_path) print( model.sample_generate_english("Nice weather today", out_max_length=300))
import sys sys.path.append( "/Users/xingzhaohu/Downloads/code/python/ml/ml_code/bert/bert_seq2seq") import torch from bert_seq2seq.utils import load_gpt from bert_seq2seq.tokenizer import load_chinese_base_vocab device = torch.device("cuda" if torch.cuda.is_available() else "cpu") vocab_path = "./state_dict/gpt2通用中文模型/vocab.txt" model_path = "./state_dict/gpt_ancient_trans_model.bin" if __name__ == "__main__": word2ix = load_chinese_base_vocab(vocab_path) model = load_gpt(word2ix) model.eval() model.set_device(device) model.load_all_params(model_path) print( model.sample_generate("余忆童稚时,能张目对日。", out_max_length=300, add_eos=True))