def mine_triples(device, input_file, output_file, use_local_model=False): if use_local_model: print('loading BERT...') bert = BertForMaskedLM.from_pretrained("../models/BertForMaskedLM") print('loading GPT2...') gpt = GPT2LMHeadModel.from_pretrained("../models/GPT2LMHeadModel") else: print('loading BERT...') bert = BertForMaskedLM.from_pretrained(bert_model) print('loading GPT2...') gpt = GPT2LMHeadModel.from_pretrained(gpt2_model) """ 'concat': KnowledgeMiner( os.path.join(data_repo, candidate_file), device, DirectTemplate, bert ), 'template': KnowledgeMiner( os.path.join(data_repo, candidate_file), device, PredefinedTemplate, bert, grammar=False, template_loc=os.path.join(template_repo, single_templates) ), 'template_grammar': KnowledgeMiner( os.path.join(data_repo, candidate_file), device, PredefinedTemplate, bert, grammar=True, template_loc=os.path.join(template_repo, single_templates) ), """ knowledge_miners = { 'coherency': KnowledgeMiner(input_file, device, EnumeratedTemplate, bert, language_model=gpt, template_loc=os.path.join(template_repo, multiple_templates), use_local_model=use_local_model) } for template_type in knowledge_miners.keys(): predictions = run_experiment(template_type, knowledge_miners) triples = knowledge_miners[template_type].sentences.tuples scored_samples = list(zip(triples, predictions)) scored_samples.sort(key=lambda x: x[1], reverse=True) with open(output_file, "w") as f: for triple, pred in scored_samples: rel, head, tail = triple triple = (rel.lower(), head, tail) f.write("\t".join(triple) + "\t" + "{:.5f}".format(pred)) f.write("\n")
def get_model(args, device): if args.scratch: config = GPT2Config(n_ctx=args.context_length, n_positions=args.context_length) model = GPT2LMHeadModel(config) else: model = GPT2LMHeadModel.from_pretrained(args.model_name_or_path) #import torchsummary #torchsummary.summary(model, (args.context_length, vocab_size), args.train_batch_size) return model.to(device)
def __init__(self, type, model_name_or_path="gpt2"): super(LM, self).__init__() self.enc = GPT2Tokenizer.from_pretrained(model_name_or_path) if type == '345M': self.model = GPT2LMHeadModel.from_pretrained('output/') elif type == '117M': self.model = GPT2LMHeadModel.from_pretrained(model_name_or_path) self.model.to(self.device) self.model.eval() self.start_token = '<|endoftext|>'
def init_model(seed=0, model_path='gpt2'): ''' Parameters: ---------- seed : int seed number for different ramdomizers model_name_or_path : string, optional either model name for existing model or path for trained model ''' np.random.seed(seed) torch.random.manual_seed(seed) torch.cuda.manual_seed(seed) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") enc = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2LMHeadModel.from_pretrained('gpt2') model = nn.DataParallel(model) model.load_state_dict(torch.load(model_path)) model = model.module model.to(device) model.eval() return model, enc, device
def run_model(): parser = argparse.ArgumentParser() parser.add_argument('--model_name_or_path', type=str, default='', help='pretrained model name or path to local checkpoint') parser.add_argument("--seed", type=int, default=42) parser.add_argument("--load_checkpoint", '-c', type=str, default='') parser.add_argument("--fp16", type=boolean_string, default=False) parser.add_argument("--max_seq_length", type=int, default=128) parser.add_argument("--generation_length", type=int, default=20) parser.add_argument("--max_history", type=int, default=2) parser.add_argument("--temperature", type=float, default=1) parser.add_argument("--top_k", type=int, default=0) parser.add_argument("--top_p", type=float, default=0.9) parser.add_argument('--use_gpu', action='store_true') parser.add_argument("--gpu", type=int, default=0) args = parser.parse_args() os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu) device = torch.device("cuda" if torch.cuda.is_available() and args.use_gpu else "cpu") n_gpu = torch.cuda.device_count() args.device, args.n_gpu = device, n_gpu np.random.seed(args.seed) torch.random.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) #### load the GPT-2 model config = GPT2Config.from_json_file(os.path.join(args.model_name_or_path, 'config.json')) enc = GPT2Tokenizer.from_pretrained(args.model_name_or_path) model = load_model(GPT2LMHeadModel(config), args.load_checkpoint, args, verbose=True) model.to(device) model.eval() history = [] while True: raw_text = input("USR >>> ") while not raw_text: print('Prompt should not be empty!') raw_text = input("USR >>> ") if raw_text.lower() == 'quit': print('SYS >>> Goodbye!') break history.append(raw_text) context_tokens = sum([enc.encode(h) + [EOS_ID] for h in history],[]) #+ [EOS_ID] context_tokens = torch.tensor(context_tokens, device=device, dtype=torch.long).unsqueeze(0) position_ids = torch.arange(0, context_tokens.size(-1), dtype=torch.long, device=context_tokens.device) out = generate_sequence(model, context_tokens, position_ids=position_ids, length=args.generation_length, temperature=args.temperature, top_k=args.top_k, top_p= args.top_p) out = out.tolist() text = enc.decode(cut_seq_to_eos(out[0])).encode('ascii','ignore').decode('ascii') print("SYS >>> ", text) history.append(text) history = history[-(2*args.max_history+1):]
def get_optimizer(model: GPT2LMHeadModel, data_loader: Any, num_epochs: int, lr: float): params = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in params if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01 }, { 'params': [p for n, p in params if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] num_train_optimization_steps = len(data_loader) * num_epochs optimizer = OpenAIAdam( optimizer_grouped_parameters, lr=lr, t_total=num_train_optimization_steps, # the following group of parameters is taken from train_gpt2.py warmup=0.002, max_grad_norm=1.0, weight_decay=0.01, schedule="warmup_linear", b2=.99) return optimizer
def fluency_score(rated_a, opt): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") enc = GPT2Tokenizer.from_pretrained(opt.pretrained_model_path) model = GPT2LMHeadModel.from_pretrained(opt.pretrained_model_path) model.to(device) model.eval() nb_steps, eval_loss, exp_average_loss = 0, 0, None score_list = [] # k = "the book is on the desk. These impressions show , when alive , they had smooth skin , robust limbs with webbed feet , and a ridge of skin on their undersides." tensor(169.6684, device='cuda:0') with torch.no_grad(): for step, s in enumerate( rated_a): # actually here is a batch with batchsize=1 # Put model in training mode. if not s: print('space sentence') score_list.append(1e6) continue s = enc.encode( s) # + [50256] #50256 is the token_id for <|endoftext|> batch = torch.tensor([s]).to(device) loss = model(batch, lm_labels=batch) # everage -logp # print(loss*len(s)) eval_loss += loss.item() nb_steps += 1 score_list.append(loss.item()) cutoff = np.quantile([-t for t in score_list], 0.05) modified_rating = np.array( [cutoff if -t < cutoff else -t for t in score_list]) normed_rating = (modified_rating - cutoff) / np.abs(cutoff) return normed_rating
def mine_from_wikipedia(hardware): print('loading BERT...') bert = BertForMaskedLM.from_pretrained(bert_model) print('loading GPT2...') gpt = GPT2LMHeadModel.from_pretrained(gpt2_model) knowledge_miners = { 'concat': KnowledgeMiner(data_repo + wikipedia_candidates, hardware, DirectTemplate, bert), 'template': KnowledgeMiner(data_repo + wikipedia_candidates, hardware, PredefinedTemplate, bert, grammar=False, template_loc=template_repo + single_templates), 'template_grammar': KnowledgeMiner(data_repo + wikipedia_candidates, hardware, PredefinedTemplate, bert, grammar=True, template_loc=template_repo + single_templates), 'coherency': KnowledgeMiner(data_repo + wikipedia_candidates, hardware, EnumeratedTemplate, bert, language_model=gpt, template_loc=template_repo + multiple_templates) } for template_type in knowledge_miners.keys(): run_experiment(template_type, knowledge_miners)
def download_model(name): if not name in MODELS: raise Exception(str(name) + ' not a model in the list') if not exists(PATH): print("# ", str(PATH), "not found, creating dir.") mkdir(PATH) print('# Downloading model: ' + str(name)) name_path = MODEL_PATH_DICT[name] if name == 'word2vec': if not exists(join(PATH, name_path)): wget.download( 'https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz' ) shutil.move(name_path, join(PATH, name_path)) print('# Downloaded word2vec') else: print('# Already downloaded') if name == 'glove': if not exists(join(PATH, name_path)): wget.download( 'http://nlp.stanford.edu/data/wordvecs/glove.840B.300d.zip') zip = zipfile.ZipFile('./glove.840B.300d.zip') zip.extractall() _ = glove2word2vec('./glove.840B.300d.txt', join(PATH, name_path)) print('# Downloaded glove') else: print('# Already downloaded') if name == 'dict2vec': if not exists(join(PATH, name_path)): wget.download( 'https://dict2vec.s3.amazonaws.com/dict2vec300.tar.bz2') tar = tarfile.open("dict2vec300.tar.bz2") tar.extractall() tar.close() shutil.move(name_path, join(PATH, name_path)) print('# Downloaded dict2vec') else: print('# Already downloaded') if name == 'conceptnet': if not exists(join(PATH, name_path)): wget.download( 'https://conceptnet.s3.amazonaws.com/downloads/2019/numberbatch/numberbatch-en-19.08.txt.gz' ) shutil.move(name_path, join(PATH, name_path)) print('# Downloaded Conceptnet Numberbatch') else: print('# Already downloaded') if name == 'bert' or name == 'bert-context': _ = BertTokenizer.from_pretrained('bert-large-uncased') _ = BertModel.from_pretrained( 'bert-large-uncased').embeddings.word_embeddings.weight.data.numpy( ) print('# Downloaded bert') if name == 'gpt2' or name == 'gpt2-context': _ = GPT2Tokenizer.from_pretrained('gpt2') _ = GPT2LMHeadModel.from_pretrained('gpt2') _ = GPT2Model.from_pretrained('gpt2') print('# Downloaded gpt-2')
def __init__(self, model_name_or_path="gpt2"): super(LM, self).__init__() self.enc = GPT2Tokenizer.from_pretrained(model_name_or_path) self.model = GPT2LMHeadModel.from_pretrained(model_name_or_path) self.model.to(self.device) self.model.eval() self.start_token = '<|endoftext|>' print("Loaded GPT-2 model!")
def __init__(self,GPU, model_name_or_path="gpt2"): self.device = torch.device(GPU if torch.cuda.is_available() else "cpu") self.enc = GPT2Tokenizer.from_pretrained(model_name_or_path) self.model = GPT2LMHeadModel.from_pretrained(model_name_or_path) self.model.to(self.device) self.model.eval() self.start_token = '<|endoftext|>' print("Loaded GPT-2 model!")
def run_model(): parser = argparse.ArgumentParser() parser.add_argument('--model_name_or_path', type=str, default='gpt2', help='pretrained model name or path to local checkpoint') parser.add_argument("--seed", type=int, default=0) parser.add_argument("--nsamples", type=int, default=1) parser.add_argument("--batch_size", type=int, default=-1) parser.add_argument("--length", type=int, default=-1) parser.add_argument("--temperature", type=int, default=1) parser.add_argument("--top_k", type=int, default=0) parser.add_argument('--unconditional', action='store_true', help='If true, unconditional generation.') args = parser.parse_args() print(args) if args.batch_size == -1: args.batch_size = 1 assert args.nsamples % args.batch_size == 0 np.random.seed(args.seed) torch.random.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") enc = GPT2Tokenizer.from_pretrained(args.model_name_or_path) model = GPT2LMHeadModel.from_pretrained(args.model_name_or_path) model.to(device) model.eval() if args.length == -1: args.length = model.config.n_ctx // 2 elif args.length > model.config.n_ctx: raise ValueError("Can't get samples longer than window size: %s" % model.config.n_ctx) while True: context_tokens = [] if not args.unconditional: raw_text = input("Model prompt >>> ") while not raw_text: print('Prompt should not be empty!') raw_text = input("Model prompt >>> ") context_tokens = enc.encode(raw_text) generated = 0 for _ in range(args.nsamples // args.batch_size): out = sample_sequence( model=model, length=args.length, context=context_tokens if not args.unconditional else None, start_token=enc.encoder['<|endoftext|>'] if args.unconditional else None, batch_size=args.batch_size, temperature=args.temperature, top_k=args.top_k, device=device ) out = out[:, len(context_tokens):].tolist() for i in range(args.batch_size): generated += 1 text = enc.decode(out[i]) print("=" * 40 + " SAMPLE " + str(generated) + " " + "=" * 40) print(text) print("=" * 80) if args.unconditional: break
def __init__( self, model_name_or_path="/data/pradeesh/detecting-fake-text/pytorch/"): super(LM, self).__init__() self.enc = GPT2Tokenizer.from_pretrained(model_name_or_path) self.model = GPT2LMHeadModel.from_pretrained(model_name_or_path) self.model.to(self.device) self.model.eval() self.start_token = '<|endoftext|>' print("Loaded GPT-2 model!")
def __init__(self, args): super().__init__() if args.gpt2_model_dir is not None: # load GPT2 model from file gpt_model_name = str(args.gpt2_model_dir) + "/" dict_file = gpt_model_name print("loading GPT2 model from {}".format(gpt_model_name)) else: # load GPT2 model from huggingface cache gpt_model_name = args.gpt2_model_name dict_file = gpt_model_name # Load pre-trained model tokenizer (vocabulary) self.tokenizer = GPT2Tokenizer.from_pretrained(dict_file) # GPT uses different way to represent BPE then BERT. Namely, the # final suffixes are indicated with </w> suffix, while pieces that must # be followed are written as is. In BERT the prefixes are written as is # while the parts that must follow (not be followed!) have '##' prefix. # There is no one-to-one coversion. But at least we may make pieces that # may form a full word look the same. # Note that we should be very careful now, # tokenizer.convert_tokens_to_ids won't work with our vocabulary. def convert_word(word): if word == GPT2_EOS: return word if word.startswith('Ġ'): # the token starts with a whitespace return word[1:] return f'_{word}_' # the token not start with a white space. # may be not a head of a word, # or may be a head of a sentence. _, gpt_vocab = zip(*sorted(self.tokenizer.decoder.items())) self.vocab = [convert_word(word) for word in gpt_vocab] self._init_inverse_vocab() # Load pre-trained model (weights) self.gpt_model = GPT2LMHeadModel.from_pretrained(gpt_model_name) self.gpt_model.eval() # print(self.gpt_model.config) # Sanity check. assert len(self.vocab) == self.gpt_model.config.vocab_size #assert 0 == self.gpt_model.config.n_special self.eos_id = self.gpt_model.config.eos_token_id self.pad_id = self.gpt_model.config.eos_token_id self.unk_id = self.gpt_model.config.eos_token_id self.bos_id = self.gpt_model.config.bos_token_id self.model_vocab = self.vocab
def init(): #seed = 42 #np.random.seed(seed) #torch.random.manual_seed(seed) #torch.cuda.manual_seed(seed) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") enc = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2LMHeadModel.from_pretrained('gpt2') model.to(device) model.eval() return enc, model
def create_gpt2_lm_head(self, config, input_ids, token_type_ids, position_ids, mc_labels, lm_labels, mc_token_ids): model = GPT2LMHeadModel(config) model.eval() loss = model(input_ids, position_ids, token_type_ids, lm_labels) lm_logits, presents = model(input_ids, position_ids, token_type_ids) outputs = { "loss": loss, "lm_logits": lm_logits, "presents": presents, } return outputs
def mine(hardware): print('Loading GPT2...') gpt = GPT2LMHeadModel.from_pretrained(gpt2_model) knowledge_miners = { 'coherency': KnowledgeMiner( data_repo + test_data, hardware, EnumeratedTemplate, language_model = gpt, template_loc = template_repo + multiple_templates) } return run_experiment('coherency', knowledge_miners)
def get_prob(context, topk, genre, title): os.environ["CUDA_VISIBLE_DEVICES"] = '0' device = "cuda" if torch.cuda.is_available() else "cpu" tokenizer = tokenization_bert.BertTokenizer(vocab_file='cache/vocab_fine_tuning.txt') model_config = pytorch_pretrained_bert.GPT2Config.from_json_file('cache/model_config_single.json') model_state_dict = torch.load('cache/model_single/model_epoch_1.pt') model = GPT2LMHeadModel(config=model_config) model.load_state_dict(model_state_dict) model.to(device) model.eval() batch_size = 1 temperature = 1 context_tokens = [] with open('./cache/label_to_id.json','r',encoding='utf-8') as f: title_to_ids = json.load(f) try: ids = title_to_ids[genre] context_tokens.append(ids) except: ids = title_to_ids['七言律诗'] context_tokens.append(ids) context_tokens.append(100) context_tokens.extend(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(title))) context_tokens.append(4282) # 4282 is # raw_text = context if raw_text != "": context_tokens.extend(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(raw_text))) watcher = WatchProb(model=model, context=context_tokens, tokenizer=tokenizer, temperature=temperature, top_k=topk, device=device) prob_dis = watcher.show_prob(topk=topk) eight_cumu = watcher.show_cumulative(0.8) nine_cumu = watcher.show_cumulative(0.9) ninefive_cumu = watcher.show_cumulative(0.95) prob_dis.append("") prob_dis.append("") prob_dis.append("0.8累计覆盖: "+str(eight_cumu)) prob_dis.append("0.9累计覆盖: "+str(nine_cumu)) prob_dis.append("0.95累计覆盖: "+str(ninefive_cumu)) return prob_dis
def main(): parser = argparse.ArgumentParser() parser.add_argument('--batch_size',default=1,type=int,help='Batch size for inference') parser.add_argument('--model_name',default='gpt2',type=str, help='Pre-trained model name') parser.add_argument('--max_seq_length',default=128,type=int, help='Maximum total input sequence length after tokenization') args = parser.parse_args() input_ids = torch.zeros([args.batch_size,args.max_seq_length],dtype=torch.long) model = GPT2LMHeadModel.from_pretrained(args.model_name) torch.onnx.export(model,input_ids,'gpt2_'+'batch'+str(args.batch_size)+'.onnx')
def load_model(self, model_path='./cache/model/model_epoch_1.pt', model_config='./cache/model_config.json', device='cpu'): # /data/disk1/private/hujinyi/gpt_poem/model_with_title/model_epoch_1.pt self.device = "cuda" if torch.cuda.is_available() else "cpu" model_config = pytorch_pretrained_bert.GPT2Config.from_json_file( model_config) model_state_dict = torch.load(model_path) model = GPT2LMHeadModel(config=model_config) model.load_state_dict(model_state_dict) model.to(self.device) model.eval() self.model = model
def main(): LENGTH = -1 BATCH_SIZE = 1 NSAMPLES = 18 TEMPERATURE = 0.5 TOPK = 5 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") tokenizer = tokenization.BertTokenizer(vocab_file='cache/vocab.txt') model_config = pytorch_pretrained_bert.GPT2Config.from_json_file( 'model_config.json') model_state_dict = torch.load('./model.pt') model = GPT2LMHeadModel(config=model_config) model.load_state_dict(model_state_dict) model.to(device) model.eval() if LENGTH == -1: LENGTH = model.config.n_ctx // 2 elif LENGTH > model.config.n_ctx: raise ValueError("Can't get samples longer than window size: %s" % model.config.n_ctx) while True: raw_text = input("Model prompt >>> ") while not raw_text: print('Prompt should not be empty!') raw_text = input("Model prompt >>> ") context_tokens = tokenizer.convert_tokens_to_ids( tokenizer.tokenize(raw_text)) generated = 0 for _ in range(NSAMPLES // BATCH_SIZE): out = sample_sequence(model=model, length=LENGTH, context=context_tokens, start_token=None, batch_size=BATCH_SIZE, temperature=TEMPERATURE, top_k=TOPK, device=device) out = out[:, len(context_tokens):].tolist() for i in range(BATCH_SIZE): generated += 1 text = tokenizer.convert_ids_to_tokens(out[i]) print("=" * 40 + " SAMPLE " + str(generated) + " " + "=" * 40) print(text) print("=" * 80)
def __init__(self, text_sequence, model_type, temperature = 1.0, top_k = 0, batch_size = 1, length = 1, nsamples =1, debug = True): self.text_sequence = text_sequence #eventually will differentiate between gpt-2, BERT, etc. self.model_type = model_type model_name = 'gpt2' self.debug = debug #detect device self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.temperature = temperature self.top_k = top_k self.batch_size = batch_size self.length = length self.nsamples = nsamples #create encoder and model self.enc = GPT2Tokenizer.from_pretrained(model_name) self.model = GPT2LMHeadModel.from_pretrained(model_name) self.model.to(self.device) self.model.eval()
def init(self, model_path, model_checkpoint): self.config = GPT2Config.from_json_file(os.path.join(model_path, "config.json")) self.tokenizer = GPT2Tokenizer.from_pretrained(model_path) self.model = GPT2LMHeadModel(self.config) model_state_dict = fix_state_dict_namespace(torch.load(model_checkpoint)) start_model = self.model if hasattr(self.model, "transformer") and all(not s.startswith('transformer.') for s in model_state_dict.keys()): print('loading transfomer only') start_model = self.model.transformer start_model.load_state_dict(model_state_dict) if self.fp16: self.model.half() self.model.to(self.device) self.model.eval()
def context_score(questions, answers, opt): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") enc = GPT2Tokenizer.from_pretrained(opt.pretrained_model_path) model = GPT2LMHeadModel.from_pretrained(opt.pretrained_model_path) model.to(device) model.eval() score_list = [] with torch.no_grad(): for step, (question, answer) in enumerate( zip(questions, answers)): # actually here is a batch with batchsize=1 # Put model in training mode. if not answer: print('space sentence') score_list.append(-1e6) continue joint_enc = enc.encode( question + ' ' + answer) # + [50256] #50256 is the token_id for <|endoftext|> q = enc.encode(question) batch_joint = torch.tensor([joint_enc]).to(device) batch_q = torch.tensor([q]).to(device) loss_joint = model(batch_joint, lm_labels=batch_joint) # everage -logp loss_q = model(batch_q, lm_labels=batch_q) p_joint = -loss_joint * (len(joint_enc) - 1) p_q = -loss_q * (len(q) - 1) score = p_joint - (p_q) score_list.append(score.item()) cutoff = np.quantile(score_list, 0.05) modified_rating = np.array( [cutoff if t < cutoff else t for t in score_list]) normed_rating = (modified_rating - cutoff) / np.abs(cutoff) return normed_rating
def load_model_fromlist(name): if not name in MODELS: raise Exception(str(name) + ' not a model in the list') print('# Loading model: ' + str(name)) name_path = MODEL_PATH_DICT[name] if name == 'word2vec': if not exists(join(PATH, name_path)): download_model(name) return (gensim.models.KeyedVectors.load_word2vec_format(join( PATH, name_path), binary=True)) if name == 'glove': if not exists(join(PATH, name_path)): download_model(name) return (gensim.models.KeyedVectors.load_word2vec_format( join(PATH, name_path))) if name == 'dict2vec': if not exists(join(PATH, name_path)): download_model(name) return (gensim.models.KeyedVectors.load_word2vec_format( join(PATH, name_path), binary=False, unicode_errors="ignore")) if name == 'conceptnet': if not exists(join(PATH, name_path)): download_model(name) return (gensim.models.KeyedVectors.load_word2vec_format( join(PATH, name_path))) if name == 'bert': tokenizer = BertTokenizer.from_pretrained('bert-large-uncased') model = BertModel.from_pretrained( 'bert-large-uncased').embeddings.word_embeddings.weight.data.numpy( ) return ([model, tokenizer]) if name == 'bert-context': tokenizer = BertTokenizer.from_pretrained('bert-large-uncased') model = BertModel.from_pretrained('bert-large-uncased', output_hidden_states=True) return ([model, tokenizer]) if name == 'gpt2': tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2LMHeadModel.from_pretrained( 'gpt2').transformer.wte.weight.data.numpy() return ([model, tokenizer]) if name == 'gpt2-context': tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2Model.from_pretrained('gpt2', output_hidden_states=True) return ([model, tokenizer])
def run_model(): parser = argparse.ArgumentParser() parser.add_argument('--model_name_or_path', type=str, default='gpt2', help='pretrained model name or path to local checkpoint') parser.add_argument("--seed", type=int, default=0) parser.add_argument("--nsamples", type=int, default=1) parser.add_argument("--batch_size", type=int, default=-1) parser.add_argument("--length", type=int, default=-1) parser.add_argument("--temperature", type=float, default=1.0) parser.add_argument("--top_k", type=int, default=0) parser.add_argument('--unconditional', action='store_true', help='If true, unconditional generation.') parser.add_argument('--inputs_file', type=str, default=None) parser.add_argument('--output_file', type=str, default='results.json') parser.add_argument('--do_beam_search', type=bool, default=False) args = parser.parse_args() print(args) if args.batch_size == -1: args.batch_size = 1 assert args.nsamples % args.batch_size == 0 np.random.seed(args.seed) torch.random.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") enc = GPT2Tokenizer.from_pretrained(args.model_name_or_path) model = GPT2LMHeadModel.from_pretrained(args.model_name_or_path) model.to(device) model.eval() if args.length == -1: args.length = model.config.n_ctx // 2 elif args.length > model.config.n_ctx: raise ValueError("Can't get samples longer than window size: %s" % model.config.n_ctx) if args.inputs_file is None: decode_interactive(model, enc, device, args) else: decode_from_file(model, enc, device, args)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model_name', type=str, default='openai-gpt', help='pretrained model name') parser.add_argument("--do_train", action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", action='store_true', help="Whether to run eval on the dev set.") parser.add_argument("--output_dir", default='tuned_gpt2', type=str, required=True, help="The output directory where the model predictions and checkpoints will be written.") parser.add_argument('--train_dataset', type=str, default='') parser.add_argument('--source_eval', type=str, default='') parser.add_argument('--target_eval', type=str, default='') parser.add_argument('--source_train', type=str, default='') parser.add_argument('--target_train', type=str, default='') parser.add_argument('--eval_dataset', type=str, default='') parser.add_argument('--seed', type=int, default=42) parser.add_argument('--num_train_epochs', type=int, default=10) parser.add_argument('--train_batch_size', type=int, default=8) parser.add_argument('--effective_batch_size',type=int, default=64) parser.add_argument('--eval_batch_size', type=int, default=16) parser.add_argument('--max_grad_norm', type=int, default=1) parser.add_argument('--learning_rate', type=float, default=6.25e-5) parser.add_argument('--warmup_proportion', type=float, default=0.002) parser.add_argument('--lr_schedule', type=str, default='warmup_linear') parser.add_argument('--weight_decay', type=float, default=0.01) parser.add_argument('--lm_coef', type=float, default=0.9) parser.add_argument('--n_valid', type=int, default=374) parser.add_argument('--bsz', type=int, default = 20) parser.add_argument('--bptt', type=int, default = 40) parser.add_argument('--server_ip', type=str, default='', help="Can be used for distant debugging.") parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.") args = parser.parse_args() # print(args) model_type = 'gpt2' if args.server_ip and args.server_port: # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script import ptvsd print("Waiting for debugger attach") ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) ptvsd.wait_for_attach() random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) # device = torch.device("cuda" if torch.cuda.is_available() else "cpu") device = torch.device(type='cuda') n_gpu = torch.cuda.device_count() logger.info("device: {}, n_gpu {}".format(device, n_gpu)) # if not args.do_train and not args.do_eval: # raise ValueError("At least one of `do_train` or `do_eval` must be True.") if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2LMHeadModel.from_pretrained('gpt2').to('cuda') model.to(device) #file_train = args.train_dataset #'cnn_train.txt' #file_eval = args.eval_dataset #'cnn_valid.txt' bptt = args.bptt bsz = args.bsz # X_eval, nbatch_eval = load_dataset(file_eval, tokenizer, bptt, bsz) # X_train, nbatch_train = load_dataset(file_train, tokenizer, bptt, bsz) batches_eval, labels_eval, nbatch_eval = load_dataset(args.source_eval, args.target_eval, tokenizer, bptt, bsz) batches_train, labels_train, nbatch_train = load_dataset(args.source_train, args.target_train, tokenizer, bptt, bsz) # Prepare optimizer # param_optimizer = list(model.parameters()) param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [ {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01}, {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] print('here 3') # num_train_optimization_steps = len(train_data) * args.num_train_epochs // args.train_batch_size num_train_optimization_steps = nbatch_train * args.num_train_epochs optimizer = OpenAIAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, max_grad_norm=args.max_grad_norm, weight_decay=args.weight_decay, t_total=num_train_optimization_steps) eval_loss_min = None print('here 4') model.to(device) nb_tr_steps, tr_loss, exp_average_loss = 0, 0, None model.train() for epoch_i in trange(int(args.num_train_epochs), desc="Epoch"): tr_loss = 0 nb_tr_steps = 0 for i_batch in tqdm(list(range(nbatch_train)), desc='Evaluating epoch {}'.format(epoch_i)): batch = batches_train[i_batch]#X_train[:, i_batch*bsz:(1+i_batch)*bsz].permute(1,0) batch = batch.cuda() lm_labels = labels_train[i_batch].cuda() if batch.numel() == 0: break #loss = model(batch, lm_labels = labels_train[i_batch].cuda()) # TRY DOING IT MANUALLY loss_fct = CrossEntropyLoss(reduction = 'none') lm_logits,_ = model(batch) shift_logits = lm_logits[:, :-1, :].contiguous() shift_labels = batch[:,1:].contiguous() shift_labels_mask = (lm_labels[:,1:].contiguous().view(-1) != -1).float() loss_mat = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) loss = (loss_mat*shift_labels_mask).view(-1).sum()/shift_labels_mask.sum() # avg over non-masked indices loss.backward() # only step the model if you've gone through 'effective_batch_size' examples if (i_batch*args.train_batch_size) % args.effective_batch_size == 0 and i_batch != 0: optimizer.step() optimizer.zero_grad() tr_loss += loss.item() exp_average_loss = loss.item() if exp_average_loss is None else 0.7*exp_average_loss+0.3*loss.item() nb_tr_steps += 1 ### # Evaluations ### if i_batch % 1000 == 0: # get eval score eval_loss = eval_model(model, nbatch_eval,batches_eval,labels_eval, bsz) # if eval_loss improves, save model if eval_loss_min is None or eval_loss < eval_loss_min: eval_loss_min = eval_loss # save model if eval loss is lower model_to_save = model # If we save using the predefined names, we can load using `from_pretrained` output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME) output_config_file = os.path.join(args.output_dir, CONFIG_NAME) torch.save(model_to_save.state_dict(), output_model_file) to_json_file(model_to_save.config,output_config_file) print('eval_loss {}',format(eval_loss)) model.train() if i_batch % 200 == 0: # try generating from model print("Training loss: {:.2e} lr: {:.2e}".format(exp_average_loss, optimizer.get_lr()[0])) model.eval() if model_type == 'gpt': encode = lambda a: tokenizer.convert_tokens_to_ids(tokenizer.tokenize(a)) decode = tokenizer.decode elif model_type == 'gpt2': encode = tokenizer.encode decode = tokenizer.decode generate_from_model(encode, decode, model = model,model_type = model_type) model.train()
import numpy as np import torch import torch.nn.functional as F import tqdm from tensorboardX import SummaryWriter from torch.utils.data import DataLoader, Dataset from tqdm import trange import pytorch_pretrained_bert from data_loader import get_data_loader from model_sampler import print_samples from pytorch_pretrained_bert import GPT2LMHeadModel, GPT2Tokenizer, OpenAIAdam from torch.utils.data import DataLoader, Dataset, Subset model_name = 'gpt2' enc = GPT2Tokenizer.from_pretrained(model_name) model = GPT2LMHeadModel.from_pretrained(model_name) model_name = 'gpt2' enc = GPT2Tokenizer.from_pretrained(model_name) model = GPT2LMHeadModel.from_pretrained(model_name) device='cpu' beam_width = 130 stopwords = [] def to_list(tensor): return list(tensor.cpu().numpy()) def predict(line, max_predictions): """Give continuation of the line with at most max_predictions BPE tokens. Returns line extended with predictions of the model."""
def run(): parser = ArgumentParser() parser.add_argument("--model_type", type=str, default="gpt", help="gpt or gpt2") parser.add_argument("--model_checkpoint", type=str, default="", help="Path, url or short name of the model") parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu", help="Device (cuda or cpu)") parser.add_argument("--filename", type=str, default="data/instances_dev.pkl", help="File to use for decoding") parser.add_argument("--no_sample", action='store_true', help="Set to use greedy decoding instead of sampling") parser.add_argument("--max_length", type=int, default=50, help="Maximum length of the output utterances") parser.add_argument("--min_length", type=int, default=1, help="Minimum length of the output utterances") parser.add_argument("--seed", type=int, default=42, help="Seed") parser.add_argument("--temperature", type=int, default=0.7, help="Sampling softmax temperature") parser.add_argument("--top_k", type=int, default=0, help="Filter top-k tokens before sampling (<=0: no filtering)") parser.add_argument("--top_p", type=float, default=0.9, help="Nucleus filtering (top-p) before sampling (<=0.0: no filtering)") args = parser.parse_args() logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__file__) logger.info(pformat(args)) random.seed(args.seed) torch.random.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) logger.info("Get pretrained model and tokenizer") if args.model_type == 'gpt2': tokenizer = GPT2Tokenizer.from_pretrained(args.model_checkpoint) model = GPT2LMHeadModel.from_pretrained(args.model_checkpoint) else: tokenizer = OpenAIGPTTokenizer.from_pretrained(args.model_checkpoint) model = OpenAIGPTLMHeadModel.from_pretrained(args.model_checkpoint) model.to(args.device) model.eval() data = get_dataset_from_file(tokenizer, args.filename) final_output_dict = { "version": "squash-2.0", "data": [{ "paragraphs": [] }] } question_number = 0 # For all the instances corresponding one paragraph, model input format is: paragraph + answer + question) # Paragraph will be common accross all the instances. # "past" can be used to reuse precomputed hidden state for paragraph in a subsequent predictions imort copy previous_para_index = None past = None for inst in tqdm.tqdm(data): with torch.no_grad(): current_para_index = inst['para_index'] if current_para_index != prev_para_index: past = None currrent_inst = copy.deepcopy(inst) # only keeping paragraph details in the instance to get its hidden states current_inst['question'] = [] current_inst['answer'] = [] instance, _ = build_input_from_segments(current_inst,tokenizer,with_eos=False) input_ids = torch.tensor(instance['input_ids'][:-2],device=args.device).unsqueeze(0) token_type_ids = torch.tensor(instance['token_type_ids'][:-2],device=args.device).unsqueeze(0) _,past=model(input_ids,toekn_type_ids=toekn_type_ids,past=past) #output "past" will have paragraph embeddings output = sample_sequence(inst, tokenizer, model, args,past) original_paragraph = tokenizer.decode(output['paragraph']) generated_question = tokenizer.decode(output['question'], skip_special_tokens=True) original_answer = tokenizer.decode(output['answer'], skip_special_tokens=True) para_index = inst['para_index'] # Output in a SQUAD-like format with questions clumped together under their parent paragraph if len(final_output_dict["data"][0]["paragraphs"]) > para_index: # verify whether the paragraph text is identical assert original_paragraph == final_output_dict["data"][0]["paragraphs"][para_index]['context'] # append the question answer pair final_output_dict["data"][0]["paragraphs"][para_index]['qas'].append({ 'id': 'question_%d' % question_number, 'question': generated_question, 'answers': [{ 'text': original_answer, 'answer_start': original_paragraph.index(original_answer) }], 'class': output['class'], 'algorithm': output['algorithm'], 'is_impossible': False }) else: # add a new question to the list of QA pairs final_output_dict['data'][0]['paragraphs'].append({ 'context': original_paragraph, 'qas': [{ 'id': 'question_%d' % question_number, 'question': generated_question, 'answers': [{ 'text': original_answer, 'answer_start': original_paragraph.index(original_answer) }], 'class': output['class'], 'algorithm': output['algorithm'], 'is_impossible': False }] }) question_number += 1 with open("squash/temp/generated_questions.json", "w") as f: f.write(json.dumps(final_output_dict))
def run_model(): print(socket.gethostname()) parser = argparse.ArgumentParser() parser.add_argument( '--model_name_or_path', type=str, default='', help='pretrained model name or path to local checkpoint') parser.add_argument("--seed", type=int, default=42) parser.add_argument("--load_checkpoint", '-c', type=str, default='') parser.add_argument("--fp16", type=boolean_string, default=False) parser.add_argument("--test_file", '-t', type=str, default=None, help='input file for testing') parser.add_argument("--output_file", '-o', type=str, default=None, help='output file for testing') parser.add_argument("--normalize_data", type=boolean_string, default=True) parser.add_argument("--batch_size", '-b', type=int, default=256) parser.add_argument("--max_seq_length", type=int, default=512) parser.add_argument("--no_token_id", action='store_true') parser.add_argument("--no_attn_mask", action='store_true') parser.add_argument("--no_eos", action='store_true') parser.add_argument("--generation_length", type=int, default=20) parser.add_argument("--temperature", type=float, default=1) parser.add_argument("--top_k", type=int, default=0) parser.add_argument('--unconditional', action='store_true', help='If true, unconditional generation.') parser.add_argument('--is_sampling', action='store_true', help='If true, sampling for generation.') parser.add_argument('--output_ref', action='store_true', help='If true, output ref') #BEAM parser.add_argument("--beam", action='store_true', help='If true, beam search') parser.add_argument("--beam_width", type=int, default=1) parser.add_argument('--use_gpu', action='store_true') parser.add_argument("--gpu", type=int, default=0) parser.add_argument('--config', help='JSON config file') parser.add_argument('--eval', action='store_true') parser.add_argument('--cstr_decode', action='store_true') parser.add_argument("--bonus", type=float, default=0.0) args = parser.parse_args() os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu) if args.config is not None: # override argparse defaults by config JSON opts = json.load(open(args.config)) for k, v in opts.items(): if isinstance(v, str): # PHILLY ENV special cases if 'PHILLY_JOB_DIRECTORY' in v: v = v.replace('PHILLY_JOB_DIRECTORY', os.environ['PHILLY_JOB_DIRECTORY']) elif 'PHILLY_LOG_DIRECTORY' in v: v = v.replace('PHILLY_LOG_DIRECTORY', os.environ['PHILLY_LOG_DIRECTORY']) setattr(args, k, v) # command line should override config JSON argv = sys.argv[1:] overrides, _ = parser.parse_known_args(argv) for k, v in vars(overrides).items(): if f'--{k}' in argv: setattr(args, k, v) # setattr(args, 'local_rank', overrides.local_rank) # do normal parsing device = torch.device( "cuda" if torch.cuda.is_available() and args.use_gpu else "cpu") n_gpu = torch.cuda.device_count() args.device, args.n_gpu = device, n_gpu print(args) np.random.seed(args.seed) torch.random.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) config = GPT2Config.from_json_file( os.path.join(args.model_name_or_path, 'config.json')) enc = GPT2Tokenizer.from_pretrained(args.model_name_or_path) model = load_model(GPT2LMHeadModel(config), args.load_checkpoint, args, verbose=True) model.to(device) model.eval() if args.test_file: eval_dataloader = get_eval_list_same_length_with_order( args.test_file, enc, args.batch_size, True) model.eval() outs = [] targets = [] loss_all = [] ppl_all = [] sources = [] conv_ids = [] with torch.no_grad(): with tqdm.tqdm(total=len(eval_dataloader), desc=f"Test") as pbar: for step, batch in enumerate( tqdm.tqdm(eval_dataloader, desc="Iteration")): new_batch = [] for t in batch: if isinstance(t, list): new_batch.append(t) else: new_batch.append(t.to(device)) input_ids, position_ids, token_ids, attn_masks, label_ids, context_len, conv_id = new_batch if args.no_token_id: token_ids = None if args.no_eos: input_ids = input_ids[:, :-1] if args.no_attn_mask: attn_masks = None if args.beam: out = beam_search_naive(model, input_ids, position_ids=position_ids, token_type_ids=token_ids, attn_masks=attn_masks, length=args.generation_length, beam_width=args.beam_width, device=args.device, use_bonus=args.cstr_decode, bonus=args.bonus, enc=enc) else: out = generate_sequence(model, input_ids, position_ids=position_ids, token_type_ids=token_ids, attn_masks=attn_masks, length=args.generation_length, start_token=None, temperature=args.temperature, top_k=args.top_k, sample=args.is_sampling, use_bonus=args.cstr_decode, bonus=args.bonus, enc=enc) sources.extend(input_ids.cpu().numpy()) out = out.tolist() outs.extend(out) targets.extend(label_ids) conv_ids.extend(conv_id.cpu().numpy()) conv_id_map = {conv_ids[i]: i for i in range(len(conv_ids))} val_src = [ enc.decode( cut_seq_to_eos(s)).encode('utf-8').decode('utf-8') for s in sources ] #print(len(val_src),len(targets)) val_set = [ enc.decode(s).encode('utf-8').decode('utf-8') for s in targets ] gen = [ enc.decode( cut_seq_to_eos(s)).encode('utf-8').decode('utf-8') for s in outs ] val_src_orders = [ val_src[conv_id_map[i]] for i in sorted(conv_id_map) ] val_set_orders = [ val_set[conv_id_map[i]] for i in sorted(conv_id_map) ] gen_orders = [gen[conv_id_map[i]] for i in sorted(conv_id_map)] print("=" * 40 + " SAMPLE " + "=" * 40) src = enc.decode([ x for x in input_ids[-1].cpu().numpy() if x != 0 ]).encode('utf-8').decode('utf-8') gt = val_set[-1] resp = gen[-1] print( f"Source: \t {src} \n Oracle: \t {gt} \n Resp: \t {resp}\n" ) if args.output_file: with open(args.output_file + '.resp.txt', "w") as resp_f: for i, r in enumerate(gen_orders): r = re.sub("\n", "", r) if args.output_ref: # import pdb; pdb.set_trace() resp_f.write(val_src_orders[i] + '\t' + val_set_orders[i] + '\t' + r + '\n') else: resp_f.write(r + '\n') print("=" * 80) sys.stdout.flush() else: generated = 0 while True: raw_text = input("Model prompt >>> ") while not raw_text: print('Prompt should not be empty!') raw_text = input("Model prompt >>> ") context_tokens = enc.encode(raw_text) + [EOS_ID] context_tokens = torch.tensor(context_tokens, device=device, dtype=torch.long).unsqueeze( 0) #.repeat(batch_size, 1) generated += 1 position_ids = torch.arange(0, context_tokens.size(-1), dtype=torch.long, device=context_tokens.device) token_ids = None if args.no_token_id else torch.zeros_like( context_tokens, dtype=torch.long, device=context_tokens.device) if args.beam: out = beam_search_naive(model, context_tokens, position_ids=None, token_type_ids=token_ids, length=args.generation_length, beam_width=args.beam_width, device=args.device) else: out = generate_sequence(model, context_tokens, position_ids=None, token_type_ids=token_ids, length=args.generation_length, start_token=None, temperature=args.temperature, top_k=args.top_k, sample=args.is_sampling) out = out.tolist() text = enc.decode(cut_seq_to_eos( out[0])).encode('utf-8').decode('utf-8') print("=" * 40 + " RESPONSE " + str(generated) + " " + "=" * 40) print(text) print("=" * 80)
def __init__(self): self.model = GPT2LMHeadModel.from_pretrained('gpt2') self.tokenizer = GPT2Tokenizer.from_pretrained('gpt2') self.model.cuda() self.model.eval()
# "vocab_size": 50257 # } ## Predict hidden states features for each layer with torch.no_grad(): hidden_states_1, past = model(tokens_tensor_1) print(hidden_states_1.shape) # torch.Size([1, 6, 768]) print(len(past), past[0].shape) # 12 torch.Size([2, 1, 12, 6, 64]) hidden_states_2, past = model(tokens_tensor_2, past=past) print(hidden_states_2.shape) # torch.Size([1, 8, 768]) print(len(past), past[0].shape) # 12 torch.Size([2, 1, 12, 14, 64]); 14 = 8 + 6 ## past can be used to reuse precomputed hidden state in a subsequent predictions (see beam-search examples in the run_gpt2.py example). ################################################################## ## GPT2LMHeadModel model = GPT2LMHeadModel.from_pretrained('/Users/coder352/datasets/WordVec/pytorch_pretrained_bert/gpt2/') model.eval() ## Predict all tokens with torch.no_grad(): predictions_1, past = model(tokens_tensor_1) predictions_2, past = model(tokens_tensor_2, past=past) print(hidden_states_2.shape) # torch.Size([1, 8, 768]) print(len(past), past[0].shape) # 12 torch.Size([2, 1, 12, 14, 64]) ## get the predicted last token predicted_index = torch.argmax(predictions_2[0, -1, :]).item(); print(predicted_index) # 508 predicted_token = tokenizer.decode([predicted_index]); print(predicted_token) # who ################################################################## ## Transformer-XL