def generate_openai(model,
text_encoder,
n_samples,
seed_text,
batch_size=10,
gen_len=20,
topk=100,
temperature=1.0,
sample=True,
n_special=0,
print_every=1):
sents = []
start_time = time.time()
n_batches = math.ceil(n_samples / batch_size)
for batch_n in range(n_batches):
batch_sents = _generate_sentence_openai(model,
text_encoder,
seed_text,
batch_size=batch_size,
gen_len=gen_len,
topk=topk,
sample=sample,
n_special=n_special)
sents += batch_sents
settings.write_debug(
"Generated batch %d of %d in %.3fs" %
(batch_n + 1, n_batches, time.time() - start_time))
start_time = time.time()
return sents
def load_perplexity_model():
settings.write_debug("Loading pre-trained transformer_lm.wiki103.adaptive")
# Load an English LM trained on wiki103 data
en_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wiki103.adaptive', tokenizer='moses')
en_lm.requires_grad=False
en_lm.eval() # disable dropout
settings.write_debug('Finished Loading pre-trained transformer_lm.wiki103.adaptive')
return en_lm
def get_device():
#Check if there is a GPU available
if torch.cuda.is_available():
device = torch.device("cuda")
settings.write_debug('Using GPU:' + torch.cuda.get_device_name(0))
# Uses the CPU if no GPU available
else:
settings.write_debug('Using CPU.')
device = torch.device("cpu")
return device
def evaluate_gpt(output_file_path:str):
settings.write_debug("Loading and Generating GPT model")
gpt_model, gpt_text_encoder = load_openai_gpt(n_special=1)
openai_sents = generate_openai(gpt_model,
gpt_text_encoder,
seed_text="",
n_samples=settings.get_num_eval_samples(),
batch_size=settings.get_eval_batch_size(),
gen_len=settings.get_sample_size(),
topk=settings.get_eval_top_k(),
temperature=settings.get_eval_temp(),
sample=settings.get_eval_sample(),
n_special=1,
print_every=1)
with open(output_file_path, 'a+') as f:
[f.write('[%d/%d]\t' % (index, len(openai_sents)) + " ".join(sent) + "\n") for index, sent in enumerate(openai_sents)]
max_n = settings.get_bleu_max_n()
openai_avg_p = np.average([(perplexity_model.score(" ".join(sent))['positional_scores'].mean().neg().exp()).item() for sent in openai_sents])
settings.write_result("GPT Perplexity: %.2f" % openai_avg_p)
with open(settings.get_proc_wiki_path(), 'rb') as proc_wiki_f, open(settings.get_proc_tbc_path(), 'rb') as proc_tbc_f:
wiki_data = pickle.load(proc_wiki_f)
tbc_data = pickle.load(proc_tbc_f)
settings.write_result("OpenAI self-BLEU: %.2f" % (100 * self_bleu(openai_sents)))
settings.write_result("GPT-TBC BLEU: %.2f" % (100 * corpus_bleu(openai_sents, tbc_data)))
settings.write_result("GPT-Wiki103 BLEU: %.2f" % (100 * corpus_bleu(openai_sents, wiki_data)))
settings.write_result("GPT-{TBC + Wiki103} BLEU: %.2f" % (100 * corpus_bleu(openai_sents, tbc_data[:2500] + wiki_data[:2500])))
pct_uniques = ref_unique_ngrams(openai_sents, wiki_data, max_n)
for i in range(1, max_n + 1):
settings.write_result("GPT unique %d-grams relative to Wiki: %.2f" % (i, 100 * pct_uniques[i]))
pct_uniques = ref_unique_ngrams(openai_sents, tbc_data, max_n)
for i in range(1, max_n + 1):
settings.write_result("GPT unique %d-grams relative to TBC: %.2f" % (i, 100 * pct_uniques[i]))
pct_uniques = self_unique_ngrams(openai_sents, max_n)
for i in range(1, max_n + 1):
settings.write_result("GPT unique %d-grams relative to self: %.2f" % (i, 100 * pct_uniques[i]))
settings.write_result("")
def evaluate_bert(generator, output_file_path:str):
settings.write_debug("Entering Bleu EVAL")
with torch.no_grad():
### Using 1 eval topk here breaks computation because it is essentially argmax and non-differentiable
# Choose the prefix context
generated_input_ids = generator.generate(settings.get_num_eval_samples(),
seed_text=settings.get_eval_seed_text().split(),
batch_size=settings.get_eval_batch_size(),
max_len=settings.get_sample_size(),
generation_mode=settings.get_eval_gen_mode_key(),
sample=settings.get_eval_sample(),
top_k=settings.get_eval_top_k(),
temperature=settings.get_eval_temp(),
burnin=settings.get_eval_burnin(),
max_iter=settings.get_eval_max_iter())
bert_sents = [generator.detokenize(tokenizer.convert_ids_to_tokens(sent.tolist())).split() for sent in generated_input_ids]
with open(output_file_path, 'a+') as f:
[f.write('[%d/%d]\t' % (index, len(bert_sents)) + " ".join(sent) + "\n") for index, sent in enumerate(bert_sents)]
avg_p = np.average([(perplexity_model.score(" ".join(sent))['positional_scores'].mean().neg().exp()).item() for sent in bert_sents])
settings.write_result("BERT Perplexity: %.2f" % avg_p)
settings.write_result("BERT self-BLEU: %.2f" % (100 * self_bleu(bert_sents)))
max_n = settings.get_bleu_max_n()
with open(settings.get_proc_wiki_path(), 'rb') as proc_wiki_f, open(settings.get_proc_tbc_path(), 'rb') as proc_tbc_f:
wiki_data = pickle.load(proc_wiki_f)
tbc_data = pickle.load(proc_tbc_f)
settings.write_result("BERT-TBC BLEU: %.2f" % (100 * corpus_bleu(bert_sents, tbc_data)))
settings.write_result("BERT-Wiki103 BLEU: %.2f" % (100 * corpus_bleu(bert_sents, wiki_data)))
settings.write_result("BERT-{TBC + Wiki103} BLEU: %.2f" % (100 * corpus_bleu(bert_sents, tbc_data[:2500] + wiki_data[:2500])))
pct_uniques = ref_unique_ngrams(bert_sents, wiki_data, max_n)
for i in range(1, max_n + 1):
settings.write_result("BERT unique %d-grams relative to Wiki: %.2f" % (i, 100 * pct_uniques[i]))
pct_uniques = ref_unique_ngrams(bert_sents, tbc_data, max_n)
for i in range(1, max_n + 1):
settings.write_result("BERT unique %d-grams relative to TBC: %.2f" % (i, 100 * pct_uniques[i]))
pct_uniques = self_unique_ngrams(bert_sents, max_n)
for i in range(1, max_n + 1):
settings.write_result("BERT unique %d-grams relative to self: %.2f" % (i, 100 * pct_uniques[i]))
settings.write_result("")
#Helper method used to get GPU or CPU object for selecting model attachement
def get_device():
#Check if there is a GPU available
if torch.cuda.is_available():
device = torch.device("cuda")
settings.write_debug('Using GPU:' + torch.cuda.get_device_name(0))
# Uses the CPU if no GPU available
else:
settings.write_debug('Using CPU.')
device = torch.device("cpu")
return device
def load_perplexity_model():
settings.write_debug("Loading pre-trained transformer_lm.wiki103.adaptive")
# Load an English LM trained on wiki103 data
en_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wiki103.adaptive', tokenizer='moses')
en_lm.requires_grad=False
en_lm.eval() # disable dropout
settings.write_debug('Finished Loading pre-trained transformer_lm.wiki103.adaptive')
return en_lm
settings.write_debug('Loading Utility models')
device = get_device() #Using for loading models to GPU
model_type=settings.get_model_type() #bert_base_uncased
tokenizer = BertTokenizer.from_pretrained(model_type, do_lower_case=model_type.endswith("uncased"))
perplexity_model = load_perplexity_model().to(device) #Outside model used for evaluating perplexity of BERT models
settings.write_debug('Finished loading utility models')