def openAIGPTModel(*args, **kwargs):
"""
OpenAIGPTModel is the basic OpenAI GPT Transformer model based on
identical stacked masked self-attention blocks and pre-trained
on large scale dataset using language modeling signal.
Example:
# Load the tokenizer
>>> import torch
>>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'openAIGPTTokenizer', 'openai-gpt')
# Prepare tokenized input
>>> text = "Who was Jim Henson ? Jim Henson was a puppeteer"
>>> tokenized_text = tokenizer.tokenize(text)
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
>>> tokens_tensor = torch.tensor([indexed_tokens])
# Load openAIGPTModel
>>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'openAIGPTModel', 'openai-gpt')
>>> model.eval()
# Predict hidden states features for each layer
>>> with torch.no_grad():
hidden_states = model(tokens_tensor)
"""
model = OpenAIGPTModel.from_pretrained(*args, **kwargs)
return model
def convert_openai_checkpoint_to_pytorch(openai_checkpoint_folder_path, openai_config_file, pytorch_dump_folder_path):
# Construct model
if openai_config_file == "":
config = OpenAIGPTConfig()
else:
config = OpenAIGPTConfig(openai_config_file)
model = OpenAIGPTModel(config)
# Load weights from numpy
load_tf_weights_in_openai_gpt(model, openai_checkpoint_folder_path)
# Save pytorch-model
pytorch_weights_dump_path = pytorch_dump_folder_path + '/' + WEIGHTS_NAME
pytorch_config_dump_path = pytorch_dump_folder_path + '/' + CONFIG_NAME
print("Save PyTorch model to {}".format(pytorch_weights_dump_path))
torch.save(model.state_dict(), pytorch_weights_dump_path)
print("Save configuration file to {}".format(pytorch_config_dump_path))
with open(pytorch_config_dump_path, "w", encoding="utf-8") as f:
f.write(config.to_json_string())
def extractOpenAI():
model = OpenAIGPTModel.from_pretrained(modelPath[args.model])
embeddings = model.tokens_embed
print(embeddings.num_embeddings)
print(embeddings.weight.size())
tokenizer = OpenAIGPTTokenizer.from_pretrained(modelPath[args.model])
weight = embeddings.weight.detach().numpy()
#print(tokenizer.decoder)
with open(programmingalpha.openAI768 + "embeddings.txt", "w") as f:
for i in range(len(weight)):
vec = weight[i]
vec_str = list(map(lambda x: str(x), vec))
token = tokenizer.decoder[i]
vec_str.insert(0, token)
vec_str = " ".join(vec_str)
f.writelines(vec_str + "\n")
with open(programmingalpha.openAI768 + "vocab.txt", "w") as f:
for i in range(len(weight)):
token = tokenizer.decoder[i]
f.write(token + "\n")
def __init__(self, config, num_labels):
super(OpenAIGPTForClassification, self).__init__(config)
self.transformer = OpenAIGPTModel(config)
self.classifier = nn.Linear(config.n_embd, num_labels)
self.classifier.weight.data.uniform_(-0.1, 0.1)
self.classifier.bias.data.zero_()
class OpenAIGPTForClassification(OpenAIGPTPreTrainedModel):
"""OpenAI GPT model with a Language Modeling and a Multiple Choice head ("Improving Language Understanding by Generative Pre-Training").
OpenAI GPT use a single embedding matrix to store the word and special embeddings.
Special tokens embeddings are additional tokens that are not pre-trained: [SEP], [CLS]...
Special tokens need to be trained during the fine-tuning if you use them.
The number of special embeddings can be controled using the `set_num_special_tokens(num_special_tokens)` function.
The embeddings are ordered as follow in the token embeddings matrice:
[0, ----------------------
... -> word embeddings
config.vocab_size - 1, ______________________
config.vocab_size,
... -> special embeddings
config.vocab_size + config.n_special - 1] ______________________
where total_tokens_embeddings can be obtained as config.total_tokens_embeddings and is:
total_tokens_embeddings = config.vocab_size + config.n_special
You should use the associate indices to index the embeddings.
Params:
config: a OpenAIGPTConfig class instance with the configuration to build a new model
Inputs:
`input_ids`: a torch.LongTensor of shape [batch_size, num_choices, sequence_length] with the BPE token
indices selected in the range [0, total_tokens_embeddings[
`mc_token_ids`: a torch.LongTensor of shape [batch_size, num_choices] with the index of the token from
which we should take the hidden state to feed the multiple choice classifier (usually last token of the sequence)
`position_ids`: an optional torch.LongTensor with the same shape as input_ids
with the position indices (selected in the range [0, config.n_positions - 1[.
`token_type_ids`: an optional torch.LongTensor with the same shape as input_ids
You can use it to add a third type of embedding to each input token in the sequence
(the previous two being the word and position embeddings).
The input, position and token_type embeddings are summed inside the Transformer before the first
self-attention block.
`lm_labels`: optional language modeling labels: torch.LongTensor of shape [batch_size, num_choices, sequence_length]
with indices selected in [-1, 0, ..., total_tokens_embeddings]. All labels set to -1 are ignored (masked), the loss
is only computed for the labels set in [0, ..., total_tokens_embeddings]
`multiple_choice_labels`: optional multiple choice labels: torch.LongTensor of shape [batch_size]
with indices selected in [0, ..., num_choices].
Outputs:
if `lm_labels` and `multiple_choice_labels` are not `None`:
Outputs a tuple of losses with the language modeling loss and the multiple choice loss.
else: a tuple with
`lm_logits`: the language modeling logits as a torch.FloatTensor of size [batch_size, num_choices, sequence_length, total_tokens_embeddings]
`multiple_choice_logits`: the multiple choice logits as a torch.FloatTensor of size [batch_size, num_choices]
Example usage:
```python
# Already been converted into BPE token ids
input_ids = torch.LongTensor([[[31, 51, 99], [15, 5, 0]]]) # (bsz, number of choice, seq length)
mc_token_ids = torch.LongTensor([[2], [1]]) # (bsz, number of choice)
config = modeling_openai.OpenAIGPTConfig()
model = modeling_openai.OpenAIGPTLMHeadModel(config)
lm_logits, multiple_choice_logits = model(input_ids, mc_token_ids)
```
"""
def __init__(self, config, num_labels):
super(OpenAIGPTForClassification, self).__init__(config)
self.transformer = OpenAIGPTModel(config)
self.classifier = nn.Linear(config.n_embd, num_labels)
self.classifier.weight.data.uniform_(-0.1, 0.1)
self.classifier.bias.data.zero_()
def set_num_special_tokens(self, num_special_tokens):
""" Update input and output embeddings with new embedding matrice
Make sure we are sharing the embeddings
"""
self.transformer.set_num_special_tokens(num_special_tokens)
def forward(self,
input_ids,
input_mask,
labels=None,
token_type_ids=None,
position_ids=None):
# get sum of mask
hidden_states = self.transformer(input_ids, position_ids,
token_type_ids)
# calculate the position of last element
input_mask_sel = input_mask.sum(dim=1) - 1
input_mask_sel = input_mask_sel.unsqueeze(dim=1).unsqueeze(
dim=1).repeat(1, 1, 768)
# get the last hidden state
sentence_hidden = hidden_states.gather(index=input_mask_sel, dim=1)
sentence_hidden = sentence_hidden.squeeze(dim=1)
# hidden states pooling
logits = self.classifier(sentence_hidden)
return logits
word_to_idx = pickle.load(f)
train_val_dataset = FlowerDataset(
img_folder=FLOWERS_DATA_ROOT + 'jpg',
text_folder=FLOWERS_DATA_ROOT + 'train_val',
word_to_idx=word_to_idx,
idx_to_word=idx_to_word,
)
if args.use_skip_thought:
model = BayesianUniSkip('data/skip_thoughts', word_to_idx.keys())
elif args.use_bert:
model = BertModel.from_pretrained('bert-base-uncased')
model.eval()
elif args.use_gpt:
model = OpenAIGPTModel.from_pretrained('openai-gpt')
model.eval()
else:
model = RnnEncoder(dict_size=len(word_to_idx),
embed_size=args.embed_size,
hidden_dim=args.hidden_dim,
drop_prob=0.5)
generator = Generator()
discriminator = Discriminator()
dataloader = torch.utils.data.DataLoader(train_val_dataset,
batch_size=1,
shuffle=True)
model = model.to(device)
generator = generator.to(device)
discriminator = discriminator.to(device)
def inception_score(imgs,
model_file,
cuda=True,
batch_size=32,
resize=False,
splits=1):
"""Computes the inception score of the generated images imgs
imgs -- Torch dataset of (3xHxW) numpy images normalized in the range [-1, 1]
cuda -- whether or not to run on GPU
batch_size -- batch size for feeding into Inception v3
splits -- number of splits
"""
N = len(imgs)
assert batch_size > 0
assert N > batch_size
device = torch.device('cuda' if cuda else 'cpu')
# Set up dtype
if cuda:
dtype = torch.cuda.FloatTensor
else:
if torch.cuda.is_available():
print(
"WARNING: You have a CUDA device, so you should probably set cuda=True"
)
dtype = torch.FloatTensor
# Set up dataloader
dataloader = torch.utils.data.DataLoader(imgs,
batch_size=batch_size,
drop_last=True)
# Load generator and embeddings
if args.use_skip_thought:
model = BayesianUniSkip('data/skip_thoughts', imgs.word_to_idx.keys())
for param in model.parameters():
param.requires_grad = False
elif args.use_bert:
model = BertModel.from_pretrained('bert-base-uncased')
model.eval()
elif args.use_gpt:
model = OpenAIGPTModel.from_pretrained('openai-gpt')
model.eval()
else:
model = RnnEncoder(dict_size=len(word_to_idx),
embed_size=args.embed_size,
hidden_dim=args.rnn_hidden_dim,
drop_prob=0.5)
generator = Generator().to(device)
trainer = Trainer(dataloader, model, generator, None, None, None, None,
device, None)
trainer.load_model(model_file)
trainer.rnn_encoder.eval()
trainer.generator.eval()
# Load inception model
inception_model = inception_v3(pretrained=True,
transform_input=False).type(dtype)
inception_model.eval()
up = nn.Upsample(size=(299, 299), mode='bilinear').type(dtype)
def get_pred(x):
if resize:
x = up(x)
x = inception_model(x)
return F.softmax(x).data.cpu().numpy()
# Get predictions
preds = np.zeros((N // batch_size, 1000))
for i, batch in enumerate(dataloader, 0):
print("Calculating Inception Score... iter: {} / {} ".format(
i, N // batch_size),
end='\r')
# batch = batch.type(dtype)
# batchv = Variable(batch)
imgs, caps, cap_lens, fake_caps, fake_cap_lens = trainer.prepare_data(
batch)
# Text embedding
sent_emb, fake_sent_emb = trainer.embed_text(caps, cap_lens, fake_caps,
fake_cap_lens, batch_size)
batch_size_i = caps.size()[0]
sampled = torch.randn((batch_size_i, generator.z_size)).to(device)
batchv = generator(sent_emb, sampled)
preds[i * batch_size:i * batch_size + batch_size_i] = get_pred(batchv)
print()
# Now compute the mean kl-div
split_scores = []
for k in range(splits):
part = preds[k * (N // splits):(k + 1) * (N // splits), :]
py = np.mean(part, axis=0)
scores = []
for i in range(part.shape[0]):
pyx = part[i, :]
scores.append(entropy(pyx, py))
split_scores.append(np.exp(np.mean(scores)))
return np.mean(split_scores), np.std(split_scores)