def getValue():
clear_session()
if (request.form['submit_btn'] == "Submit"):
text = request.form['user_str']
length = request.form['user_len']
elif (request.form['submit_btn'] == "Generate"):
text = " "
length = random.randint(1, 40)
print('Load model from checkpoint...')
model = load_trained_model_from_checkpoint(config_path, checkpoint_path)
print('Load BPE from files...')
bpe = get_bpe_from_files(encoder_path, vocab_path)
#if(text!=None):
print('Generate text...')
#output = generate(model, bpe, ['From the day forth, my arm'], length=20, top_k=1)
output = generate(model, bpe, [str(text)], length=int(length), top_k=2)
#print(output)
ind = output[0].rfind("\n")
temp = output[0]
temp = temp[0:ind]
#print(temp)
output[0] = temp
#print(output)
try:
if (request.form['tweet'] == "post"):
Tweet(str(output[0]))
except:
print("")
return render_template('index.html', t=output)
def test_load_from_checkpoint(self):
current_path = os.path.dirname(os.path.abspath(__file__))
toy_checkpoint_path = os.path.join(current_path, 'toy_checkpoint')
config_path = os.path.join(toy_checkpoint_path, 'hparams.json')
checkpoint_path = os.path.join(toy_checkpoint_path, 'model.ckpt')
model = load_trained_model_from_checkpoint(config_path=config_path, checkpoint_path=checkpoint_path)
model.summary()
def _build_model(self, **kwargs):
if self.embed_model is None and self.sequence_length != 'auto':
config_path = os.path.join(self.model_folder, 'hparams.json')
checkpoint_path = os.path.join(self.model_folder, 'model.ckpt')
model = gpt2.load_trained_model_from_checkpoint(
config_path, checkpoint_path, self.sequence_length)
model.summary()
self.embed_model = model
def load_model_and_bpe():
model_folder = 'models/' + config_model
config_path = os.path.join(model_folder, 'hparams.json')
checkpoint_path = os.path.join(model_folder, 'model.ckpt')
encoder_path = os.path.join(model_folder, 'encoder.json')
vocab_path = os.path.join(model_folder, 'vocab.bpe')
text_model = keras_gpt_2.load_trained_model_from_checkpoint(
config_path, checkpoint_path)
bpe = keras_gpt_2.get_bpe_from_files(encoder_path, vocab_path)
return text_model, bpe
def get_gpt_model(config_path, checkpoint_path):
model = load_trained_model_from_checkpoint(config_path, checkpoint_path)
inputs = model.inputs[0]
mask = Lambda(lambda x: K.reshape(K.sum(K.cast(K.greater(x, 0), 'float32'), axis=-1), [K.shape(x)[0], 1]) - 1,
name='Mask')(inputs)
# mask = Lambda(lambda x: print(K.shape(x)),
# name='Mask')(inputs)
layer = model.get_layer(name='Norm').output
layer = Lambda(seq_gather, name='Gather')([layer, mask])
predict = Dense(1, activation='sigmoid', name='Predict-Dense')(layer)
aux = Dense(6, activation='sigmoid', name='Predict-Aux')(layer)
model = Model(inputs=inputs, outputs=[predict, aux])
model.summary()
return model
def build(self, **kwargs):
self.embedding_type = 'gpt2'
config_path = os.path.join(self.name, 'hparams.json')
checkpoint_path = os.path.join(self.name, 'model.ckpt')
encoder_path = os.path.join(self.name, 'encoder.json')
vocab_path = os.path.join(self.name, 'vocab.bpe')
self._model: Model = load_trained_model_from_checkpoint(
config_path, checkpoint_path)
for layer in self._model.layers:
layer.trainable = False
self.bpe: BytePairEncoding = get_bpe_from_files(
encoder_path, vocab_path)
word2idx = self.bpe.token_dict.copy()
word2idx[k.PAD] = word2idx['pad']
word2idx[k.UNK] = word2idx['unk']
word2idx[k.BOS] = word2idx['pad']
word2idx[k.EOS] = word2idx['pad']
self.token2idx = word2idx
is_uuid_pkl = re.compile("^[a-f0-9]{40}.pkl$")
from keras_gpt_2 import load_trained_model_from_checkpoint, get_bpe_from_files, generate
global model
global bpe
model_folder = './models/117M'
config_path = os.path.join(model_folder, 'hparams.json')
checkpoint_path = os.path.join(model_folder, 'model.ckpt')
encoder_path = os.path.join(model_folder, 'encoder.json')
vocab_path = os.path.join(model_folder, 'vocab.bpe')
print('Load model from checkpoint...')
model = load_trained_model_from_checkpoint(config_path, checkpoint_path)
print('Load BPE from files...')
bpe = get_bpe_from_files(encoder_path, vocab_path)
print('Running...')
def cleanup_old_tasks(ttl=900):
now = time.time()
for job_type in ['jobs/pending','jobs/done']:
jobs = os.listdir(job_type)
jobs = cleanup_task_list(jobs)
for job in jobs:
job_path = job_type+"/"+job
job_age = os.path.getmtime(job_path)
if(now - ttl > job_age):
os.remove(job_path)
print("Delete too old "+job_path)
mc_token_ids = mc_token_ids[:index]
print(lm_labels.shape)
print(input_ids.shape)
print(mc_token_ids.shape)
print(mc_labels.shape)
if not os.path.isdir(model_folder):
gpt2.download_gpt2(model_name = '117M')
strategy = tf.distribute.MirroredStrategy()
print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
batch_size = 2
with strategy.scope():
model = load_trained_model_from_checkpoint(config_path, checkpoint_path, batch_size=batch_size)
print("starting fit")
history_output = model.fit(
{
'LMInput': input_ids,
'MCInput': mc_token_ids
},
{
'LMOutput': lm_labels,
'MCOutput': mc_labels
},
batch_size=batch_size * strategy.num_replicas_in_sync,
epochs=2,
callbacks=[tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,
save_weights_only=True)]
)
import os
import sys
from keras_gpt_2 import load_trained_model_from_checkpoint, get_bpe_from_files, generate
epoch_number = 2
already_trained = True
checkpoint_dir = './training_checkpoints'
checkpoint_path = os.path.join(checkpoint_dir, f"ckpt_{epoch_number}")
model_folder = 'models/117M'
config_path = os.path.join(model_folder, 'hparams.json')
checkpoint_path = os.path.join(model_folder, 'model.ckpt')
encoder_path = os.path.join(model_folder, 'encoder.json')
vocab_path = os.path.join(model_folder, 'vocab.bpe')
print('Load model from checkpoint...')
model = load_trained_model_from_checkpoint(config_path,
checkpoint_path,
already_trained=already_trained)
print('Load BPE from files...')
bpe = get_bpe_from_files(encoder_path, vocab_path)
print('Generate text...')
output = generate(model,
bpe, ['From the day forth, my arm'],
length=20,
top_k=40)
# If you are using the 117M model and top_k equals to 1, then the result would be:
# "From the day forth, my arm was broken, and I was in a state of pain. I was in a state of pain,"
print(output[0])
if epoch_number == 0:
strategy = tf.distribute.MirroredStrategy()
elif epoch_number == 1:
strategy = tf.distribute.MirroredStrategy(devices=[
'/device:GPU:0', '/device:GPU:1', '/device:GPU:2', '/device:GPU:3'
])
else:
strategy = tf.distribute.MirroredStrategy(devices=[
'/device:GPU:4', '/device:GPU:5', '/device:GPU:6', '/device:GPU:7'
])
#print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
with strategy.scope():
model = load_trained_model_from_checkpoint(config_path,
checkpoint_path,
batch_size=None,
already_trained=already_trained)
timer = Timer()
i = 0
print(f"Time since last iteration to do: {timer()}")
#print("Done")
# print(input_ids[i:i+batch_size].shape)
# print(mc_token_ids[i:i+batch_size].shape)
# print(lm_labels[i:i+batch_size].shape)
# print(mc_labels[i:i+batch_size].shape)
metrics = model.evaluate(x={
'LMInput': input_ids,
'MCInput': mc_token_ids
},
y={
