def val_gpt():
bpe = get_bpe_from_files(encoder_path, vocab_path)
df = pd.read_csv('new_processed_data/train.csv')
text = df['comment_text'].fillna('none').values
idens = df[IDENTITY_COLUMNS].values
label = df['target'].values
_, test_text, _, test_label, _, test_iden = train_test_split(
text, label, idens, test_size=0.055, random_state=59)
model = get_gpt_model(config_path, checkpoint_path)
def pad_fn(ts):
ts = [bpe.encode(t)[:512] for t in ts]
return seq_padding(ts, truncate=False)
print(bpe.encode(test_text[0]))
model.load_weights('save_models/gpt.weights-new_weight-2.h5')
val_gen = GeneralPredictGenerator(test_text, 24, pad_fn=pad_fn)
res = model.predict_generator(val_gen.__iter__(),
len(val_gen))[0].flatten()
eva = JigsawEvaluator(test_label, test_iden)
final_auc, overall_auc, sub_auc, bpsn_auc, bnsp_auc, bias_metrics = eva.get_final_metric(
res)
print(
'Final AUC:{}\nOverall AUC:{}\nSub AUC:{}\nBPSN AUC:{}\nBNSP AUC:{}\n'.
format(final_auc, overall_auc, sub_auc, bpsn_auc, bnsp_auc))
print('Detail Bias:\n', bias_metrics)
res.save('results/gpt_res_2.npy')
def getValue():
clear_session()
if (request.form['submit_btn'] == "Submit"):
text = request.form['user_str']
length = request.form['user_len']
elif (request.form['submit_btn'] == "Generate"):
text = " "
length = random.randint(1, 40)
print('Load model from checkpoint...')
model = load_trained_model_from_checkpoint(config_path, checkpoint_path)
print('Load BPE from files...')
bpe = get_bpe_from_files(encoder_path, vocab_path)
#if(text!=None):
print('Generate text...')
#output = generate(model, bpe, ['From the day forth, my arm'], length=20, top_k=1)
output = generate(model, bpe, [str(text)], length=int(length), top_k=2)
#print(output)
ind = output[0].rfind("\n")
temp = output[0]
temp = temp[0:ind]
#print(temp)
output[0] = temp
#print(output)
try:
if (request.form['tweet'] == "post"):
Tweet(str(output[0]))
except:
print("")
return render_template('index.html', t=output)
def _build_token2idx_from_gpt(self):
encoder_path = os.path.join(self.model_folder, 'encoder.json')
vocab_path = os.path.join(self.model_folder, 'vocab.bpe')
bpe: gpt2.BytePairEncoding = gpt2.get_bpe_from_files(
encoder_path, vocab_path)
token2idx = bpe.token_dict.copy()
self.processor.token2idx = token2idx
self.processor.idx2token = dict([(value, key)
for key, value in token2idx.items()])
def test_encode_and_decode(self):
current_path = os.path.dirname(os.path.abspath(__file__))
toy_checkpoint_path = os.path.join(current_path, 'toy_checkpoint')
encoder_path = os.path.join(toy_checkpoint_path, 'encoder.json')
vocab_path = os.path.join(toy_checkpoint_path, 'vocab.bpe')
bpe = get_bpe_from_files(encoder_path, vocab_path)
text = 'Power, give me more power!'
indices = bpe.encode(text)
self.assertEqual([13434, 11, 1577, 502, 517, 1176, 0], indices)
self.assertEqual(text, bpe.decode(indices))
self.assertEqual(text, bpe.decode(bpe.encode(text)))
def load_model_and_bpe():
model_folder = 'models/' + config_model
config_path = os.path.join(model_folder, 'hparams.json')
checkpoint_path = os.path.join(model_folder, 'model.ckpt')
encoder_path = os.path.join(model_folder, 'encoder.json')
vocab_path = os.path.join(model_folder, 'vocab.bpe')
text_model = keras_gpt_2.load_trained_model_from_checkpoint(
config_path, checkpoint_path)
bpe = keras_gpt_2.get_bpe_from_files(encoder_path, vocab_path)
return text_model, bpe
def build(self, **kwargs):
self.embedding_type = 'gpt2'
config_path = os.path.join(self.name, 'hparams.json')
checkpoint_path = os.path.join(self.name, 'model.ckpt')
encoder_path = os.path.join(self.name, 'encoder.json')
vocab_path = os.path.join(self.name, 'vocab.bpe')
self._model: Model = load_trained_model_from_checkpoint(
config_path, checkpoint_path)
for layer in self._model.layers:
layer.trainable = False
self.bpe: BytePairEncoding = get_bpe_from_files(
encoder_path, vocab_path)
word2idx = self.bpe.token_dict.copy()
word2idx[k.PAD] = word2idx['pad']
word2idx[k.UNK] = word2idx['unk']
word2idx[k.BOS] = word2idx['pad']
word2idx[k.EOS] = word2idx['pad']
self.token2idx = word2idx
from keras_gpt_2 import load_trained_model_from_checkpoint, get_bpe_from_files, generate
global model
global bpe
model_folder = './models/117M'
config_path = os.path.join(model_folder, 'hparams.json')
checkpoint_path = os.path.join(model_folder, 'model.ckpt')
encoder_path = os.path.join(model_folder, 'encoder.json')
vocab_path = os.path.join(model_folder, 'vocab.bpe')
print('Load model from checkpoint...')
model = load_trained_model_from_checkpoint(config_path, checkpoint_path)
print('Load BPE from files...')
bpe = get_bpe_from_files(encoder_path, vocab_path)
print('Running...')
def cleanup_old_tasks(ttl=900):
now = time.time()
for job_type in ['jobs/pending','jobs/done']:
jobs = os.listdir(job_type)
jobs = cleanup_task_list(jobs)
for job in jobs:
job_path = job_type+"/"+job
job_age = os.path.getmtime(job_path)
if(now - ttl > job_age):
os.remove(job_path)
print("Delete too old "+job_path)
def cleanup_task_list(task_list):
def train_gpt():
bpe = get_bpe_from_files(encoder_path, vocab_path)
import pickle
# with open('tok_text_uncased.pkl', 'rb') as h:
# text = pickle.load(h)
with open('y_train.pkl', 'rb') as h:
label = pickle.load(h)
with open('y_aux.pkl', 'rb') as h:
aux = pickle.load(h)
iden_df = pd.read_csv('processed_data/train_tok_iden.csv')
weights = get_weights_new(iden_df)
del iden_df
df = pd.read_csv('new_processed_data/train.csv')
text = df['comment_text'].values
del df
train_text, _, train_label, _, train_aux, _, train_weights, _ = train_test_split(
text, label, aux, weights, test_size=0.055, random_state=59)
def pad_fn(ts):
ts = [bpe.encode(t)[:512] for t in ts]
return seq_padding(ts, truncate=False)
train_gen = GeneralDataGenerator(
inputs=[
train_text,
],
outputs=[train_label, train_aux],
sample_weights=[train_weights,
np.ones_like(train_weights)],
batch_size=16,
pad_fn=[
pad_fn,
])
with tf.device('/cpu:0'):
model = get_gpt_model(config_path, checkpoint_path)
lr = 2e-5
weight_decay = 0.01
bsz = 32
decay_steps = 2 * len(train_gen)
warmup_steps = int(0.05 * decay_steps)
optimizer = AdamWarmup(
decay_steps=decay_steps,
warmup_steps=warmup_steps,
lr=lr,
weight_decay=weight_decay,
)
lw = 1 / np.mean(train_weights)
model.load_weights('save_models/gpt.weights-new_weight.h5')
parallel_model = multi_gpu_model(model, gpus=2)
parallel_model.compile(loss='binary_crossentropy',
optimizer=optimizer,
loss_weights=[lw, 1.])
parallel_model.fit_generator(train_gen.__iter__(),
steps_per_epoch=len(train_gen),
epochs=2,
max_queue_size=100,
initial_epoch=1)
model.save('save_models/gpt.weights-new_weight-2.h5')
print("DONE")