def val_gpt(): bpe = get_bpe_from_files(encoder_path, vocab_path) df = pd.read_csv('new_processed_data/train.csv') text = df['comment_text'].fillna('none').values idens = df[IDENTITY_COLUMNS].values label = df['target'].values _, test_text, _, test_label, _, test_iden = train_test_split( text, label, idens, test_size=0.055, random_state=59) model = get_gpt_model(config_path, checkpoint_path) def pad_fn(ts): ts = [bpe.encode(t)[:512] for t in ts] return seq_padding(ts, truncate=False) print(bpe.encode(test_text[0])) model.load_weights('save_models/gpt.weights-new_weight-2.h5') val_gen = GeneralPredictGenerator(test_text, 24, pad_fn=pad_fn) res = model.predict_generator(val_gen.__iter__(), len(val_gen))[0].flatten() eva = JigsawEvaluator(test_label, test_iden) final_auc, overall_auc, sub_auc, bpsn_auc, bnsp_auc, bias_metrics = eva.get_final_metric( res) print( 'Final AUC:{}\nOverall AUC:{}\nSub AUC:{}\nBPSN AUC:{}\nBNSP AUC:{}\n'. format(final_auc, overall_auc, sub_auc, bpsn_auc, bnsp_auc)) print('Detail Bias:\n', bias_metrics) res.save('results/gpt_res_2.npy')
def getValue(): clear_session() if (request.form['submit_btn'] == "Submit"): text = request.form['user_str'] length = request.form['user_len'] elif (request.form['submit_btn'] == "Generate"): text = " " length = random.randint(1, 40) print('Load model from checkpoint...') model = load_trained_model_from_checkpoint(config_path, checkpoint_path) print('Load BPE from files...') bpe = get_bpe_from_files(encoder_path, vocab_path) #if(text!=None): print('Generate text...') #output = generate(model, bpe, ['From the day forth, my arm'], length=20, top_k=1) output = generate(model, bpe, [str(text)], length=int(length), top_k=2) #print(output) ind = output[0].rfind("\n") temp = output[0] temp = temp[0:ind] #print(temp) output[0] = temp #print(output) try: if (request.form['tweet'] == "post"): Tweet(str(output[0])) except: print("") return render_template('index.html', t=output)
def _build_token2idx_from_gpt(self): encoder_path = os.path.join(self.model_folder, 'encoder.json') vocab_path = os.path.join(self.model_folder, 'vocab.bpe') bpe: gpt2.BytePairEncoding = gpt2.get_bpe_from_files( encoder_path, vocab_path) token2idx = bpe.token_dict.copy() self.processor.token2idx = token2idx self.processor.idx2token = dict([(value, key) for key, value in token2idx.items()])
def test_encode_and_decode(self): current_path = os.path.dirname(os.path.abspath(__file__)) toy_checkpoint_path = os.path.join(current_path, 'toy_checkpoint') encoder_path = os.path.join(toy_checkpoint_path, 'encoder.json') vocab_path = os.path.join(toy_checkpoint_path, 'vocab.bpe') bpe = get_bpe_from_files(encoder_path, vocab_path) text = 'Power, give me more power!' indices = bpe.encode(text) self.assertEqual([13434, 11, 1577, 502, 517, 1176, 0], indices) self.assertEqual(text, bpe.decode(indices)) self.assertEqual(text, bpe.decode(bpe.encode(text)))
def load_model_and_bpe(): model_folder = 'models/' + config_model config_path = os.path.join(model_folder, 'hparams.json') checkpoint_path = os.path.join(model_folder, 'model.ckpt') encoder_path = os.path.join(model_folder, 'encoder.json') vocab_path = os.path.join(model_folder, 'vocab.bpe') text_model = keras_gpt_2.load_trained_model_from_checkpoint( config_path, checkpoint_path) bpe = keras_gpt_2.get_bpe_from_files(encoder_path, vocab_path) return text_model, bpe
def build(self, **kwargs): self.embedding_type = 'gpt2' config_path = os.path.join(self.name, 'hparams.json') checkpoint_path = os.path.join(self.name, 'model.ckpt') encoder_path = os.path.join(self.name, 'encoder.json') vocab_path = os.path.join(self.name, 'vocab.bpe') self._model: Model = load_trained_model_from_checkpoint( config_path, checkpoint_path) for layer in self._model.layers: layer.trainable = False self.bpe: BytePairEncoding = get_bpe_from_files( encoder_path, vocab_path) word2idx = self.bpe.token_dict.copy() word2idx[k.PAD] = word2idx['pad'] word2idx[k.UNK] = word2idx['unk'] word2idx[k.BOS] = word2idx['pad'] word2idx[k.EOS] = word2idx['pad'] self.token2idx = word2idx
from keras_gpt_2 import load_trained_model_from_checkpoint, get_bpe_from_files, generate global model global bpe model_folder = './models/117M' config_path = os.path.join(model_folder, 'hparams.json') checkpoint_path = os.path.join(model_folder, 'model.ckpt') encoder_path = os.path.join(model_folder, 'encoder.json') vocab_path = os.path.join(model_folder, 'vocab.bpe') print('Load model from checkpoint...') model = load_trained_model_from_checkpoint(config_path, checkpoint_path) print('Load BPE from files...') bpe = get_bpe_from_files(encoder_path, vocab_path) print('Running...') def cleanup_old_tasks(ttl=900): now = time.time() for job_type in ['jobs/pending','jobs/done']: jobs = os.listdir(job_type) jobs = cleanup_task_list(jobs) for job in jobs: job_path = job_type+"/"+job job_age = os.path.getmtime(job_path) if(now - ttl > job_age): os.remove(job_path) print("Delete too old "+job_path) def cleanup_task_list(task_list):
def train_gpt(): bpe = get_bpe_from_files(encoder_path, vocab_path) import pickle # with open('tok_text_uncased.pkl', 'rb') as h: # text = pickle.load(h) with open('y_train.pkl', 'rb') as h: label = pickle.load(h) with open('y_aux.pkl', 'rb') as h: aux = pickle.load(h) iden_df = pd.read_csv('processed_data/train_tok_iden.csv') weights = get_weights_new(iden_df) del iden_df df = pd.read_csv('new_processed_data/train.csv') text = df['comment_text'].values del df train_text, _, train_label, _, train_aux, _, train_weights, _ = train_test_split( text, label, aux, weights, test_size=0.055, random_state=59) def pad_fn(ts): ts = [bpe.encode(t)[:512] for t in ts] return seq_padding(ts, truncate=False) train_gen = GeneralDataGenerator( inputs=[ train_text, ], outputs=[train_label, train_aux], sample_weights=[train_weights, np.ones_like(train_weights)], batch_size=16, pad_fn=[ pad_fn, ]) with tf.device('/cpu:0'): model = get_gpt_model(config_path, checkpoint_path) lr = 2e-5 weight_decay = 0.01 bsz = 32 decay_steps = 2 * len(train_gen) warmup_steps = int(0.05 * decay_steps) optimizer = AdamWarmup( decay_steps=decay_steps, warmup_steps=warmup_steps, lr=lr, weight_decay=weight_decay, ) lw = 1 / np.mean(train_weights) model.load_weights('save_models/gpt.weights-new_weight.h5') parallel_model = multi_gpu_model(model, gpus=2) parallel_model.compile(loss='binary_crossentropy', optimizer=optimizer, loss_weights=[lw, 1.]) parallel_model.fit_generator(train_gen.__iter__(), steps_per_epoch=len(train_gen), epochs=2, max_queue_size=100, initial_epoch=1) model.save('save_models/gpt.weights-new_weight-2.h5') print("DONE")