def getModel(): # load data from pickle and npy files metadata, idx_q, idx_a = data.load_data(PATH='datasets/cornell_corpus/') (trainX, trainY), (testX, testY), (validX, validY) = data_utils.split_dataset(idx_q, idx_a) train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 32) #print len(trainX) test_batch_gen = data_utils.rand_batch_gen(testX, testY, 256) input_ = test_batch_gen.next()[0] xseq_len = 25 yseq_len = 25 batch_size = 16 xvocab_size = len(metadata['idx2w']) yvocab_size = xvocab_size emb_dim = 1024 model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len, yseq_len=yseq_len, xvocab_size=xvocab_size, yvocab_size=yvocab_size, ckpt_path='ckpt/cornell_corpus/', emb_dim=emb_dim, num_layers=3 ) sess = model.restore_last_session() output = model.predict(sess, input_[0:25]) #print(output) return model, sess
def __init__(self, train=False): # load data from pickle and npy files self.metadata, idx_q, idx_a = data.load_data(PATH='datasets/twitter/') (trainX, trainY), (testX, testY), (validX, validY) = data_utils.split_dataset( idx_q, idx_a) # parameters xseq_len = trainX.shape[-1] yseq_len = trainY.shape[-1] batch_size = 16 xvocab_size = len(self.metadata['idx2w']) yvocab_size = xvocab_size emb_dim = 1024 importlib.reload(seq2seq_wrapper) self.model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len, yseq_len=yseq_len, xvocab_size=xvocab_size, yvocab_size=yvocab_size, ckpt_path='ckpt/twitter/', emb_dim=emb_dim, num_layers=3) if train: val_batch_gen = data_utils.rand_batch_gen(validX, validY, 32) train_batch_gen = data_utils.rand_batch_gen( trainX, trainY, batch_size) sess = self.model.train(train_batch_gen, val_batch_gen) self.sess = self.model.restore_last_session()
def get_output(question, sess, w2idx, model, metadata): import data_utils from datasets.facebook2 import data # get output for input phrase idx_q, idx_a = data.process_input(question, w2idx) gen = data_utils.rand_batch_gen(idx_q, idx_a, 1) input_ = gen.__next__()[0] output = model.predict(sess, input_) # return ouput phrase for ii, oi in zip(input_.T, output): q = data_utils.decode(sequence=ii, lookup=metadata['idx2w'], separator=' ') decoded = data_utils.decode(sequence=oi, lookup=metadata['idx2w'], separator=' ').split(' ') return ' '.join(decoded)
def get_response(self, text, metadata, sess): questions = [ text.lower() ] questions = [ filter_line(line, EN_WHITELIST) for line in questions ] answers = questions qlines, alines = filter_data(questions, answers) qtokenized = [ [w.strip() for w in wordlist.split(' ') if w] for wordlist in qlines ] atokenized = [ [w.strip() for w in wordlist.split(' ') if w] for wordlist in alines ] w2idx = pickle.load(open("datasets/cornell_corpus/w2idx.pkl","rb")) idx_q, idx_a = zero_pad(qtokenized, atokenized, w2idx) query = data_utils.rand_batch_gen(idx_q, idx_a, 1) input_q = query.__next__()[0] output = self.predict(sess, input_q) #print(input_q.shape) #replies = [] for ii, oi in zip(input_q.T, output): q = data_utils.decode(sequence=ii, lookup=metadata['idx2w'], separator=' ') decoded = data_utils.decode(sequence=oi, lookup=metadata['idx2w'], separator=' ').split(' ') ''' if decoded.count('unk') == 0: if decoded not in replies: print('q : [{0}]; a : [{1}]'.format(q, ' '.join(decoded))) replies.append(decoded) ''' return ' '.join(decoded)
import importlib importlib.reload(seq2seq_wrapper) # In[5]: model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len, yseq_len=yseq_len, xvocab_size=xvocab_size, yvocab_size=yvocab_size, ckpt_path='ckpt/twitter/', emb_dim=emb_dim, num_layers=3) # In[6]: val_batch_gen = data_utils.rand_batch_gen(validX, validY, 256) test_batch_gen = data_utils.rand_batch_gen(testX, testY, 256) train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, batch_size) # In[9]: sess = model.train(train_batch_gen, val_batch_gen) # In[7]: sess = model.restore_last_session() # In[10]: input_ = test_batch_gen.__next__()[0] output = model.predict(sess, input_)
xseq_len = trainX.shape[-1] yseq_len = trainY.shape[-1] batch_size = 32 xvocab_size = len(metadata['idx2w']) yvocab_size = xvocab_size emb_dim = 1024 import seq2seq_wrapper # In[7]: model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len, yseq_len=yseq_len, xvocab_size=xvocab_size, yvocab_size=yvocab_size, ckpt_path='ckpt/cornell_corpus/', emb_dim=emb_dim, num_layers=3 ) # In[8]: val_batch_gen = data_utils.rand_batch_gen(validX, validY, 32) train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, batch_size) # In[9]: #sess = model.restore_last_session() sess = model.train(train_batch_gen, val_batch_gen)
importlib.reload(seq2seq_wrapper) # In[47]: model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len, yseq_len=yseq_len, xvocab_size=xvocab_size, yvocab_size=yvocab_size, ckpt_path='ckpt/', emb_dim=emb_dim, num_layers=1, epochs=500) # In[21]: val_batch_gen = data_utils.rand_batch_gen(small_validX, small_validY, 100) test_batch_gen = data_utils.rand_batch_gen(small_testX, small_testY, 100) train_batch_gen = data_utils.rand_batch_gen(trainX_filter_10, trainA_filter_10, batch_size) train_batch_gen_story = data_utils.rand_batch_gen(trainX_filter_10, trainA_filter_10, 1) # In[ ]: sess = model.restore_last_session() sess = model.train(train_batch_gen, val_batch_gen, sess) # In[25]: sess = model.restore_last_session() # In[43]:
"train_ip size: {} train_lbl size: {} test_ip size: {}, test_lbl size:{}". format(train_ip.shape, train_lbl.shape, test_ip.shape, test_lbl.shape)) ip_seq_len = train_ip.shape[-1] lbl_seq_len = train_lbl.shape[-1] ip_vocab_size = len(block2idx) del block2idx lbl_vocab_size = ip_vocab_size model_name = os.path.splitext(fname)[0] + "segmentseq_model" train_ip_count = train_ip.shape[0] print("Train count: {}".format(train_ip_count)) print("Test count: {}".format(test_ip.shape[0])) train_batch_gen = rand_batch_gen(train_ip, train_lbl, batch_size) val_batch_gen = rand_batch_gen(test_ip, test_lbl, batch_size) model = SegSeq2SegSeq(train_size=train_ip_count, batch_size=batch_size, xseq_len=ip_seq_len, yseq_len=lbl_seq_len, xvocab_size=ip_vocab_size, yvocab_size=lbl_vocab_size, ckpt_path='ckpt/', epochs=epochs, emb_dim=emb_dim, model_name=model_name, num_layers=num_layers) sess = model.restore_last_session()
print("test set size=%s" % test_q.shape[0]) logdir = "./logs/" logdir += str(int(time.time())) if not FLAGS.logdir else FLAGS.logdir print("Checkpoint saved at %s" % logdir) model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len, yseq_len=yseq_len, xvocab_size=xvocab_size, yvocab_size=yvocab_size, emb_dim=emb_dim, num_layers=FLAGS.num_layers, learning_rate=FLAGS.learning_rate, use_lstm=FLAGS.use_lstm) eval_batch_gen = data_utils.rand_batch_gen(test_q, test_a, batch_size) train_batch_gen = data_utils.rand_batch_gen(train_q, train_a, batch_size) # create session for training gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=FLAGS.memory_usage_percentage / 100) session_conf = tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options) sess = tf.Session(config=session_conf) # init all variables # In[9]: # sess = model.restore_last_session() sess = train_seq2seq(model, train_batch_gen, eval_batch_gen,
import seq2seq_wrapper # load data from pickle and npy files metadata, idx_q, idx_a = data.load_data(PATH='datasets/may_reddit/') (trainX, trainY), (testX, testY), (validX, validY) = split_dataset(idx_q, idx_a) # parameters xseq_len = trainX.shape[-1] yseq_len = trainY.shape[-1] batch_size = 32 xvocab_size = len(metadata['idx2w']) yvocab_size = xvocab_size emb_dim = 1024 # In[7]: model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len, yseq_len=yseq_len, xvocab_size=xvocab_size, yvocab_size=yvocab_size, ckpt_path='ckpt/reddit/', emb_dim=emb_dim, num_layers=3) val_batch_gen = rand_batch_gen(validX, validY, 32) train_batch_gen = rand_batch_gen(trainX, trainY, batch_size) sess = model.restore_last_session() sess = model.train(train_batch_gen, val_batch_gen)
def eval_batch(eval_batch_gen, num_batches): losses, predict_loss = [], [] for i in range(num_batches): loss_v, dec_op_v, batchX, batchY = eval_step(eval_batch_gen) losses.append(loss_v) for j in range(len(dec_op_v)): real = batchX.T[j] predict = np.argmax(dec_op_v, axis=2)[j] predict_loss.append(all(real == predict)) return np.mean(losses), np.mean(predict_loss) if __name__ == '__main__': val_batch_gen = data_utils.rand_batch_gen(validX, validY, 16) train_eval_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 16) train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 128) with tf.Session() as sess: sess.run(tf.initialize_all_variables()) for i in range(100000): try: train_batch(train_batch_gen) if i % 1000 == 0: val_loss, val_predict = eval_batch(val_batch_gen, 16) train_loss, train_predict = eval_batch( train_eval_batch_gen, 16) print("val loss : {0}, val predict = {1}%".format( val_loss, val_predict * 100))
xseq_len = trainX.shape[-1] yseq_len = trainY.shape[-1] batch_size = 32 xvocab_size = len(metadata['idx2w']) yvocab_size = xvocab_size emb_dim = 1024 model = seq2seq_model.Seq2Seq(xseq_len=xseq_len, yseq_len=yseq_len, xvocab_size=xvocab_size, yvocab_size=yvocab_size, ckpt_path='ckpt/', emb_dim=emb_dim, num_layers=3) test_batch_gen = data_utils.rand_batch_gen(testX, testY, 256) sess = model.restore_last_session() input_ = test_batch_gen.__next__()[0] output = model.predict(sess, input_) replies = [] for ii, oi in zip(input_.T, output): q = data_utils.decode(sequence=ii, lookup=metadata['idx2w'], separator=' ') decoded = data_utils.decode(sequence=oi, lookup=metadata['idx2w'], separator=' ').split(' ') if decoded.count('unk') == 0: if decoded not in replies: print('q : [{0}]; a : [{1}]'.format(q, ' '.join(decoded)))
dec_op_v = np.array(dec_op_v).transpose([1,0,2]) return loss_v, dec_op_v, batchX, batchY def eval_batch(eval_batch_gen, num_batches): losses, predict_loss = [], [] for i in range(num_batches): loss_v, dec_op_v, batchX, batchY = eval_step(eval_batch_gen) losses.append(loss_v) for j in range(len(dec_op_v)): real = batchX.T[j] predict = np.argmax(dec_op_v, axis=2)[j] predict_loss.append(all(real == predict)) return np.mean(losses), np.mean(predict_loss) if __name__ == '__main__': val_batch_gen = data_utils.rand_batch_gen(validX, validY, 16) train_eval_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 16) train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 128) with tf.Session() as sess: sess.run(tf.initialize_all_variables()) for i in range(100000): try: train_batch(train_batch_gen) if i % 1000 == 0: val_loss, val_predict = eval_batch(val_batch_gen, 16) train_loss, train_predict = eval_batch(train_eval_batch_gen, 16) print("val loss : {0}, val predict = {1}%".format(val_loss, val_predict * 100)) print("train loss : {0}, train predict = {1}%".format(train_loss, train_predict * 100)) print
# parameters xseq_len = trainX.shape[-1] yseq_len = trainA.shape[-1] batch_size = 16 xvocab_size = len(metadata['idx2w']) yvocab_size = xvocab_size emb_dim = 1024 model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len, yseq_len=yseq_len, xvocab_size=xvocab_size, yvocab_size=yvocab_size, ckpt_path='ckpt/', emb_dim=emb_dim, num_layers=1, epochs=100 ) sess = model.restore_last_session() story_batch_gen = data_utils.rand_batch_gen(trainX_filter_10, trainA_filter_10, 1) input_ = story_batch_gen.__next__()[0] for i in range(5): output = model.predict(sess, input_) for ii, oi in zip(input_.T, output): q = data_utils.decode(sequence=ii, lookup=metadata['idx2w'], separator=' ') decoded = data_utils.decode(sequence=oi, lookup=metadata['idx2w'], separator=' ').split(' ') print('q : [{0}]; a : [{1}]'.format(q, ' '.join(decoded))) input_ = output.T
zseq_len=zseq_len, xvocab_size=xvocab_size, yvocab_size=yvocab_size, zvocab_size=zvocab_size, >>>>>>> 0d81aa006b893195d432ed2dfbbf7f930ec62226 ckpt_path='ckpt/', emb_dim=emb_dim, num_layers=1, epochs=5 ) <<<<<<< HEAD # In[21]: val_batch_gen = data_utils.rand_batch_gen(valid_data[:,1,:], valid_data[:,2,:], valid_data[:,3,:], 100) #test_batch_gen = data_utils.rand_batch_gen(small_testX, small_testY, small_testAx, 100) train_batch_gen = data_utils.rand_batch_gen(train_data[:,1,:][:40], train_data[:,2,:][:40], train_data[:,3,:][:40], batch_size) ======= # In[21]: val_batch_gen = data_utils.rand_batch_gen(valid_data[:,1,:], valid_data[:,2,:], valid_data[:,0,:], valid_data[:,3,:], 100) #test_batch_gen = data_utils.rand_batch_gen(small_testX, small_testY, small_testAx, 100) train_batch_gen = data_utils.rand_batch_gen(train_data[:,1,:], train_data[:,2,:], train_data[:,0,:], train_data[:,3,:], batch_size) >>>>>>> 0d81aa006b893195d432ed2dfbbf7f930ec62226 #train_batch_gen_story = data_utils.rand_batch_gen(trainX_filter_10, trainN_filter_10, trainAx_filter_10, 1) # In[ ]: #sess = model.restore_last_session() #sess = model.train(train_batch_gen, val_batch_gen, sess) if(False):