def getModel():
# load data from pickle and npy files
metadata, idx_q, idx_a = data.load_data(PATH='datasets/cornell_corpus/')
(trainX, trainY), (testX, testY), (validX, validY) = data_utils.split_dataset(idx_q, idx_a)
train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 32)
#print len(trainX)
test_batch_gen = data_utils.rand_batch_gen(testX, testY, 256)
input_ = test_batch_gen.next()[0]
xseq_len = 25
yseq_len = 25
batch_size = 16
xvocab_size = len(metadata['idx2w'])
yvocab_size = xvocab_size
emb_dim = 1024
model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
ckpt_path='ckpt/cornell_corpus/',
emb_dim=emb_dim,
num_layers=3
)
sess = model.restore_last_session()
output = model.predict(sess, input_[0:25])
#print(output)
return model, sess
def __init__(self, train=False):
# load data from pickle and npy files
self.metadata, idx_q, idx_a = data.load_data(PATH='datasets/twitter/')
(trainX, trainY), (testX, testY), (validX,
validY) = data_utils.split_dataset(
idx_q, idx_a) # parameters
xseq_len = trainX.shape[-1]
yseq_len = trainY.shape[-1]
batch_size = 16
xvocab_size = len(self.metadata['idx2w'])
yvocab_size = xvocab_size
emb_dim = 1024
importlib.reload(seq2seq_wrapper)
self.model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
ckpt_path='ckpt/twitter/',
emb_dim=emb_dim,
num_layers=3)
if train:
val_batch_gen = data_utils.rand_batch_gen(validX, validY, 32)
train_batch_gen = data_utils.rand_batch_gen(
trainX, trainY, batch_size)
sess = self.model.train(train_batch_gen, val_batch_gen)
self.sess = self.model.restore_last_session()
def get_output(question, sess, w2idx, model, metadata):
import data_utils
from datasets.facebook2 import data
# get output for input phrase
idx_q, idx_a = data.process_input(question, w2idx)
gen = data_utils.rand_batch_gen(idx_q, idx_a, 1)
input_ = gen.__next__()[0]
output = model.predict(sess, input_)
# return ouput phrase
for ii, oi in zip(input_.T, output):
q = data_utils.decode(sequence=ii,
lookup=metadata['idx2w'],
separator=' ')
decoded = data_utils.decode(sequence=oi,
lookup=metadata['idx2w'],
separator=' ').split(' ')
return ' '.join(decoded)
def get_response(self, text, metadata, sess):
questions = [ text.lower() ]
questions = [ filter_line(line, EN_WHITELIST) for line in questions ]
answers = questions
qlines, alines = filter_data(questions, answers)
qtokenized = [ [w.strip() for w in wordlist.split(' ') if w] for wordlist in qlines ]
atokenized = [ [w.strip() for w in wordlist.split(' ') if w] for wordlist in alines ]
w2idx = pickle.load(open("datasets/cornell_corpus/w2idx.pkl","rb"))
idx_q, idx_a = zero_pad(qtokenized, atokenized, w2idx)
query = data_utils.rand_batch_gen(idx_q, idx_a, 1)
input_q = query.__next__()[0]
output = self.predict(sess, input_q)
#print(input_q.shape)
#replies = []
for ii, oi in zip(input_q.T, output):
q = data_utils.decode(sequence=ii, lookup=metadata['idx2w'], separator=' ')
decoded = data_utils.decode(sequence=oi, lookup=metadata['idx2w'], separator=' ').split(' ')
'''
if decoded.count('unk') == 0:
if decoded not in replies:
print('q : [{0}]; a : [{1}]'.format(q, ' '.join(decoded)))
replies.append(decoded)
'''
return ' '.join(decoded)
import importlib
importlib.reload(seq2seq_wrapper)
# In[5]:
model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
ckpt_path='ckpt/twitter/',
emb_dim=emb_dim,
num_layers=3)
# In[6]:
val_batch_gen = data_utils.rand_batch_gen(validX, validY, 256)
test_batch_gen = data_utils.rand_batch_gen(testX, testY, 256)
train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, batch_size)
# In[9]:
sess = model.train(train_batch_gen, val_batch_gen)
# In[7]:
sess = model.restore_last_session()
# In[10]:
input_ = test_batch_gen.__next__()[0]
output = model.predict(sess, input_)
xseq_len = trainX.shape[-1]
yseq_len = trainY.shape[-1]
batch_size = 32
xvocab_size = len(metadata['idx2w'])
yvocab_size = xvocab_size
emb_dim = 1024
import seq2seq_wrapper
# In[7]:
model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
ckpt_path='ckpt/cornell_corpus/',
emb_dim=emb_dim,
num_layers=3
)
# In[8]:
val_batch_gen = data_utils.rand_batch_gen(validX, validY, 32)
train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, batch_size)
# In[9]:
#sess = model.restore_last_session()
sess = model.train(train_batch_gen, val_batch_gen)
importlib.reload(seq2seq_wrapper)
# In[47]:
model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
ckpt_path='ckpt/',
emb_dim=emb_dim,
num_layers=1,
epochs=500)
# In[21]:
val_batch_gen = data_utils.rand_batch_gen(small_validX, small_validY, 100)
test_batch_gen = data_utils.rand_batch_gen(small_testX, small_testY, 100)
train_batch_gen = data_utils.rand_batch_gen(trainX_filter_10, trainA_filter_10,
batch_size)
train_batch_gen_story = data_utils.rand_batch_gen(trainX_filter_10,
trainA_filter_10, 1)
# In[ ]:
sess = model.restore_last_session()
sess = model.train(train_batch_gen, val_batch_gen, sess)
# In[25]:
sess = model.restore_last_session()
# In[43]:
"train_ip size: {} train_lbl size: {} test_ip size: {}, test_lbl size:{}".
format(train_ip.shape, train_lbl.shape, test_ip.shape, test_lbl.shape))
ip_seq_len = train_ip.shape[-1]
lbl_seq_len = train_lbl.shape[-1]
ip_vocab_size = len(block2idx)
del block2idx
lbl_vocab_size = ip_vocab_size
model_name = os.path.splitext(fname)[0] + "segmentseq_model"
train_ip_count = train_ip.shape[0]
print("Train count: {}".format(train_ip_count))
print("Test count: {}".format(test_ip.shape[0]))
train_batch_gen = rand_batch_gen(train_ip, train_lbl, batch_size)
val_batch_gen = rand_batch_gen(test_ip, test_lbl, batch_size)
model = SegSeq2SegSeq(train_size=train_ip_count,
batch_size=batch_size,
xseq_len=ip_seq_len,
yseq_len=lbl_seq_len,
xvocab_size=ip_vocab_size,
yvocab_size=lbl_vocab_size,
ckpt_path='ckpt/',
epochs=epochs,
emb_dim=emb_dim,
model_name=model_name,
num_layers=num_layers)
sess = model.restore_last_session()
print("test set size=%s" % test_q.shape[0])
logdir = "./logs/"
logdir += str(int(time.time())) if not FLAGS.logdir else FLAGS.logdir
print("Checkpoint saved at %s" % logdir)
model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
emb_dim=emb_dim,
num_layers=FLAGS.num_layers,
learning_rate=FLAGS.learning_rate,
use_lstm=FLAGS.use_lstm)
eval_batch_gen = data_utils.rand_batch_gen(test_q, test_a, batch_size)
train_batch_gen = data_utils.rand_batch_gen(train_q, train_a, batch_size)
# create session for training
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.memory_usage_percentage / 100)
session_conf = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
sess = tf.Session(config=session_conf)
# init all variables
# In[9]:
# sess = model.restore_last_session()
sess = train_seq2seq(model,
train_batch_gen,
eval_batch_gen,
import seq2seq_wrapper
# load data from pickle and npy files
metadata, idx_q, idx_a = data.load_data(PATH='datasets/may_reddit/')
(trainX, trainY), (testX, testY), (validX,
validY) = split_dataset(idx_q, idx_a)
# parameters
xseq_len = trainX.shape[-1]
yseq_len = trainY.shape[-1]
batch_size = 32
xvocab_size = len(metadata['idx2w'])
yvocab_size = xvocab_size
emb_dim = 1024
# In[7]:
model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
ckpt_path='ckpt/reddit/',
emb_dim=emb_dim,
num_layers=3)
val_batch_gen = rand_batch_gen(validX, validY, 32)
train_batch_gen = rand_batch_gen(trainX, trainY, batch_size)
sess = model.restore_last_session()
sess = model.train(train_batch_gen, val_batch_gen)
def eval_batch(eval_batch_gen, num_batches):
losses, predict_loss = [], []
for i in range(num_batches):
loss_v, dec_op_v, batchX, batchY = eval_step(eval_batch_gen)
losses.append(loss_v)
for j in range(len(dec_op_v)):
real = batchX.T[j]
predict = np.argmax(dec_op_v, axis=2)[j]
predict_loss.append(all(real == predict))
return np.mean(losses), np.mean(predict_loss)
if __name__ == '__main__':
val_batch_gen = data_utils.rand_batch_gen(validX, validY, 16)
train_eval_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 16)
train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 128)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(100000):
try:
train_batch(train_batch_gen)
if i % 1000 == 0:
val_loss, val_predict = eval_batch(val_batch_gen, 16)
train_loss, train_predict = eval_batch(
train_eval_batch_gen, 16)
print("val loss : {0}, val predict = {1}%".format(
val_loss, val_predict * 100))
xseq_len = trainX.shape[-1]
yseq_len = trainY.shape[-1]
batch_size = 32
xvocab_size = len(metadata['idx2w'])
yvocab_size = xvocab_size
emb_dim = 1024
model = seq2seq_model.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
ckpt_path='ckpt/',
emb_dim=emb_dim,
num_layers=3)
test_batch_gen = data_utils.rand_batch_gen(testX, testY, 256)
sess = model.restore_last_session()
input_ = test_batch_gen.__next__()[0]
output = model.predict(sess, input_)
replies = []
for ii, oi in zip(input_.T, output):
q = data_utils.decode(sequence=ii, lookup=metadata['idx2w'], separator=' ')
decoded = data_utils.decode(sequence=oi,
lookup=metadata['idx2w'],
separator=' ').split(' ')
if decoded.count('unk') == 0:
if decoded not in replies:
print('q : [{0}]; a : [{1}]'.format(q, ' '.join(decoded)))
dec_op_v = np.array(dec_op_v).transpose([1,0,2])
return loss_v, dec_op_v, batchX, batchY
def eval_batch(eval_batch_gen, num_batches):
losses, predict_loss = [], []
for i in range(num_batches):
loss_v, dec_op_v, batchX, batchY = eval_step(eval_batch_gen)
losses.append(loss_v)
for j in range(len(dec_op_v)):
real = batchX.T[j]
predict = np.argmax(dec_op_v, axis=2)[j]
predict_loss.append(all(real == predict))
return np.mean(losses), np.mean(predict_loss)
if __name__ == '__main__':
val_batch_gen = data_utils.rand_batch_gen(validX, validY, 16)
train_eval_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 16)
train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 128)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(100000):
try:
train_batch(train_batch_gen)
if i % 1000 == 0:
val_loss, val_predict = eval_batch(val_batch_gen, 16)
train_loss, train_predict = eval_batch(train_eval_batch_gen, 16)
print("val loss : {0}, val predict = {1}%".format(val_loss, val_predict * 100))
print("train loss : {0}, train predict = {1}%".format(train_loss, train_predict * 100))
print
# parameters
xseq_len = trainX.shape[-1]
yseq_len = trainA.shape[-1]
batch_size = 16
xvocab_size = len(metadata['idx2w'])
yvocab_size = xvocab_size
emb_dim = 1024
model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
ckpt_path='ckpt/',
emb_dim=emb_dim,
num_layers=1,
epochs=100
)
sess = model.restore_last_session()
story_batch_gen = data_utils.rand_batch_gen(trainX_filter_10, trainA_filter_10, 1)
input_ = story_batch_gen.__next__()[0]
for i in range(5):
output = model.predict(sess, input_)
for ii, oi in zip(input_.T, output):
q = data_utils.decode(sequence=ii, lookup=metadata['idx2w'], separator=' ')
decoded = data_utils.decode(sequence=oi, lookup=metadata['idx2w'], separator=' ').split(' ')
print('q : [{0}]; a : [{1}]'.format(q, ' '.join(decoded)))
input_ = output.T
zseq_len=zseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
zvocab_size=zvocab_size,
>>>>>>> 0d81aa006b893195d432ed2dfbbf7f930ec62226
ckpt_path='ckpt/',
emb_dim=emb_dim,
num_layers=1,
epochs=5
)
<<<<<<< HEAD
# In[21]:
val_batch_gen = data_utils.rand_batch_gen(valid_data[:,1,:], valid_data[:,2,:], valid_data[:,3,:], 100)
#test_batch_gen = data_utils.rand_batch_gen(small_testX, small_testY, small_testAx, 100)
train_batch_gen = data_utils.rand_batch_gen(train_data[:,1,:][:40], train_data[:,2,:][:40], train_data[:,3,:][:40], batch_size)
=======
# In[21]:
val_batch_gen = data_utils.rand_batch_gen(valid_data[:,1,:], valid_data[:,2,:], valid_data[:,0,:], valid_data[:,3,:], 100)
#test_batch_gen = data_utils.rand_batch_gen(small_testX, small_testY, small_testAx, 100)
train_batch_gen = data_utils.rand_batch_gen(train_data[:,1,:], train_data[:,2,:], train_data[:,0,:], train_data[:,3,:], batch_size)
>>>>>>> 0d81aa006b893195d432ed2dfbbf7f930ec62226
#train_batch_gen_story = data_utils.rand_batch_gen(trainX_filter_10, trainN_filter_10, trainAx_filter_10, 1)
# In[ ]:
#sess = model.restore_last_session()
#sess = model.train(train_batch_gen, val_batch_gen, sess)
if(False):
