def train(data, model, params):
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if params.data_opt:
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
dl.optimistic_restore(sess, ckpt)
saver = tf.train.Saver()
dev_dialog, dev_label = data.get_dev()
dev_feed_dict = {
model.x_dialog: dev_dialog[:params.batch_size],
model.y_pos: dev_pos[:params.batch_size],
model.dropout_keep: 1.0
}
print data.num_train
max_iter = params.epoch_num * data.num_train / params.batch_size
for i in range(max_iter):
x_dialog, y_pos = data.get_train_batch()
train_feed_dict = {
model.x_dialog: x_dialog,
model.y_pos: y_pos[:params.batch_size],
model.dropout_keep: 0.5
}
# print 'Size of Train Batch: %d, %d, %d' %(len(x_word), len(x_api), len(y_act))
train_loss, _ = sess.run([model.loss, model.train_step],
feed_dict=train_feed_dict)
if i % params.check_step == 0:
dev_loss = sess.run(model.loss, feed_dict=dev_feed_dict)
print('Step: %d/%d, train_loss: %.5f, dev_loss: %.5f' %
(i, max_iter, train_loss, dev_loss))
if i % params.save_step == 0:
saver.save(sess,
os.path.join(params.tmp_dir, 'e2e_model.ckpt'),
global_step=i)
sess.close()
def train(data, model, params):
print 'Train...'
sess = tf.Session()
dev_sess = tf.Session()
train_batch_size = params.batch_size
params.batch_size = data.num_dev
with tf.variable_scope('dev'):
dev_model = Pointer(params)
params.batch_size = train_batch_size
sess.run(tf.global_variables_initializer())
if params.restore:
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
dl.optimistic_restore(sess, ckpt)
saver = tf.train.Saver()
dev_dialog, dev_label_id, dev_label_pos = data.get_dev()
## Set size of devset to batch_size
# dev_dialog = dev_dialog[0:params.batch_size]+\
# dev_dialog[data.num_dev: data.num_dev+params.batch_size]+\
# dev_dialog[2*data.num_dev: 2*data.num_dev+params.batch_size]+\
# dev_dialog[3*data.num_dev: 3*data.num_dev+params.batch_size]
# dev_label_id = dev_label_id[0:params.batch_size] + \
# dev_label_id[data.num_dev: data.num_dev + params.batch_size] + \
# dev_label_id[2 * data.num_dev: 2 * data.num_dev + params.batch_size] + \
# dev_label_id[3 * data.num_dev: 3 * data.num_dev + params.batch_size]
# dev_label_pos = dev_label_pos[0:params.batch_size] + \
# dev_label_pos[data.num_dev: data.num_dev + params.batch_size] + \
# dev_label_pos[2 * data.num_dev: 2 * data.num_dev + params.batch_size] + \
# dev_label_pos[3 * data.num_dev: 3 * data.num_dev + params.batch_size]
# print len(dev_dialog), len(dev_label_id), len(dev_label_pos)
dev_feed_dict = {
model.x_dialog: dev_dialog,
model.y_id_in: dev_label_id,
model.y_pos_out: dev_label_pos,
model.dropout_keep: 1.0
}
max_iter = params.epoch_num * data.num_train / (4 * params.batch_size)
for i in range(max_iter):
x_dialog, y_pos_in, y_pos_out = data.get_train_batch()
train_feed_dict = {
model.x_dialog: x_dialog,
model.y_id_in: y_pos_in,
model.y_pos_out: y_pos_out,
model.dropout_keep: 0.5
}
# print 'Size of Train Batch: %d, %d, %d' %(len(x_word), len(x_api), len(y_act))
train_loss, _ = sess.run([model.loss, model.train_step],
feed_dict=train_feed_dict)
if i % params.check_step == 0:
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
dl.optimistic_restore(dev_sess, ckpt)
dev_loss = sess.run(dev_model.loss, feed_dict=dev_feed_dict)
print('Step: %d/%d, train_loss: %.5f, dev_loss: %.5f' %
(i, max_iter, train_loss, dev_loss))
if i % params.save_step == 0:
saver.save(sess,
os.path.join(params.tmp_dir, 'ptr_model.ckpt'),
global_step=i)
sess.close()
dev_sess.close()
def test(data, model, params):
print 'Testing ...'
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
print ckpt
dl.optimistic_restore(sess, ckpt)
test_x_word, test_x_api, test_y_word_in, test_y_word_out = \
data.get_batch(data.test_input_id, data.test_api_num, data.test_output_id)
raw_str = dl.flatten_2D(data.test_usr)
f = open(os.path.join(params.tmp_dir, 'test_result.txt'), 'w')
for i in range(data.num_test):
x_raw_str = raw_str[i]
x_word = [test_x_word[i]]
x_api = [test_x_api[i]]
y_word_in = [test_y_word_in[i]]
y_word_out = [test_y_word_out[i]]
# Run encoder just once
state = sess.run(model.encoder_multi_cell.zero_state(1, tf.float32))
feed_dict = {
model.x_word: x_word,
model.encoder_initial_state: state,
}
encoder_last_state = sess.run(model.encoder_last_state, feed_dict)
state = encoder_last_state
# state = sess.run(model.decoder_multi_cell.zero_state(1, tf.float32))
# Run decoder
answer = ''
word = '<s>'
for j in range(params.utc_length):
x = np.zeros([1, 1])
x[0, 0] = data.convert(word, data.word2id)
feed_dict = {
model.x_api:
x_api,
model.y_word_in:
x,
# model.encoder_last_state: encoder_last_state,
model.decoder_initial_state:
state,
}
probs, state = sess.run([model.probs, model.decoder_last_state],
feed_dict)
p = probs[0]
word = data.convert(np.argmax(p), data.id2word)
if word == '</s>':
break
answer += word + ' '
show_str = ('%d\t%s\n\t%s\n' % (i + 1, x_raw_str, answer))
print(show_str)
f.write('%s\n' % show_str)
sess.close()
f.close()
print 'Testing ...'
def test_old(data, model, params):
print 'Test...'
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
print ckpt
dl.optimistic_restore(sess, ckpt)
tst_dialog, tst_label_id, tst_label_pos = data.get_tst()
f = open(os.path.join(params.tmp_dir, 'ptr_test_result.txt'), 'w')
for i in range(4 * data.num_test):
gt_start = tst_label_pos[i][0]
gt_end = tst_label_pos[i][1]
pred_range = np.zeros(2)
x_dialog = tst_dialog[i]
x_label_id = np.zeros(1)
x_label_id[0] = tst_label_id[i][0]
for j in range(params.gen_len):
feed_dict = {
model.x_dialog: [x_dialog],
model.y_id_in: [x_label_id],
model.y_pos_out: [tst_label_pos[0][j]],
model.dropout_keep: 1.0
}
probs = sess.run(model.probs, feed_dict)
print probs
p = probs[0]
x_label_id = np.argmax(p)
pred_range[j] = x_label_id
slot = data.convert(tst_label_id[i][0], data.id2word)
gt = ''
for j in range(gt_start, gt_end + 1): # get ground truth string
gt = gt + data.convert(x_dialog[0][j], data.id2word) + ' '
pred = ''
for j in range(pred_range[0],
pred_range[1] + 1): # get predication string
pred = pred + data.convert(x_dialog[0][j], data.id2word) + ' '
answer = '%s\t%s\t%s' % (slot, gt, pred)
show_str = ('%d\t%s\n\t%s\n' % (i + 1, x_raw_str, answer))
print(show_str)
f.write('%s\n' % show_str)
sess.close()
f.close()
def train(data, model, params):
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if params.restore:
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
dl.optimistic_restore(sess, ckpt)
saver = tf.train.Saver()
x_word, x_api, y_word_in, y_word_out = data.get_batch(
data.dev_input_id, data.dev_api_num, data.dev_output_id,
len(data.dev_input_id))
dev_feed_dict = {
model.x_word: x_word[:params.batch_size],
model.x_api: x_api[:params.batch_size],
model.y_word_in: y_word_in[:params.batch_size],
model.y_word_out: y_word_out[:params.batch_size],
model.y_len: [params.gen_length] * params.batch_size,
model.dropout_keep: 1.0
}
max_iter = params.epoch_num * data.num_train / params.batch_size
for i in range(max_iter):
x_word, x_api, y_word_in, y_word_out = data.get_train_batch()
feed_dict = {
model.x_word: x_word,
model.x_api: x_api,
model.y_word_in: y_word_in,
model.y_word_out: y_word_out,
model.y_len: [params.gen_length] * params.batch_size,
model.dropout_keep: 0.5
}
# outputs = sess.run(model.decoder_outputs, feed_dict=feed_dict)
# print outputs[0].shape
train_loss, _ = sess.run([model.cost, model.train_op],
feed_dict=feed_dict)
if i % params.check_step == 0:
dev_loss = sess.run(model.cost, feed_dict=dev_feed_dict)
print('Step: %d/%d, train_loss: %.5f, dev_loss: %.5f' %
(i, max_iter, train_loss, dev_loss))
if i % params.save_step == 0:
saver.save(sess,
os.path.join(params.tmp_dir, 'e2e_model.ckpt'),
global_step=i)
sess.close()
def test(data, model, params):
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
dl.optimistic_restore(sess, ckpt)
test_x_word, test_x_api, test_y_word_in, test_y_word_out = \
data.get_batch(data.test_input_id, data.test_api_num, data.test_output_id)
for i in range(data.num_test):
x_word = [test_x_word[i]]
x_api = [test_x_api[i]]
y_word_in = [test_y_word_in[i]]
y_word_out = [test_y_word_out[i]]
# Run encoder just once
state = sess.run(model.encoder_multi_cell.zero_state(1, tf.float32))
feed_dict = {
model.x_word: x_word,
model.encoder_initial_state: state,
}
encoder_last_state = sess.run(model.encoder_last_state, feed_dict)
state = sess.run(model.decoder_multi_cell.zero_state(1, tf.float32))
# Run decoder
answer = ''
word = '<s>'
for j in range(params.utc_length):
x = np.zeros([1, 1])
x[0, 0] = data.convert(word, data.word2id)
feed_dict = {
model.x_api: x_api,
model.y_word_in: x,
model.encoder_last_state: encoder_last_state,
model.decoder_initial_state: state,
}
probs, state = sess.run([model.probs, model.decoder_last_state],
feed_dict)
p = probs[0]
word = data.convert(np.argmax(p), data.id2word)
# if word == '</s>':
# break
answer += word
show_str = ('%d\t%s' % (i + 1, unicode(answer, encoding='utf-8')))
print(show_str)
sess.close()
def test(data, model, params):
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
print ckpt
dl.optimistic_restore(sess, ckpt)
test_x_word, test_x_api, test_y_word_in, test_y_word_out = data.get_batch(data.test_input_id,
data.test_api_num,
data.test_output_id,
len(data.test_input_id))
raw_str = dl.flatten_2D(data.test_usr)
f = open(os.path.join(params.tmp_dir, 'test_result.txt'), 'w')
for i in range(data.num_test):
x_raw_str = raw_str[i]
x_word = [test_x_word[i]]
x_api = [test_x_api[i]]
# Run encoder just once
answer = ''
word = '<s>'
for j in range(params.utc_length):
x = np.zeros([1, 1])
x[0, 0] = data.convert(word, data.word2id)
feed_dict = {
model.x_word: x_word,
model.x_api: x_api,
model.y_word_in: x,
model.y_len: [params.gen_length],
model.dropout_keep: 1.0,
}
probs = sess.run(model.probs, feed_dict)
print probs
p = probs[0]
word = data.convert(np.argmax(p), data.id2word)
if word == '</s>':
break
answer += word+' '
show_str = ('%d\t%s\n\t%s\n' % (i + 1, x_raw_str, answer))
print(show_str)
f.write('%s\n' % show_str)
sess.close()
f.close()
def test(data, model, params):
print 'test...'
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
dl.optimistic_restore(sess, ckpt)
tst_dialog, tst_label_id, tst_label_pos = data.get_tst()
num_data = len(tst_dialog)
accuracy = 0
tst_feed_dict = {
model.x_dialog: tst_dialog,
model.y_id_in: tst_label_id,
model.y_pos_out: tst_label_pos,
model.dropout_keep: 1.0
}
# print 'Size of Train Batch: %d, %d, %d' %(len(x_word), len(x_api), len(y_act))
tst_loss, tst_prob = sess.run([model.loss, model.probs],
feed_dict=tst_feed_dict)
tst_prob = tst_prob.reshape([num_data, params.gen_len, params.max_len])
f = open(os.path.join(params.tmp_dir, 'ptr_test_result.txt'), 'w')
for i in range(num_data):
y_pos_start = np.argmax(tst_prob[i][0])
y_pos_end = np.argmax(tst_prob[i][1])
pred = ''
gt = ''
slot = data.convert(int(tst_label_id[i][0]), data.id2word)
for j in range(y_pos_start, y_pos_end + 1):
pred = pred + data.convert(int(tst_dialog[i][j]),
data.id2word) + ' '
for j in range(int(tst_label_pos[i][0]), int(tst_label_pos[i][1] + 1)):
gt = gt + data.convert(int(tst_dialog[i][j]), data.id2word) + ' '
# print y_pos_start, y_pos_end, "\t", slot, gt, pred
f.write('%s\t%d %d\t%s %s\n' %
(slot, y_pos_start, y_pos_end, gt, pred))
if y_pos_start == int(tst_label_pos[i][0]) and y_pos_end == int(
tst_label_pos[i][1]):
accuracy += 1
accuracy = accuracy * 1.0 / len(tst_dialog)
print 'Accuracy: %.3f' % accuracy
f.close()
sess.close()
def test_individual(data, model, params):
print 'test...'
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
dl.optimistic_restore(sess, ckpt)
tst_dialog, tst_label_id, tst_label_pos = data.get_tst()
num_data = len(tst_dialog)
accuracy = 0
for i in range(num_data):
x_dialog = tst_dialog[i]
y_id_in = tst_label_id[i]
y_pos_out = tst_label_pos[i]
tst_feed_dict = {
model.x_dialog: [x_dialog],
model.y_id_in: [y_id_in],
model.y_pos_out: [y_pos_out],
model.dropout_keep: 1.0
}
# print 'Size of Train Batch: %d, %d, %d' %(len(x_word), len(x_api), len(y_act))
tst_loss, tst_prob = sess.run([model.loss, model.probs],
feed_dict=tst_feed_dict)
y_pos_start = np.argmax(tst_prob[0])
y_pos_end = np.argmax(tst_prob[1])
pred = ''
gt = ''
slot = data.convert(int(y_id_in[0]), data.id2word)
for j in range(y_pos_start, y_pos_end + 1):
pred = pred + data.convert(int(x_dialog[j]), data.id2word) + ' '
for j in range(int(y_pos_out[0]), int(y_pos_out[1] + 1)):
gt = gt + data.convert(int(x_dialog[j]), data.id2word) + ' '
print tst_loss, y_pos_start, y_pos_end, "\t", slot, gt, pred
if y_pos_start == int(y_pos_out[0]) and y_pos_end == int(y_pos_out[1]):
accuracy += 1
accuracy = accuracy * 1.0 / len(tst_dialog)
print 'Accuracy: %.3f' % accuracy
sess.close()
def test(data, model, params):
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
print ckpt
dl.optimistic_restore(sess, ckpt)
f_true = open(os.path.join(params.tmp_dir, 'test_result_true.txt'), 'w')
f_false = open(os.path.join(params.tmp_dir, 'test_result_false.txt'), 'w')
f_cm = open(os.path.join(params.tmp_dir, 'test_confusion_matrix.txt'), 'w')
error = 0
if params.data_opt:
x_api = data.get_api_vector(data.dev_api_num)
x_word = data.dev_input_id
y_act = data.get_act_vector(data.dev_label)
label = data.dev_label
usr_utc_list = data.dev_usr
sys_utc_list = data.dev_sys
api_list = data.dev_api_num
else:
x_api = data.get_api_vector(data.test_api_num)
x_word = data.test_input_id
y_act = data.get_act_vector(data.test_label)
label = data.test_label
usr_utc_list = data.test_usr
sys_utc_list = data.test_sys
api_list = data.test_api_num
feed_dict = {
model.x_word: x_word,
model.x_api: x_api,
model.y_act: y_act,
model.dropout_keep: 1.0
}
print 'Size of %s Set: %d, %d, %d' % (['Test', 'Dev'][data_opt],
len(x_word), len(x_api), len(y_act))
loss, prob = sess.run([model.loss, model.prob], feed_dict=feed_dict)
usr_utc_list = dl.flatten_2D(usr_utc_list)
sys_utc_list = dl.flatten_2D(sys_utc_list)
pred_list = np.zeros(len(label))
true_list = np.asarray(label)
pred_type_list = []
true_type_list = []
for i in range(len(label)):
p = prob[i]
act_id = np.argmax(p)
string = '%s %s\t#%s\t#%s\t%d\n' % (data.id2act[act_id], data.id2act[
label[i]], usr_utc_list[i], sys_utc_list[i], api_list[i])
if act_id == label[i]:
f_true.write(string)
else:
error += 1
f_false.write(string)
if data.id2act[act_id] not in pred_type_list:
pred_type_list.append(data.id2act[act_id])
if data.id2act[label[i]] not in true_type_list:
true_type_list.append(data.id2act[label[i]])
pred_list[i] = act_id
print 'Error rate: %.3f' % (error * 1.0 / len(label))
print 'Loss: %.3f' % loss
cm = confusion_matrix(true_list, pred_list)
# Save the confusion matrix
f_cm.write('\t\t\t')
for i in range(2, params.act_size + 1):
f_cm.write('%5d\t' % i)
f_cm.write('\n')
for i in range(params.act_size - 1):
f_cm.write('%10s\t' % data.id2act[i][:10])
for j in range(params.act_size - 1):
f_cm.write('%5d\t' % cm[i, j])
f_cm.write('\n')
f_true.close()
f_false.close()
f_cm.close()
sess.close()
def test_on_trainset(data, model, params):
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.latest_checkpoint(params.tmp_dir)
print ckpt
dl.optimistic_restore(sess, ckpt)
f_true = open(os.path.join(params.tmp_dir, 'train_result_true.txt'), 'w')
f_false = open(os.path.join(params.tmp_dir, 'train_result_false.txt'), 'w')
f_cm = open(os.path.join(params.tmp_dir, 'train_confusion_matrix.txt'),
'w')
error = 0
train_api = data.get_api_vector(data.train_api_num)
train_act = data.get_act_vector(data.train_label)
train_feed_dict = {
model.x_word: data.train_input_id,
model.x_api: train_api,
model.y_act: train_act,
model.dropout_keep: 1.0
}
print 'Size of Train Set:', len(
data.train_input_id), len(train_api), len(train_act)
prob = sess.run(model.prob, feed_dict=train_feed_dict)
train_usr = dl.flatten_2D(data.train_usr)
train_sys = dl.flatten_2D(data.train_sys)
pred_list = np.zeros(params.train_size)
true_list = np.asarray(data.train_label)
pred_type_list = []
true_type_list = []
for i in range(params.train_size):
p = prob[i]
act_id = np.argmax(p)
string = '%s %s\t#%s\t#%s\t%d\n' % (
data.id2act[act_id], data.id2act[data.train_label[i]],
train_usr[i], train_sys[i], data.train_api_num[i])
if act_id == data.train_label[i]:
f_true.write(string)
else:
error += 1
f_false.write(string)
if data.id2act[act_id] not in pred_type_list:
pred_type_list.append(data.id2act[act_id])
if data.id2act[data.train_label[i]] not in true_type_list:
true_type_list.append(data.id2act[data.train_label[i]])
pred_list[i] = act_id
# print error
# print 'Pred type: ', pred_type_list
# print 'True type: ', true_type_list
# print pred_list
print 'Error rate: %.3f' % (error * 1.0 / params.train_size)
cm = confusion_matrix(true_list, pred_list)
# Save the confusion matrix
f_cm.write('\t\t\t')
for i in range(2, params.act_size + 1):
f_cm.write('%5d\t' % i)
f_cm.write('\n')
for i in range(params.act_size):
f_cm.write('%10s\t' % data.id2act[i][:10])
for j in range(params.act_size):
f_cm.write('%5d\t' % cm[i, j])
f_cm.write('\n')
f_true.close()
f_false.close()
f_cm.close()
sess.close()
