def train(data, model, params): sess = tf.Session() sess.run(tf.global_variables_initializer()) if params.data_opt: ckpt = tf.train.latest_checkpoint(params.tmp_dir) dl.optimistic_restore(sess, ckpt) saver = tf.train.Saver() dev_dialog, dev_label = data.get_dev() dev_feed_dict = { model.x_dialog: dev_dialog[:params.batch_size], model.y_pos: dev_pos[:params.batch_size], model.dropout_keep: 1.0 } print data.num_train max_iter = params.epoch_num * data.num_train / params.batch_size for i in range(max_iter): x_dialog, y_pos = data.get_train_batch() train_feed_dict = { model.x_dialog: x_dialog, model.y_pos: y_pos[:params.batch_size], model.dropout_keep: 0.5 } # print 'Size of Train Batch: %d, %d, %d' %(len(x_word), len(x_api), len(y_act)) train_loss, _ = sess.run([model.loss, model.train_step], feed_dict=train_feed_dict) if i % params.check_step == 0: dev_loss = sess.run(model.loss, feed_dict=dev_feed_dict) print('Step: %d/%d, train_loss: %.5f, dev_loss: %.5f' % (i, max_iter, train_loss, dev_loss)) if i % params.save_step == 0: saver.save(sess, os.path.join(params.tmp_dir, 'e2e_model.ckpt'), global_step=i) sess.close()
def train(data, model, params): print 'Train...' sess = tf.Session() dev_sess = tf.Session() train_batch_size = params.batch_size params.batch_size = data.num_dev with tf.variable_scope('dev'): dev_model = Pointer(params) params.batch_size = train_batch_size sess.run(tf.global_variables_initializer()) if params.restore: ckpt = tf.train.latest_checkpoint(params.tmp_dir) dl.optimistic_restore(sess, ckpt) saver = tf.train.Saver() dev_dialog, dev_label_id, dev_label_pos = data.get_dev() ## Set size of devset to batch_size # dev_dialog = dev_dialog[0:params.batch_size]+\ # dev_dialog[data.num_dev: data.num_dev+params.batch_size]+\ # dev_dialog[2*data.num_dev: 2*data.num_dev+params.batch_size]+\ # dev_dialog[3*data.num_dev: 3*data.num_dev+params.batch_size] # dev_label_id = dev_label_id[0:params.batch_size] + \ # dev_label_id[data.num_dev: data.num_dev + params.batch_size] + \ # dev_label_id[2 * data.num_dev: 2 * data.num_dev + params.batch_size] + \ # dev_label_id[3 * data.num_dev: 3 * data.num_dev + params.batch_size] # dev_label_pos = dev_label_pos[0:params.batch_size] + \ # dev_label_pos[data.num_dev: data.num_dev + params.batch_size] + \ # dev_label_pos[2 * data.num_dev: 2 * data.num_dev + params.batch_size] + \ # dev_label_pos[3 * data.num_dev: 3 * data.num_dev + params.batch_size] # print len(dev_dialog), len(dev_label_id), len(dev_label_pos) dev_feed_dict = { model.x_dialog: dev_dialog, model.y_id_in: dev_label_id, model.y_pos_out: dev_label_pos, model.dropout_keep: 1.0 } max_iter = params.epoch_num * data.num_train / (4 * params.batch_size) for i in range(max_iter): x_dialog, y_pos_in, y_pos_out = data.get_train_batch() train_feed_dict = { model.x_dialog: x_dialog, model.y_id_in: y_pos_in, model.y_pos_out: y_pos_out, model.dropout_keep: 0.5 } # print 'Size of Train Batch: %d, %d, %d' %(len(x_word), len(x_api), len(y_act)) train_loss, _ = sess.run([model.loss, model.train_step], feed_dict=train_feed_dict) if i % params.check_step == 0: ckpt = tf.train.latest_checkpoint(params.tmp_dir) dl.optimistic_restore(dev_sess, ckpt) dev_loss = sess.run(dev_model.loss, feed_dict=dev_feed_dict) print('Step: %d/%d, train_loss: %.5f, dev_loss: %.5f' % (i, max_iter, train_loss, dev_loss)) if i % params.save_step == 0: saver.save(sess, os.path.join(params.tmp_dir, 'ptr_model.ckpt'), global_step=i) sess.close() dev_sess.close()
def test(data, model, params): print 'Testing ...' sess = tf.Session() sess.run(tf.global_variables_initializer()) ckpt = tf.train.latest_checkpoint(params.tmp_dir) print ckpt dl.optimistic_restore(sess, ckpt) test_x_word, test_x_api, test_y_word_in, test_y_word_out = \ data.get_batch(data.test_input_id, data.test_api_num, data.test_output_id) raw_str = dl.flatten_2D(data.test_usr) f = open(os.path.join(params.tmp_dir, 'test_result.txt'), 'w') for i in range(data.num_test): x_raw_str = raw_str[i] x_word = [test_x_word[i]] x_api = [test_x_api[i]] y_word_in = [test_y_word_in[i]] y_word_out = [test_y_word_out[i]] # Run encoder just once state = sess.run(model.encoder_multi_cell.zero_state(1, tf.float32)) feed_dict = { model.x_word: x_word, model.encoder_initial_state: state, } encoder_last_state = sess.run(model.encoder_last_state, feed_dict) state = encoder_last_state # state = sess.run(model.decoder_multi_cell.zero_state(1, tf.float32)) # Run decoder answer = '' word = '<s>' for j in range(params.utc_length): x = np.zeros([1, 1]) x[0, 0] = data.convert(word, data.word2id) feed_dict = { model.x_api: x_api, model.y_word_in: x, # model.encoder_last_state: encoder_last_state, model.decoder_initial_state: state, } probs, state = sess.run([model.probs, model.decoder_last_state], feed_dict) p = probs[0] word = data.convert(np.argmax(p), data.id2word) if word == '</s>': break answer += word + ' ' show_str = ('%d\t%s\n\t%s\n' % (i + 1, x_raw_str, answer)) print(show_str) f.write('%s\n' % show_str) sess.close() f.close() print 'Testing ...'
def test_old(data, model, params): print 'Test...' sess = tf.Session() sess.run(tf.global_variables_initializer()) ckpt = tf.train.latest_checkpoint(params.tmp_dir) print ckpt dl.optimistic_restore(sess, ckpt) tst_dialog, tst_label_id, tst_label_pos = data.get_tst() f = open(os.path.join(params.tmp_dir, 'ptr_test_result.txt'), 'w') for i in range(4 * data.num_test): gt_start = tst_label_pos[i][0] gt_end = tst_label_pos[i][1] pred_range = np.zeros(2) x_dialog = tst_dialog[i] x_label_id = np.zeros(1) x_label_id[0] = tst_label_id[i][0] for j in range(params.gen_len): feed_dict = { model.x_dialog: [x_dialog], model.y_id_in: [x_label_id], model.y_pos_out: [tst_label_pos[0][j]], model.dropout_keep: 1.0 } probs = sess.run(model.probs, feed_dict) print probs p = probs[0] x_label_id = np.argmax(p) pred_range[j] = x_label_id slot = data.convert(tst_label_id[i][0], data.id2word) gt = '' for j in range(gt_start, gt_end + 1): # get ground truth string gt = gt + data.convert(x_dialog[0][j], data.id2word) + ' ' pred = '' for j in range(pred_range[0], pred_range[1] + 1): # get predication string pred = pred + data.convert(x_dialog[0][j], data.id2word) + ' ' answer = '%s\t%s\t%s' % (slot, gt, pred) show_str = ('%d\t%s\n\t%s\n' % (i + 1, x_raw_str, answer)) print(show_str) f.write('%s\n' % show_str) sess.close() f.close()
def train(data, model, params): sess = tf.Session() sess.run(tf.global_variables_initializer()) if params.restore: ckpt = tf.train.latest_checkpoint(params.tmp_dir) dl.optimistic_restore(sess, ckpt) saver = tf.train.Saver() x_word, x_api, y_word_in, y_word_out = data.get_batch( data.dev_input_id, data.dev_api_num, data.dev_output_id, len(data.dev_input_id)) dev_feed_dict = { model.x_word: x_word[:params.batch_size], model.x_api: x_api[:params.batch_size], model.y_word_in: y_word_in[:params.batch_size], model.y_word_out: y_word_out[:params.batch_size], model.y_len: [params.gen_length] * params.batch_size, model.dropout_keep: 1.0 } max_iter = params.epoch_num * data.num_train / params.batch_size for i in range(max_iter): x_word, x_api, y_word_in, y_word_out = data.get_train_batch() feed_dict = { model.x_word: x_word, model.x_api: x_api, model.y_word_in: y_word_in, model.y_word_out: y_word_out, model.y_len: [params.gen_length] * params.batch_size, model.dropout_keep: 0.5 } # outputs = sess.run(model.decoder_outputs, feed_dict=feed_dict) # print outputs[0].shape train_loss, _ = sess.run([model.cost, model.train_op], feed_dict=feed_dict) if i % params.check_step == 0: dev_loss = sess.run(model.cost, feed_dict=dev_feed_dict) print('Step: %d/%d, train_loss: %.5f, dev_loss: %.5f' % (i, max_iter, train_loss, dev_loss)) if i % params.save_step == 0: saver.save(sess, os.path.join(params.tmp_dir, 'e2e_model.ckpt'), global_step=i) sess.close()
def test(data, model, params): sess = tf.Session() sess.run(tf.global_variables_initializer()) ckpt = tf.train.latest_checkpoint(params.tmp_dir) dl.optimistic_restore(sess, ckpt) test_x_word, test_x_api, test_y_word_in, test_y_word_out = \ data.get_batch(data.test_input_id, data.test_api_num, data.test_output_id) for i in range(data.num_test): x_word = [test_x_word[i]] x_api = [test_x_api[i]] y_word_in = [test_y_word_in[i]] y_word_out = [test_y_word_out[i]] # Run encoder just once state = sess.run(model.encoder_multi_cell.zero_state(1, tf.float32)) feed_dict = { model.x_word: x_word, model.encoder_initial_state: state, } encoder_last_state = sess.run(model.encoder_last_state, feed_dict) state = sess.run(model.decoder_multi_cell.zero_state(1, tf.float32)) # Run decoder answer = '' word = '<s>' for j in range(params.utc_length): x = np.zeros([1, 1]) x[0, 0] = data.convert(word, data.word2id) feed_dict = { model.x_api: x_api, model.y_word_in: x, model.encoder_last_state: encoder_last_state, model.decoder_initial_state: state, } probs, state = sess.run([model.probs, model.decoder_last_state], feed_dict) p = probs[0] word = data.convert(np.argmax(p), data.id2word) # if word == '</s>': # break answer += word show_str = ('%d\t%s' % (i + 1, unicode(answer, encoding='utf-8'))) print(show_str) sess.close()
def test(data, model, params): sess = tf.Session() sess.run(tf.global_variables_initializer()) ckpt = tf.train.latest_checkpoint(params.tmp_dir) print ckpt dl.optimistic_restore(sess, ckpt) test_x_word, test_x_api, test_y_word_in, test_y_word_out = data.get_batch(data.test_input_id, data.test_api_num, data.test_output_id, len(data.test_input_id)) raw_str = dl.flatten_2D(data.test_usr) f = open(os.path.join(params.tmp_dir, 'test_result.txt'), 'w') for i in range(data.num_test): x_raw_str = raw_str[i] x_word = [test_x_word[i]] x_api = [test_x_api[i]] # Run encoder just once answer = '' word = '<s>' for j in range(params.utc_length): x = np.zeros([1, 1]) x[0, 0] = data.convert(word, data.word2id) feed_dict = { model.x_word: x_word, model.x_api: x_api, model.y_word_in: x, model.y_len: [params.gen_length], model.dropout_keep: 1.0, } probs = sess.run(model.probs, feed_dict) print probs p = probs[0] word = data.convert(np.argmax(p), data.id2word) if word == '</s>': break answer += word+' ' show_str = ('%d\t%s\n\t%s\n' % (i + 1, x_raw_str, answer)) print(show_str) f.write('%s\n' % show_str) sess.close() f.close()
def test(data, model, params): print 'test...' sess = tf.Session() sess.run(tf.global_variables_initializer()) ckpt = tf.train.latest_checkpoint(params.tmp_dir) dl.optimistic_restore(sess, ckpt) tst_dialog, tst_label_id, tst_label_pos = data.get_tst() num_data = len(tst_dialog) accuracy = 0 tst_feed_dict = { model.x_dialog: tst_dialog, model.y_id_in: tst_label_id, model.y_pos_out: tst_label_pos, model.dropout_keep: 1.0 } # print 'Size of Train Batch: %d, %d, %d' %(len(x_word), len(x_api), len(y_act)) tst_loss, tst_prob = sess.run([model.loss, model.probs], feed_dict=tst_feed_dict) tst_prob = tst_prob.reshape([num_data, params.gen_len, params.max_len]) f = open(os.path.join(params.tmp_dir, 'ptr_test_result.txt'), 'w') for i in range(num_data): y_pos_start = np.argmax(tst_prob[i][0]) y_pos_end = np.argmax(tst_prob[i][1]) pred = '' gt = '' slot = data.convert(int(tst_label_id[i][0]), data.id2word) for j in range(y_pos_start, y_pos_end + 1): pred = pred + data.convert(int(tst_dialog[i][j]), data.id2word) + ' ' for j in range(int(tst_label_pos[i][0]), int(tst_label_pos[i][1] + 1)): gt = gt + data.convert(int(tst_dialog[i][j]), data.id2word) + ' ' # print y_pos_start, y_pos_end, "\t", slot, gt, pred f.write('%s\t%d %d\t%s %s\n' % (slot, y_pos_start, y_pos_end, gt, pred)) if y_pos_start == int(tst_label_pos[i][0]) and y_pos_end == int( tst_label_pos[i][1]): accuracy += 1 accuracy = accuracy * 1.0 / len(tst_dialog) print 'Accuracy: %.3f' % accuracy f.close() sess.close()
def test_individual(data, model, params): print 'test...' sess = tf.Session() sess.run(tf.global_variables_initializer()) ckpt = tf.train.latest_checkpoint(params.tmp_dir) dl.optimistic_restore(sess, ckpt) tst_dialog, tst_label_id, tst_label_pos = data.get_tst() num_data = len(tst_dialog) accuracy = 0 for i in range(num_data): x_dialog = tst_dialog[i] y_id_in = tst_label_id[i] y_pos_out = tst_label_pos[i] tst_feed_dict = { model.x_dialog: [x_dialog], model.y_id_in: [y_id_in], model.y_pos_out: [y_pos_out], model.dropout_keep: 1.0 } # print 'Size of Train Batch: %d, %d, %d' %(len(x_word), len(x_api), len(y_act)) tst_loss, tst_prob = sess.run([model.loss, model.probs], feed_dict=tst_feed_dict) y_pos_start = np.argmax(tst_prob[0]) y_pos_end = np.argmax(tst_prob[1]) pred = '' gt = '' slot = data.convert(int(y_id_in[0]), data.id2word) for j in range(y_pos_start, y_pos_end + 1): pred = pred + data.convert(int(x_dialog[j]), data.id2word) + ' ' for j in range(int(y_pos_out[0]), int(y_pos_out[1] + 1)): gt = gt + data.convert(int(x_dialog[j]), data.id2word) + ' ' print tst_loss, y_pos_start, y_pos_end, "\t", slot, gt, pred if y_pos_start == int(y_pos_out[0]) and y_pos_end == int(y_pos_out[1]): accuracy += 1 accuracy = accuracy * 1.0 / len(tst_dialog) print 'Accuracy: %.3f' % accuracy sess.close()
def test(data, model, params): sess = tf.Session() sess.run(tf.global_variables_initializer()) ckpt = tf.train.latest_checkpoint(params.tmp_dir) print ckpt dl.optimistic_restore(sess, ckpt) f_true = open(os.path.join(params.tmp_dir, 'test_result_true.txt'), 'w') f_false = open(os.path.join(params.tmp_dir, 'test_result_false.txt'), 'w') f_cm = open(os.path.join(params.tmp_dir, 'test_confusion_matrix.txt'), 'w') error = 0 if params.data_opt: x_api = data.get_api_vector(data.dev_api_num) x_word = data.dev_input_id y_act = data.get_act_vector(data.dev_label) label = data.dev_label usr_utc_list = data.dev_usr sys_utc_list = data.dev_sys api_list = data.dev_api_num else: x_api = data.get_api_vector(data.test_api_num) x_word = data.test_input_id y_act = data.get_act_vector(data.test_label) label = data.test_label usr_utc_list = data.test_usr sys_utc_list = data.test_sys api_list = data.test_api_num feed_dict = { model.x_word: x_word, model.x_api: x_api, model.y_act: y_act, model.dropout_keep: 1.0 } print 'Size of %s Set: %d, %d, %d' % (['Test', 'Dev'][data_opt], len(x_word), len(x_api), len(y_act)) loss, prob = sess.run([model.loss, model.prob], feed_dict=feed_dict) usr_utc_list = dl.flatten_2D(usr_utc_list) sys_utc_list = dl.flatten_2D(sys_utc_list) pred_list = np.zeros(len(label)) true_list = np.asarray(label) pred_type_list = [] true_type_list = [] for i in range(len(label)): p = prob[i] act_id = np.argmax(p) string = '%s %s\t#%s\t#%s\t%d\n' % (data.id2act[act_id], data.id2act[ label[i]], usr_utc_list[i], sys_utc_list[i], api_list[i]) if act_id == label[i]: f_true.write(string) else: error += 1 f_false.write(string) if data.id2act[act_id] not in pred_type_list: pred_type_list.append(data.id2act[act_id]) if data.id2act[label[i]] not in true_type_list: true_type_list.append(data.id2act[label[i]]) pred_list[i] = act_id print 'Error rate: %.3f' % (error * 1.0 / len(label)) print 'Loss: %.3f' % loss cm = confusion_matrix(true_list, pred_list) # Save the confusion matrix f_cm.write('\t\t\t') for i in range(2, params.act_size + 1): f_cm.write('%5d\t' % i) f_cm.write('\n') for i in range(params.act_size - 1): f_cm.write('%10s\t' % data.id2act[i][:10]) for j in range(params.act_size - 1): f_cm.write('%5d\t' % cm[i, j]) f_cm.write('\n') f_true.close() f_false.close() f_cm.close() sess.close()
def test_on_trainset(data, model, params): sess = tf.Session() sess.run(tf.global_variables_initializer()) ckpt = tf.train.latest_checkpoint(params.tmp_dir) print ckpt dl.optimistic_restore(sess, ckpt) f_true = open(os.path.join(params.tmp_dir, 'train_result_true.txt'), 'w') f_false = open(os.path.join(params.tmp_dir, 'train_result_false.txt'), 'w') f_cm = open(os.path.join(params.tmp_dir, 'train_confusion_matrix.txt'), 'w') error = 0 train_api = data.get_api_vector(data.train_api_num) train_act = data.get_act_vector(data.train_label) train_feed_dict = { model.x_word: data.train_input_id, model.x_api: train_api, model.y_act: train_act, model.dropout_keep: 1.0 } print 'Size of Train Set:', len( data.train_input_id), len(train_api), len(train_act) prob = sess.run(model.prob, feed_dict=train_feed_dict) train_usr = dl.flatten_2D(data.train_usr) train_sys = dl.flatten_2D(data.train_sys) pred_list = np.zeros(params.train_size) true_list = np.asarray(data.train_label) pred_type_list = [] true_type_list = [] for i in range(params.train_size): p = prob[i] act_id = np.argmax(p) string = '%s %s\t#%s\t#%s\t%d\n' % ( data.id2act[act_id], data.id2act[data.train_label[i]], train_usr[i], train_sys[i], data.train_api_num[i]) if act_id == data.train_label[i]: f_true.write(string) else: error += 1 f_false.write(string) if data.id2act[act_id] not in pred_type_list: pred_type_list.append(data.id2act[act_id]) if data.id2act[data.train_label[i]] not in true_type_list: true_type_list.append(data.id2act[data.train_label[i]]) pred_list[i] = act_id # print error # print 'Pred type: ', pred_type_list # print 'True type: ', true_type_list # print pred_list print 'Error rate: %.3f' % (error * 1.0 / params.train_size) cm = confusion_matrix(true_list, pred_list) # Save the confusion matrix f_cm.write('\t\t\t') for i in range(2, params.act_size + 1): f_cm.write('%5d\t' % i) f_cm.write('\n') for i in range(params.act_size): f_cm.write('%10s\t' % data.id2act[i][:10]) for j in range(params.act_size): f_cm.write('%5d\t' % cm[i, j]) f_cm.write('\n') f_true.close() f_false.close() f_cm.close() sess.close()