def test(config): with open(config.word_emb_file, "r") as fh: word_mat = np.array(json.load(fh), dtype=np.float32) with open(config.char_emb_file, "r") as fh: char_mat = np.array(json.load(fh), dtype=np.float32) with open(config.test_eval_file, "r") as fh: eval_file = json.load(fh) with open(config.test_meta, "r") as fh: meta = json.load(fh) total = meta["num_batches"] print("Loading model...") test_batch = get_batch_dataset(config.test_record_file, get_record_parser( config, is_test=True), config, is_test=True).make_one_shot_iterator() model = Model(config, test_batch, word_mat, char_mat, trainable=False) sess_config = tf.ConfigProto(allow_soft_placement=True) sess_config.gpu_options.allow_growth = True with tf.Session(config=sess_config) as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint(config.save_dir)) sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool))) losses = [] answer_dict = {} select_right = [] for step in tqdm(range(1, total + 1)): qa_id, loss, yp1, yp2 , y1, y2, is_select_p, is_select= sess.run( [model.qa_id, model.loss, model.yp1, model.yp2, model.y1, model.y2, model.is_select_p, model.is_select]) y1 = np.argmax(y1, axis=-1) y2 = np.argmax(y2, axis=-1) sp = np.argmax(is_select_p, axis=-1) s = np.argmax(is_select, axis=-1) sp = [ n+i*config.passage_num for i,n in enumerate(sp.tolist()) ] s = [ m+i*config.passage_num for i,m in enumerate(s.tolist()) ] select_right.append(len(set(s).intersection(set(sp)))) answer_dict_, _ = convert_tokens( eval_file, [qa_id[n] for n in sp], [yp1[n] for n in sp], [yp2[n] for n in sp], [y1[n] for n in sp], [y2[n] for n in sp], sp, s) answer_dict.update(answer_dict_) losses.append(loss) loss = np.mean(losses) select_accu = sum(select_right)/ (len(select_right)*(config.batch_size/config.passage_num)) write_prediction(eval_file, answer_dict, 'answer_for_evl.json', config) metrics = evaluate(eval_file, answer_dict, filter=False) metrics['Selection Accuracy'] = select_accu print("Exact Match: {}, F1: {}, selection accuracy: {}".format( metrics['exact_match'], metrics['f1'], metrics['Selection Accuracy']))
def train(config): with open(config.word_emb_file, "r") as fh: word_mat = np.array(json.load(fh), dtype=np.float32) with open(config.char_emb_file, "r") as fh: char_mat = np.array(json.load(fh), dtype=np.float32) with open(config.train_eval_file, "r") as fh: train_eval_file = json.load(fh) with open(config.dev_eval_file, "r") as fh: dev_eval_file = json.load(fh) with open(config.dev_meta, "r") as fh: meta = json.load(fh) dev_total = meta["total"] print("Building model...") parser = get_record_parser(config) graph = tf.Graph() with graph.as_default() as g: train_dataset = get_batch_dataset(config.train_record_file, parser, config) dev_dataset = get_dataset(config.dev_record_file, parser, config) handle = tf.placeholder(tf.string, shape=[]) iterator = tf.data.Iterator.from_string_handle( handle, train_dataset.output_types, train_dataset.output_shapes) train_iterator = train_dataset.make_one_shot_iterator() dev_iterator = dev_dataset.make_one_shot_iterator() model = Model(config, iterator, word_mat, char_mat, graph = g) sess_config = tf.ConfigProto(allow_soft_placement=True) sess_config.gpu_options.allow_growth = True loss_save = 100.0 patience = 0 best_f1 = 0. best_em = 0. with tf.Session(config=sess_config) as sess: writer = tf.summary.FileWriter(config.log_dir) sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() train_handle = sess.run(train_iterator.string_handle()) dev_handle = sess.run(dev_iterator.string_handle()) if os.path.exists(os.path.join(config.save_dir, "checkpoint")): saver.restore(sess, tf.train.latest_checkpoint(config.save_dir)) global_step = max(sess.run(model.global_step), 1) for _ in tqdm(range(global_step, config.num_steps + 1)): global_step = sess.run(model.global_step) + 1 loss, train_op = sess.run([model.loss, model.train_op], feed_dict={ handle: train_handle, model.dropout: config.dropout}) if global_step % config.period == 0: loss_sum = tf.Summary(value=[tf.Summary.Value( tag="model/loss", simple_value=loss), ]) writer.add_summary(loss_sum, global_step) if global_step % config.checkpoint == 0: _, summ = evaluate_batch( model, config.val_num_batches, train_eval_file, sess, "train", handle, train_handle) for s in summ: writer.add_summary(s, global_step) metrics, summ = evaluate_batch( model, dev_total // config.batch_size + 1, dev_eval_file, sess, "dev", handle, dev_handle) dev_f1 = metrics["f1"] dev_em = metrics["exact_match"] if dev_f1 < best_f1 and dev_em < best_em: patience += 1 if patience > config.early_stop: break else: patience = 0 best_em = max(best_em, dev_em) best_f1 = max(best_f1, dev_f1) for s in summ: writer.add_summary(s, global_step) writer.flush() filename = os.path.join( config.save_dir, "model_{}.ckpt".format(global_step)) saver.save(sess, filename)
def train(config): with open(config.word_emb_file, "r") as fh: word_mat = np.array(json.load(fh), dtype=np.float32) with open(config.char_emb_file, "r") as fh: char_mat = np.array(json.load(fh), dtype=np.float32) with open(config.train_eval_file, "r") as fh: train_eval_file = json.load(fh) with open(config.dev_eval_file, "r") as fh: dev_eval_file = json.load(fh) with open(config.dev_meta, "r") as fh: meta = json.load(fh) dev_total = meta["total"] print("Building model...") parser = get_record_parser(config) train_dataset = get_batch_dataset(config.train_record_file, parser, config) dev_dataset = get_dataset(config.dev_record_file, parser, config) handle = tf.placeholder(tf.string, shape=[]) iterator = tf.data.Iterator.from_string_handle( handle, train_dataset.output_types, train_dataset.output_shapes) train_iterator = train_dataset.make_one_shot_iterator() dev_iterator = dev_dataset.make_one_shot_iterator() model = Model(config, iterator, word_mat, char_mat) sess_config = tf.ConfigProto(allow_soft_placement=True) sess_config.gpu_options.allow_growth = True loss_save = 100.0 patience = 0 lr = config.init_lr with tf.Session(config=sess_config) as sess: writer = tf.summary.FileWriter(config.log_dir) sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() train_handle = sess.run(train_iterator.string_handle()) dev_handle = sess.run(dev_iterator.string_handle()) sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool))) sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32))) for _ in tqdm(range(1, config.num_steps + 1)): global_step = sess.run(model.global_step) + 1 loss, train_op = sess.run([model.loss, model.train_op], feed_dict={ handle: train_handle}) if global_step % config.period == 0: loss_sum = tf.Summary(value=[tf.Summary.Value( tag="model/loss", simple_value=loss), ]) writer.add_summary(loss_sum, global_step) if global_step % config.checkpoint == 0: sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool))) _, summ = evaluate_batch( model, config.val_num_batches, train_eval_file, sess, "train", handle, train_handle) for s in summ: writer.add_summary(s, global_step) metrics, summ = evaluate_batch( model, dev_total // config.batch_size + 1, dev_eval_file, sess, "dev", handle, dev_handle) sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool))) dev_loss = metrics["loss"] if dev_loss < loss_save: loss_save = dev_loss patience = 0 else: patience += 1 if patience >= config.patience: lr /= 2.0 loss_save = dev_loss patience = 0 sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32))) for s in summ: writer.add_summary(s, global_step) writer.flush() filename = os.path.join( config.save_dir, "model_{}.ckpt".format(global_step)) saver.save(sess, filename)
def train(config): with open(config.word_emb_file, "r") as fh: word_mat = np.array(json.load(fh), dtype=np.float32) with open(config.char_emb_file, "r") as fh: char_mat = np.array(json.load(fh), dtype=np.float32) with open(config.train_eval_file, "r") as fh: train_eval_file = json.load(fh) with open(config.dev_eval_file, "r") as fh: dev_eval_file = json.load(fh) with open(config.dev_meta, "r") as fh: meta = json.load(fh) ''' Iterator: Represents the state of iterating through a Dataset. from_string_handle(): https://www.tensorflow.org/api_docs/python/tf/data/Iterator This method allows you to define a "feedable" iterator where you can choose between concrete iterators by feeding a value in a tf.Session.run call. In that case, string_handle would a tf.placeholder, and you would feed it with the value of tf.data.Iterator.string_handle in each step. make_one_shot_iterator():Creates an Iterator for enumerating the elements of this dataset. The returned iterator will be initialized automatically. A "one-shot" iterator does not currently support re-initialization. ''' dev_total = meta["total"] print("get recorded feature parser...") parser = get_record_parser(config) graph = tf.Graph() with graph.as_default() as g: print("get dataset batch iterator...") train_dataset = get_batch_dataset(config.train_record_file, parser, config) train_iterator = train_dataset.make_one_shot_iterator() dev_dataset = get_dataset(config.dev_record_file, parser, config) dev_iterator = dev_dataset.make_one_shot_iterator() handle = tf.placeholder(tf.string, shape=[]) iterator = tf.data.Iterator.from_string_handle( handle, train_dataset.output_types, train_dataset.output_shapes) print("Building model...") model = Model(config, iterator, word_mat, char_mat, graph=g) sess_config = tf.ConfigProto(allow_soft_placement=True) sess_config.gpu_options.allow_growth = True patience = 0 best_em = 0.0 best_f1 = 0.0 best_acc = 0.0 with tf.Session(config=sess_config) as sess: sess.run(tf.global_variables_initializer()) train_handle = sess.run(train_iterator.string_handle()) dev_handle = sess.run(dev_iterator.string_handle()) saver = tf.train.Saver(max_to_keep=100) if os.path.exists(os.path.join(config.save_dir, "checkpoint")): saver.restore(sess, tf.train.latest_checkpoint(config.save_dir)) writer = tf.summary.FileWriter(config.log_dir) global_step = max(sess.run(model.global_step), 1) for _ in tqdm(range(global_step, config.num_steps + 1)): global_step = sess.run(model.global_step) + 1 # train on one batch loss, train_op = sess.run([model.loss, model.train_op], feed_dict={ handle: train_handle, model.dropout: config.dropout }) # DEBUG # temp = sess.run(model.debug_ops, feed_dict={handle: train_handle, model.dropout: config.dropout}) # for t in temp: # # print(t) # print(t.shape) # sys.exit(0) # save batch loss if global_step % config.period == 0: loss_sum = tf.Summary(value=[ tf.Summary.Value(tag="model/loss", simple_value=loss), ]) writer.add_summary(loss_sum, global_step) if global_step % config.checkpoint == 0: # evaluate on train print('Evaluating on Train...') num_bt = config.val_num_batches # _, summ = evaluate_batch(model, num_bt, train_eval_file, sess, "train", handle, train_handle) _, summ = evaluate_batch_cand(model, num_bt, train_eval_file, sess, "train", handle, train_handle) # write to summary for s in summ: writer.add_summary(s, global_step) # evaluate on dev print('Evaluating on Dev...') num_bt = dev_total // config.batch_size + 1 # metrics, summ = evaluate_batch(model, num_bt, dev_eval_file, sess, "dev", handle, dev_handle) metrics, summ = evaluate_batch_cand( model, num_bt, dev_eval_file, sess, "dev", handle, dev_handle) # write to summary for s in summ: writer.add_summary(s, global_step) writer.flush() # save checkpoint model #filename = os.path.join(config.save_dir, "model_{}.ckpt".format(global_step)) #saver.save(sess, filename) # early stop # dev_f1 = metrics["f1"] # dev_em = metrics["exact_match"] dev_acc = metrics["acc"] if dev_acc < best_acc: patience += 1 if patience > config.early_stop: print( '>>>>>> WARNING !!! <<<<<<< Early_stop reached!!!' ) # save best model else: patience = 0 best_acc = dev_acc #filename = os.path.join(config.save_dir, "model_{}.best".format(global_step)) filename = os.path.join( config.save_dir, "model_{}.ckpt".format(global_step)) saver.save(sess, filename)