def test(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
graph = tf.Graph()
print("Loading model...")
with graph.as_default() as g:
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config,
test_batch,
word_mat,
char_mat,
trainable=False,
graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
if config.decay < 1.0:
sess.run(model.assign_vars)
losses = []
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
loss = np.mean(losses)
#with open(config.answer_file, "w") as fh:
# json.dump(remapped_dict, fh)
'''
metrics = evaluate(eval_file, answer_dict)
print("Exact Match: {}, F1: {}".format(
metrics['exact_match'], metrics['f1']))
'''
with open(config.answer_csv, 'w') as f:
print('dumping ans file to : %s' % str(config.answer_csv))
s = csv.writer(f, delimiter=',', lineterminator='\n')
for i in sorted(remapped_dict):
s.writerow([remapped_dict[i]])
def test(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False)
graph_handler = GraphHandler(config, model)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
#saver = tf.train.Saver()
graph_handler.initialize(sess)
#saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
losses = []
answer_dict = {}
remapped_dict = {}
ensember_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
start_logits, stop_logits, qa_id, loss, yp1, yp2 = sess.run([
model.start_logits, model.stop_logits, model.qa_id, model.loss,
model.yp1, model.yp2
])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
start_logits.tolist()
stop_logits.tolist()
for id, start, stop in zip(qa_id, start_logits, stop_logits):
ensember_dict[str(id)] = {'yp1': start, 'yp2': stop}
loss = np.mean(losses)
metrics = evaluate(eval_file, answer_dict)
answer_path = config.answer_file + "_" + str(config.load_step)
with open(answer_path, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {}".format(metrics['exact_match'],
metrics['f1']))
ensember_dict['loss'] = loss
ensember_dict['exact_math'] = metrics['exact_match']
ensember_dict['f1'] = metrics['f1']
file_name = config.model_name + '_' + config.run_id + '.pklz'
save_path = os.path.join(config.result_path, file_name)
with gzip.open(save_path, 'wb', compresslevel=3) as fh:
pickle.dump(ensember_dict, fh)
def gparam(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file, get_record_parser(
config, is_test=True), config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False)
all_weights = {v.name: v for v in tf.trainable_variables()}
total_size = 0
for v_name in sorted(list(all_weights)):
v = all_weights[v_name]
print("%s\tshape %s", v.name[:-2].ljust(80),
str(v.shape).ljust(20))
v_size = np.prod(np.array(v.shape.as_list())).tolist()
total_size += v_size
print("Total trainable variables size: %d", total_size)
def test(config, dataset="test"):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
if dataset == "test":
test_eval_file = config.test_eval_file
test_meta = config.test_meta
test_record_file = config.test_record_file
elif dataset == "addsent":
print('HELLO')
test_eval_file = config.addsent_eval_file
test_meta = config.addsent_meta
test_record_file = config.addsent_record_file
elif dataset == "addonesent":
test_eval_file = config.addonesent_eval_file
test_meta = config.addonesent_meta
test_record_file = config.addonesent_record_file
with open(test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
losses = []
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
loss = np.mean(losses)
metrics = evaluate(eval_file, answer_dict)
with open(config.answer_file, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {}".format(metrics['exact_match'],
metrics['f1']))
def test(config):
gpu_options = tf.GPUOptions(visible_device_list="2")
sess_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
sess_config.gpu_options.allow_growth = True
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False)
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
losses = []
answer_dict = {}
remapped_dict = {}
# tqdm
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_, outlier = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
print("\n", loss)
if (loss > 50):
for i, j, k in zip(qa_id.tolist(), yp1.tolist(), yp2.tolist()):
print(answer_dict[str(i)], j, k)
#print("IDs: {} Losses: {} Yp1: {} Yp2: {}".format(qa_id.tolist(),\
# loss.tolist(), yp1.tolist(), yp2.tolist()))
loss = np.mean(losses)
# evaluate with answer_dict, but in evaluate-v1.1.py, evaluate with remapped_dict
# since only that is saved. Both dict are a little bit different, check evaluate-v1.1.py
metrics = evaluate(eval_file, answer_dict)
with open(config.answer_file, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {} Rouge-l-f: {} Rouge-l-p: {} Rouge-l-r: {}".format(\
metrics['exact_match'], metrics['f1'], metrics['rouge-l-f'], metrics['rouge-l-p'],\
metrics['rouge-l-r']))
def predict(config):
prepro_predict(config)
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.bpe_emb_file, "r") as fh:
bpe_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.pos_emb_file, "r") as fh:
pos_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.predict_eval_file, "r") as fh:
predict_eval_file = json.load(fh)
with open(config.predict_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.predict_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config,
test_batch,
word_mat,
char_mat,
bpe_mat,
pos_mat,
trainable=False)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
# TODO: add restoring from best model or from model name
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
print('Restoring from: {}'.format(
tf.train.latest_checkpoint(config.save_dir)))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
predict_eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
path_to_save_answer = config.predict_file + '_ans'
with open(path_to_save_answer, "w") as fh:
json.dump(remapped_dict, fh)
print("Answer dumped: {}".format(path_to_save_answer))
def test(config):
os.environ["CUDA_VISIBLE_DEVICES"] = config.choose_gpu
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
graph = tf.Graph()
print("Loading model...")
with graph.as_default() as g:
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = QANet(config,
test_batch,
word_mat,
char_mat,
trainable=False,
graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
sess_config.gpu_options.per_process_gpu_memory_fraction = config.gpu_memory_fraction
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
if config.decay < 1.0:
sess.run(model.assign_vars)
losses = []
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
loss = np.mean(losses)
metrics = evaluate(eval_file, answer_dict)
with open(config.answer_file, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {}".format(metrics['exact_match'],
metrics['f1']))
def test(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["num_batches"]
print("Loading model...")
test_batch = get_batch_dataset(config.test_record_file, get_record_parser(
config, is_test=True), config, is_test=True).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
losses = []
answer_dict = {}
select_right = []
for step in tqdm(range(1, total + 1)):
qa_id, loss, yp1, yp2 , y1, y2, is_select_p, is_select= sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2, model.y1, model.y2, model.is_select_p, model.is_select])
y1 = np.argmax(y1, axis=-1)
y2 = np.argmax(y2, axis=-1)
sp = np.argmax(is_select_p, axis=-1)
s = np.argmax(is_select, axis=-1)
sp = [ n+i*config.passage_num for i,n in enumerate(sp.tolist()) ]
s = [ m+i*config.passage_num for i,m in enumerate(s.tolist()) ]
select_right.append(len(set(s).intersection(set(sp))))
answer_dict_, _ = convert_tokens(
eval_file, [qa_id[n] for n in sp], [yp1[n] for n in sp], [yp2[n] for n in sp], [y1[n] for n in sp], [y2[n] for n in sp], sp, s)
answer_dict.update(answer_dict_)
losses.append(loss)
loss = np.mean(losses)
select_accu = sum(select_right)/ (len(select_right)*(config.batch_size/config.passage_num))
write_prediction(eval_file, answer_dict, 'answer_for_evl.json', config)
metrics = evaluate(eval_file, answer_dict, filter=False)
metrics['Selection Accuracy'] = select_accu
print("Exact Match: {}, F1: {}, selection accuracy: {}".format(
metrics['exact_match'], metrics['f1'], metrics['Selection Accuracy']))
def process(config):
print("Building model...")
parser = get_record_parser(config)
test_dataset = get_dataset(config.test_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(handle,
test_dataset.output_types,
test_dataset.output_shapes)
test_iterator = test_dataset.make_one_shot_iterator()
model = xDeepFMModel(config, iterator)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
save_f1 = 0.1
patience = 0
lr = config.init_lr
test_total = load_data_total(config.test_meta)
with tf.Session(config=sess_config) as sess:
param_num = sum(
[np.prod(sess.run(tf.shape(v))) for v in model.all_params])
print('There are {} parameters in the model'.format(param_num))
writer = tf.summary.FileWriter(config.event_dir)
sess.run(tf.global_variables_initializer())
summary_writer = tf.summary.FileWriter('./log/log', sess.graph)
saver = tf.train.Saver(max_to_keep=10000)
test_handle = sess.run(test_iterator.string_handle())
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
## 加载训练好的模型
if os.path.exists(config.save_dir + "/checkpoint"):
print("Restoring Variables from Checkpoint.", config.save_dir)
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
test_metrics = evaluate_batch(model, test_total,
test_total // config.batch_size + 1,
sess, handle, test_handle)
print(test_metrics)
def test(config):
with open(config.word_emb_file, "r", encoding="utf-8") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r", encoding="utf-8") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r", encoding="utf-8") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, trainable=False)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
answer_dict = {}
remapped_dict = {}
for _ in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp = sess.run([model.qa_id, model.loss, model.yp])
remapped_dict_, answer_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
f = open(config.answer_file, "w", encoding="utf-8")
for key in answer_dict:
f.write(str(key) + "\t" + answer_dict[key] + "\n")
# 处理不合法（被丢弃）的测试样本
# 直接选第一个答案
ans_list = list(answer_dict.keys())
with open(config.test_file, "r", encoding="utf-8") as fh:
for line in fh:
sample = json.loads(line)
if sample["query_id"] not in ans_list:
f.write(
str(sample["query_id"]) + "\t" +
sample['alternatives'].split("|")[0] + "\n")
f.close()
def test(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
graph = tf.Graph()
print("Loading model...")
with graph.as_default() as g:
test_batch = get_dataset(config.test_record_file, get_record_parser(
config, is_test=True), config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False, graph = g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
if config.decay < 1.0:
sess.run(model.assign_vars)
losses = []
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
loss = np.mean(losses)
metrics = evaluate(eval_file, answer_dict)
with open(config.answer_file, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {}".format(
metrics['exact_match'], metrics['f1']))
def test(config):
with codecs.open(config.word_emb_file, "r", encoding="utf-8") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with codecs.open(config.test_eval_file, "r", encoding="utf-8") as fh:
eval_file = json.load(fh)
with codecs.open(config.test_meta, "r", encoding="utf-8") as fh:
meta = json.load(fh)
# with open(config.char_emb_file, "r") as fh:
# char_mat = np.array(json.load(fh), dtype=np.float32)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
# model = Model(config, test_batch, word_mat,char_mat, trainable=False)
model = Model(config, test_batch, word_mat, False)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp = sess.run([model.qa_id, model.loss, model.yp])
remapped_dict_, answer_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
with codecs.open(config.answer_file, "w", encoding="utf-8") as fh:
for key in answer_dict:
fh.write(str(key) + "\t" + answer_dict[key] + "\n")
print("test")
def test(config):
word_mat = np.array(load_from_gs(config.word_emb_file), dtype=np.float32)
char_mat = np.array(load_from_gs(config.char_emb_file), dtype=np.float32)
eval_file = load_from_gs(config.test_eval_file)
meta = load_from_gs(config.test_meta)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model_cls = get_model(config.model)
model = model_cls(config, test_batch, word_mat, char_mat, trainable=False)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
losses = []
answer_dict = {}
remapped_dict = {}
for step in xrange(total // config.batch_size + 1):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
loss = np.mean(losses)
metrics = evaluate(eval_file, answer_dict)
write_to_gs(config.answer_file, remapped_dict)
print("Exact Match: {}, F1: {}".format(metrics['exact_match'],
metrics['f1']))
def train(config):
with open(config.word_emb_file, "rb") as fh:
print(fh)
word_mat = np.array(pickle.load(fh),
dtype=np.float32) #加载对应的单词以及词向量,加载字符以及对应的词向量
with open(config.char_emb_file, "rb") as fh:
char_mat = np.array(pickle.load(fh), dtype=np.float32) #加载字符向量对应的
# with open(config.train_eval_file, "r") as fh:
# train_eval_file = json.load(fh) #"context": context, "spans": spans, "answers": answer_texts, "uuid": qa["id"]
# #span 对应的段落单词字符开始结束位置 context:全文内容 ,answer 实际词汇
# with open(config.dev_eval_file, "r") as fh:
# dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
# dev_total = meta["total"]
#
print("Building model1111111...")
parser = get_record_parser(config) #加载对应的训练数据特征,与存储特征数据的build_features对应，
train_dataset = get_batch_dataset(config.train_record_file, parser,
config) #返回已经batch好的训练数据集，用iterator进行迭代
dev_dataset = get_dataset(config.dev_record_file, parser,
config) #对这个数据进行同样的处理
# #把不同的数据dataset的数据feed进模型，首先需要建立iterator handle，既iterator placeholder
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes) #
train_iterator = train_dataset.make_one_shot_iterator(
) #首先提取数据，先对数据构建迭代器iterator
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config,
iterator,
word_mat=word_mat,
char_mat=char_mat,
filter_sizes=[3, 4, 5],
embedding_size=300,
num_filters=128)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7)
config_sess = tf.ConfigProto(gpu_options=gpu_options)
loss_save = 0
patience = 0
lr = config.init_lr
with tf.Session(config=config_sess) as sess:
fig = plt.figure()
ax = fig.add_subplot(211)
ax2 = fig.add_subplot(212)
plt.grid(True)
plt.ion()
writer = tf.summary.FileWriter(config.log_dir)
ckpt = tf.train.get_checkpoint_state(config.save_dir)
if ckpt is not None:
print(ckpt.model_checkpoint_path)
model.saver.restore(sess, ckpt.model_checkpoint_path)
else:
sess.run(tf.global_variables_initializer())
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
for _ in tqdm(range(1, config.num_steps + 1)):
try:
global_step = sess.run(model.global_step) + 1
logits, loss, train_op, accuracy = sess.run(
[
model.predictions, model.loss, model.train_op,
model.accuracy
],
feed_dict={handle: train_handle})
# print(logits)
# print(np.array(real_y),"hhhhhhhhhhhhhhhhhhhhhhhh")
if global_step % 10 == 0:
ax.scatter(global_step, loss, c='b', marker='.')
plt.pause(0.001)
print("the loss is:", loss)
print('the accuracy is', accuracy)
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss)
])
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
# sess.run(tf.assign(model.is_train,tf.constant(False, dtype=tf.bool)))
dev_loss = 0
for k in range(500):
dev_loss += sess.run([model.accuracy],
feed_dict={handle: dev_handle})[0]
dev_loss = dev_loss / 500
ax2.scatter(global_step, dev_loss, c='b', marker='.')
plt.pause(0.001)
# _, summ = evaluate_batch(
# model1111111, config.val_num_batches, train_eval_file, sess, "train", handle, train_handle)
# for s in summ:
# writer.add_summary(s, global_step)
# metrics, summ = evaluate_batch(
# model1111111, dev_total // config.batch_size + 1, dev_eval_file, sess, "dev", handle, dev_handle)
# sess.run(tf.assign(model.is_train,tf.constant(True, dtype=tf.bool)))
# dev_loss = metrics["loss"]
if dev_loss > loss_save:
print(
dev_loss, loss_save,
'222222222222222222222222222222222222222222222222222222222222222'
)
loss_file = os.path.join(config.save_dir,
'loss_file.txt')
with open(loss_file, 'w') as fi:
fi.write(str(dev_loss))
fi.write('\t')
fi.write(str(loss_save))
fi.write('\n')
loss_save = dev_loss
filename = os.path.join(
config.save_dir,
"model_{}.ckpt".format(global_step))
model.saver.save(sess, filename)
figure_path = os.path.join(config.save_dir, 'img.png')
print(figure_path,
"ttttttttttttttttttttttttttttttttttt")
plt.savefig(figure_path)
# sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
# for s in summ:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss_dev",
simple_value=dev_loss)
])
writer.add_summary(loss_sum, global_step)
writer.flush()
# filename = os.path.join(config.save_dir, "model_{}.ckpt".format(global_step))
# model.saver.save(sess, filename)
except Exception as e:
print(e)
def test(config):
questionid = [] #对应的顺序questionid
new_alternatives = [] #更改之后的答案顺序
new_answer_index = [] #真正答案的下标
with open(config.test_file, "r") as fh:
lines = fh.readlines()
for line in tqdm(lines):
new_alternative = []
source = json.loads(line) # 加载训练数据集
questionid.append(source['query_id'])
alter_answer_origanl = [
ele.strip()
for ele in source['alternatives'].strip().split("|")
]
min_word = 'dfsdfdsafdsaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
for temp_alter in alter_answer_origanl:
if len(temp_alter) < len(min_word):
min_word = temp_alter
new_alternative.append(min_word)
for temp_alter in alter_answer_origanl:
if temp_alter.replace(min_word,
'') == '不' or temp_alter.replace(
min_word, '') == '没':
new_alternative.append(temp_alter)
break
for temp_alter in alter_answer_origanl:
if temp_alter not in new_alternative:
new_alternative.append(temp_alter)
new_alternative = np.array(new_alternative)
new_alternatives.append(new_alternative)
with open(config.word2idx_file, "r") as fh:
word2idx_dict = json.load(fh)
index_word = dict(zip(word2idx_dict.values(), word2idx_dict.keys()))
with open(config.word_emb_file, "rb") as fh:
word_mat = np.array(pickle.load(fh),
dtype=np.float32) # 加载对应的单词以及词向量,加载字符以及对应的词向量
with open(config.char_emb_file, "rb") as fh:
char_mat = np.array(pickle.load(fh), dtype=np.float32) # 加载字符向量对应的
# with open(config.test_eval_file, "r") as fh:
# eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model1111111...")
# parser = get_record_parser(config,is_test=True)#加载对应的训练数据特征,与存储特征数据的build_features对应，
# test_batch = get_dataset(config.test_record_file, parser, config)
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config,
test_batch,
word_mat=word_mat,
char_mat=char_mat,
filter_sizes=[3, 4, 5],
trainable=False,
embedding_size=300,
num_filters=128)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
# sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool)))
for step in tqdm(range(total // config.batch_size + 1)):
ture_pre = sess.run(model.predictions) #qa_id 预测对应的类别下标
new_answer_index.extend(ture_pre) #不改变顺序
print(len(new_answer_index))
with open(config.answer_file, "w") as fh:
normal = 0
for ele_k in range(len(questionid)):
alter1 = new_alternatives[ele_k]
if new_answer_index[ele_k] >= len(alter1):
ture_answer = alter1[0]
else:
normal += 1
ture_answer = alter1[new_answer_index[ele_k]]
ture_queid = str(questionid[ele_k])
fh.write(ture_queid)
fh.write('\t')
fh.write(ture_answer)
fh.write('\n')
print('the normal quiestion is:', normal)
def train(config):
gpu_options = tf.GPUOptions(visible_device_list="2")
sess_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
sess_config.gpu_options.allow_growth = True
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
train_dataset = get_batch_dataset(config.train_record_file, parser, config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(handle,
train_dataset.output_types,
train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat)
loss_save = 100.0
patience = 0
lr = config.init_lr
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
#saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool)))
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
print("Started training")
for _ in tqdm(range(1, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={handle: train_handle})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
sess.run(
tf.assign(model.is_train, tf.constant(False,
dtype=tf.bool)))
_, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle)
for s in summ:
writer.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1, dev_eval_file,
sess, "dev", handle, dev_handle)
sess.run(
tf.assign(model.is_train, tf.constant(True,
dtype=tf.bool)))
dev_loss = metrics["loss"]
if dev_loss < loss_save:
loss_save = dev_loss
patience = 0
else:
patience += 1
if patience >= config.patience:
lr /= 2.0
loss_save = dev_loss
patience = 0
sess.run(tf.assign(model.lr, tf.constant(lr,
dtype=tf.float32)))
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(config.save_dir,
"model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
train_dataset = get_batch_dataset(config.train_record_file, parser, config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
#创建一个迭代器
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(handle,
train_dataset.output_types,
train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat)
graph_handler = GraphHandler(
config, model
) # controls all tensors and variables in the graph, including loading /saving
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
lr = config.init_lr
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
#saver = tf.train.Saver()
graph_handler.initialize(sess)
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool)))
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
for _ in tqdm(range(1, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={handle: train_handle})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
graph_handler.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
sess.run(
tf.assign(model.is_train, tf.constant(False,
dtype=tf.bool)))
_, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle)
for s in summ:
graph_handler.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1, dev_eval_file,
sess, "dev", handle, dev_handle)
sess.run(
tf.assign(model.is_train, tf.constant(True,
dtype=tf.bool)))
dev_loss = metrics["loss"]
#特殊的学习率变化规律
if dev_loss < loss_save:
loss_save = dev_loss
patience = 0
else:
patience += 1
#学习率的变化
if patience >= config.patience:
lr /= 2.0
loss_save = dev_loss
patience = 0
sess.run(tf.assign(model.lr, tf.constant(lr,
dtype=tf.float32)))
graph_handler.add_summaries(summ, global_step)
graph_handler.writer.flush()
filename = os.path.join(
config.save_dir,
"{}_{}.ckpt".format(config.model_name, global_step))
graph_handler.save(sess, filename)
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
'''
Iterator: Represents the state of iterating through a Dataset.
from_string_handle(): https://www.tensorflow.org/api_docs/python/tf/data/Iterator
This method allows you to define a "feedable" iterator where you can choose between concrete iterators
by feeding a value in a tf.Session.run call. In that case, string_handle would a tf.placeholder,
and you would feed it with the value of tf.data.Iterator.string_handle in each step.
make_one_shot_iterator():Creates an Iterator for enumerating the elements of this dataset.
The returned iterator will be initialized automatically.
A "one-shot" iterator does not currently support re-initialization.
'''
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
train_dataset = get_batch_dataset(config.train_record_file, parser,
config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat, graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
saver = tf.train.Saver()
if os.path.exists(os.path.join(config.save_dir, "checkpoint")):
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir))
writer = tf.summary.FileWriter(config.log_dir)
global_step = max(sess.run(model.global_step), 1)
for _ in tqdm(range(global_step, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
'''
enc, start_logits, end_logits, logits1, logits2, outer, yp1, yp2, losses, losses2, loss \
= sess.run(model.debug_ops,feed_dict={handle: train_handle, model.dropout: config.dropout})
print(outer)
print(yp1)
print(yp2)
for i in [enc, start_logits, end_logits, logits1, logits2, outer, yp1, yp2, losses, losses2, loss]:
print(i.shape)
'''
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
_, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle)
for s in summ:
writer.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1,
dev_eval_file, sess, "dev", handle, dev_handle)
dev_loss = metrics["loss"]
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
train_dataset = get_batch_dataset(config.train_record_file, parser,
config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model_dict = {
'Raw': Raw_Model,
'TPE': TPE_Model,
'T5': T5_Model,
'T5_Nob': T5_Nob_Model,
'Soft_T5': Soft_T5_Model,
'Soft_T5_Nob': Soft_T5_NoB_Model
}
if config.model not in model_dict:
print('wrong %s model name' % (config.model))
exit(0)
model = model_dict[config.model](config,
iterator,
word_mat,
char_mat,
graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
best_f1 = 0.
best_em = 0.
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.event_log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
total_params()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(
config.save_dir, "checkpoint")): #restore the model...
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir))
global_step = max(sess.run(model.global_step),
1) #to restore the global_step
for _ in tqdm(range(global_step, config.num_steps +
1)): #the max optimize steps
global_step = sess.run(model.global_step) + 1
if config.model in ['Soft_T5', 'Soft_Multi_T5']:
loss, train_op, soft_t5_alpha = sess.run([
model.loss,
model.train_op,
model.soft_t5_alpha,
],
feed_dict={
handle:
train_handle,
model.dropout:
config.dropout
})
else:
loss, train_op = sess.run([
model.loss,
model.train_op,
],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
writer.add_summary(loss_sum, global_step)
if config.model in ['Soft_T5', 'Soft_Multi_T5']:
for hidx in range(config.num_heads):
hidx_val = soft_t5_alpha[
hidx] * config.fixed_c_maxlen
writer.add_summary(
tf.Summary(value=[
tf.Summary.Value(tag="model/alpha_" +
str(hidx),
simple_value=hidx_val),
]), global_step)
if global_step % config.checkpoint == 0:
_, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle, config)
for s in summ:
writer.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1,
dev_eval_file, sess, "dev", handle, dev_handle, config)
dev_f1 = metrics["f1"] #early stop strategy ...
dev_em = metrics["exact_match"]
if dev_f1 < best_f1 and dev_em < best_em:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_em = max(best_em, dev_em)
best_f1 = max(best_f1, dev_f1)
#we save the best performance on evaluation...
filename = os.path.join(
config.save_dir,
"model_{}.ckpt".format(global_step))
saver.save(sess, filename)
if config.model in ['T5', 'T5_TPE', 'T5_Nob']:
c_t5_bias_mat, q_t5_bias_mat, model_c_t5_bias_mat = sess.run(
[
model.c_t5_bias_mat, model.q_t5_bias_mat,
model.model_c_t5_bias_mat
])
pkl_param = {
'c_t5_bias_mat': c_t5_bias_mat,
'q_t5_bias_mat': q_t5_bias_mat,
'model_c_t5_bias_mat': model_c_t5_bias_mat
}
ret_filename = os.path.join(
config.save_dir, 'att_ret.p')
pickle.dump(pkl_param, open(ret_filename, 'wb'))
if config.model in ['Soft_T5', 'Soft_T5_TPE']:
pkl_param = sess.run(
[[
model.soft_t5_alpha, model.soft_t5_beta,
model.c_t5_bias, model.q_t5_bias,
model.model_c_t5_bias_list
]],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
ret_filename = os.path.join(
config.save_dir, 'att_ret.p')
pickle.dump(pkl_param, open(ret_filename, 'wb'))
if config.model in ['Soft_T5_Nob']:
pkl_param = sess.run(
[[
model.c_soft_t5_rd_bucket_mat,
model.q_soft_t5_rd_bucket_mat,
model.model_soft_t5_rd_bucket_list,
model.c_t5_bias, model.q_t5_bias,
model.model_c_t5_bias_list
]],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
ret_filename = os.path.join(
config.save_dir, 'att_ret.p')
pickle.dump(pkl_param, open(ret_filename, 'wb'))
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
def train(config):
gpu_options = sess_config = None
if config.use_cudnn:
gpu_options = tf.GPUOptions(visible_device_list=config.gpu_id)
sess_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
sess_config.gpu_options.allow_growth = True
else:
gpu_options = tf.GPUOptions(visible_device_list="")
sess_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
train_dataset = get_batch_dataset(config.train_record_file, parser, config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(handle,
train_dataset.output_types,
train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat)
loss_save = 100.0
patience = 0
lr = config.init_lr
with tf.Session(config=sess_config) as sess:
sess = tf_debug.LocalCLIDebugWrapperSession(sess)
writer = tf.summary.FileWriter(config.log_dir,
graph=tf.get_default_graph())
writer.add_graph(sess.graph)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=config.max_checkpoint_to_keep,
save_relative_paths=True)
#saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
if config.restore_checkpoint:
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir_temp))
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool)))
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
print("Started training")
for _ in tqdm(range(1, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
if config.with_passage_ranking:
if global_step % config.period == 0:
summary, loss_esp, loss_pr, loss_ee, train_op_ee = sess.run(
[
model.loss, model.pr_loss, model.e_loss,
model.train_op_ee
],
feed_dict={handle: train_handle})
loss_sum1 = tf.Summary(value=[
tf.Summary.Value(tag="model/loss_esp",
simple_value=loss_esp),
])
loss_sum2 = tf.Summary(value=[
tf.Summary.Value(tag="model/loss_pr",
simple_value=loss_pr),
])
loss_sum3 = tf.Summary(value=[
tf.Summary.Value(tag="model/loss_ee",
simple_value=loss_ee),
])
writer.add_summary(loss_sum2, global_step)
writer.add_summary(loss_sum3, global_step)
writer.add_summary(loss_sum1, global_step)
writer.add_summary(summary, global_step)
else:
loss_esp, loss_pr, loss_ee, train_op_ee = sess.run(
[
model.loss, model.pr_loss, model.e_loss,
model.train_op_ee
],
feed_dict={handle: train_handle})
else:
if global_step % config.period == 0:
summary, loss_esp, train_op = sess.run(
[model.merged, model.loss, model.train_op],
feed_dict={handle: train_handle})
loss_sum1 = tf.Summary(value=[
tf.Summary.Value(tag="model/loss_esp",
simple_value=loss_esp),
])
writer.add_summary(loss_sum1, global_step)
writer.add_summary(summary, global_step)
else:
loss_esp, train_op = sess.run(
[model.loss, model.train_op],
feed_dict={handle: train_handle})
if global_step % config.checkpoint == 0 or global_step in [
1, 10, 50, 100, 500
]:
sess.run(
tf.assign(model.is_train, tf.constant(False,
dtype=tf.bool)))
_, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle, config)
for s in summ:
writer.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1, dev_eval_file,
sess, "dev", handle, dev_handle, config)
sess.run(
tf.assign(model.is_train, tf.constant(True,
dtype=tf.bool)))
dev_loss = metrics["loss_ee"]
if dev_loss < loss_save:
loss_save = dev_loss
patience = 0
else:
patience += 1
if patience >= config.patience:
lr /= 2.0
loss_save = dev_loss
patience = 0
sess.run(tf.assign(model.lr, tf.constant(lr,
dtype=tf.float32)))
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(config.save_dir,
"model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
train_dataset = get_batch_dataset(config.train_record_file, parser,
config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat, graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
for _ in tqdm(range(1, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
_, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle)
for s in summ:
writer.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1,
dev_eval_file, sess, "dev", handle, dev_handle)
dev_loss = metrics["loss"]
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def train(config):
with open(config.word_emb_file, "r", encoding="utf-8") as fh:
# 载入词向量矩阵，shape=(存在词向量的词数+2,300) //2是指NULL,OOV
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r", encoding="utf-8") as fh:
# 载入评估数据
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r", encoding="utf-8") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
# dev集样本数
meta = json.load(fh)
dev_total = meta["total"]
print("Building model ...")
# 解析器，解析单个example的函数
parser = get_record_parser(config)
# 使用解析器批量获取record文件中的数据
train_dataset = get_batch_dataset(config.train_record_file, parser, config)
dev_dataset = get_batch_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
# 通过tf.data.Iterator.from_string_handle来定义一个 feedable iterator，达到切换数据集的目的
# 创建迭代器访问dataset，handle参数控制使用哪个迭代器（对应一个dataset）
iterator = tf.data.Iterator.from_string_handle(handle,
train_dataset.output_types,
train_dataset.output_shapes)
# 为trainset，devset各自创建迭代器
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
#加载模型
model = Model(config, iterator, word_mat)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0.0
lr = config.init_lr
# 训练过程的loss保存到文件中
logging.basicConfig(
level=logging.DEBUG, # 控制台打印的日志级别
filename=config.log_file,
format='%(message)s' # 日志格式
)
with tf.Session(config=sess_config) as sess:
# log&summary
writer = tf.summary.FileWriter(config.log_dir)
# 保存model
saver = tf.train.Saver(max_to_keep=config.max_to_keep)
# sess.run()获取不同数据集的handel，feed in的方式切换数据集
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
# 载入之前训练的model
ckpt = tf.train.get_checkpoint_state(config.save_dir)
if not config.restart and ckpt and tf.train.checkpoint_exists(
ckpt.model_checkpoint_path):
saver.restore(sess, ckpt.model_checkpoint_path)
sess.run(
tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
metrics, _ = evaluate_batch(model,
dev_total // config.batch_size + 1,
dev_eval_file, sess, "dev", handle,
dev_handle)
loss_save = metrics["loss"]
logging.info(
"Restore from saved model,saved loss is {},saved acc is {}".
format(loss_save, metrics["acc"]))
else:
sess.run(tf.global_variables_initializer())
# 给model传参
sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool)))
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
for _ in tqdm(range(1, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
# train一个batch
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={handle: train_handle})
# 周期地保存loss
if global_step % config.period == 0:
# 记录loss
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
writer.add_summary(loss_sum, global_step)
# checkpoint表示每经过一定数量的迭代后就进行一次evaluation
if global_step % config.checkpoint == 0:
# 关闭训练模式
sess.run(
tf.assign(model.is_train, tf.constant(False,
dtype=tf.bool)))
logging.info("At {} step,model loss is {}".format(
global_step, loss))
# evaluation
metrics, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle)
for s in summ:
writer.add_summary(s, global_step)
logging.info(
"At {} step,train loss is {},train acc is {}".format(
global_step, metrics["loss"], metrics["acc"]))
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1, dev_eval_file,
sess, "dev", handle, dev_handle)
# 开启训练模式，继续训练
sess.run(
tf.assign(model.is_train, tf.constant(True,
dtype=tf.bool)))
dev_loss = metrics["loss"]
logging.info("At {} step,dev loss is {},dev acc is {}".format(
global_step, dev_loss, metrics["acc"]))
# print("At {} step,train loss is {},dev loss is {}".format(global_step, loss, dev_loss), file=loss_log)
if dev_loss < loss_save:
logging.info(
"saving model at step {},dev loss is {},dev acc is {}..."
.format(global_step, dev_loss, metrics['acc']))
# print("saving model,dev loss is {}...".format(global_step, dev_loss))
loss_save = dev_loss
patience = 0
# 保存model
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
else:
patience += 1
if patience > config.early_stop:
break
for s in summ:
writer.add_summary(s, global_step)
# 将未满的缓冲区数据刷到文件中
writer.flush()
def train(config):
word_mat = np.array(load_from_gs(config.word_emb_file), dtype=np.float32)
train_eval = load_from_gs(config.train_eval_file)
dev_eval = load_from_gs(config.dev_eval_file)
print("Building model...")
# data_manager = DataManager(config)
train_graph = tf.Graph()
dev_graph = tf.Graph()
parser = get_record_parser(config)
train_dataset = get_batch_dataset(config.train_record_file, parser, config)
dev_dataset = get_batch_dataset(config.dev_record_file, parser, config)
# initialize train model and dev model separately
with train_graph.as_default():
train_iterator_manager = IteratorManager(train_dataset)
train_model = BaseLineModel(config, train_iterator_manager.iterator,
word_mat)
initializer = tf.global_variables_initializer()
with dev_graph.as_default():
dev_iterator_manager = IteratorManager(dev_dataset)
dev_model = BaseLineModel(config,
dev_iterator_manager.iterator,
word_mat,
is_train=False)
checkpoints_path = os.path.join(config.save_dir, "checkpoints")
# initialize train and dev session
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
train_sess = tf.Session(graph=train_graph, config=sess_config)
dev_sess = tf.Session(graph=dev_graph, config=sess_config)
train_sess.run(initializer)
train_iterator_manager.get_string_handle(train_sess)
dev_iterator_manager.get_string_handle(dev_sess)
summary_writer = SummaryWriter(config.log_dir)
lr_updater = LearningRateUpdater(patience=config.patience,
init_lr=config.init_lr,
loss_save=100.0)
lr_updater.assign(train_sess, train_model)
# checkpoint_path = tf.train.latest_checkpoint(config.save_dir, latest_filename=None)
# train_model.saver.restore(train_sess, checkpoint_path)
for _ in xrange(1, config.num_steps + 1):
global_step = train_sess.run(train_model.global_step) + 1
loss, train_op, grad_summ, weight_summ = train_sess.run(
[
train_model.loss, train_model.train_op, train_model.grad_summ,
train_model.weight_summ
],
feed_dict=train_iterator_manager.make_feed_dict())
# tf.logging.info("training step: step {} adding loss: {}".format(global_step, loss))
if global_step % config.period == 0:
tf.logging.info("training step: step {} adding loss: {}".format(
global_step, loss))
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
summary_writer.write_summaries([loss_sum], global_step)
summary_writer.write_summaries([grad_summ], global_step)
summary_writer.write_summaries([weight_summ], global_step)
lr_summ = tf.Summary(value=[
tf.Summary.Value(tag="model/lr", simple_value=lr_updater.lr),
])
summary_writer.write_summaries([lr_summ], global_step)
# summary_writer.flush()
# lr_updater.update(loss)
# lr_updater.assign(train_sess, train_model)
if global_step % config.checkpoint == 0:
lr_updater.setZero(train_sess, train_model)
tf.logging.info("training step: step {} checking the model".format(
global_step))
checkpoint_path = train_model.saver.save(train_sess,
checkpoints_path,
global_step=global_step)
_, summ = evaluate_batch(train_model, config.val_num_batches,
train_eval, train_sess, "train",
train_iterator_manager)
summary_writer.write_summaries(summ, global_step)
dev_model.saver.restore(dev_sess, checkpoint_path)
metrics, summ = evaluate_batch(dev_model, config.val_num_batches,
dev_eval, dev_sess, "dev",
dev_iterator_manager)
summary_writer.write_summaries(summ, global_step)
summary_writer.flush()
lr_updater.update(metrics["loss"], global_step)
lr_updater.assign(train_sess, train_model)
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
train_dataset = get_batch_dataset(config.train_record_file, parser, config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat, graph = g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
best_f1 = 0.
best_em = 0.
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(config.save_dir, "checkpoint")):
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
global_step = max(sess.run(model.global_step), 1)
for _ in tqdm(range(global_step, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op], feed_dict={
handle: train_handle, model.dropout: config.dropout})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[tf.Summary.Value(
tag="model/loss", simple_value=loss), ])
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
_, summ = evaluate_batch(
model, config.val_num_batches, train_eval_file, sess, "train", handle, train_handle)
for s in summ:
writer.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1, dev_eval_file, sess, "dev", handle, dev_handle)
dev_f1 = metrics["f1"]
dev_em = metrics["exact_match"]
if dev_f1 < best_f1 and dev_em < best_em:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_em = max(best_em, dev_em)
best_f1 = max(best_f1, dev_f1)
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def test(config):
""" Testing the model on the dev set (the test set of SQuAD2 is not available.) """
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
graph = tf.Graph()
print("Loading model...")
with graph.as_default() as g:
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config,
test_batch,
word_mat,
char_mat,
trainable=False,
graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
if os.path.exists(os.path.join(config.save_dir, "checkpoint")):
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir))
print("loaded from save.", config.save_dir)
elif os.path.exists(os.path.join(config.load_dir, "checkpoint")):
# define the pretrained variable scopes
scopes_to_be_loaded = [
'Input_Embedding_Layer', 'Embedding_Encoder_Layer',
'Context_to_Query_Attention_Layer', 'Model_Encoder_Layer',
'Output_Layer'
]
for scope in scopes_to_be_loaded:
variables_can_be_restored = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope=scope)
temp_saver = tf.train.Saver(variables_can_be_restored)
temp_saver.restore(
sess, tf.train.latest_checkpoint(config.load_dir))
print("from: ", config.load_dir, " loaded: ",
scopes_to_be_loaded)
if config.decay < 1.0:
sess.run(model.assign_vars)
losses = []
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2, yp3 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2, model.yp3])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist(),
yp3.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
loss = np.mean(losses)
metrics = evaluate(eval_file, answer_dict)
print(metrics)
with open(config.answer_file, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {}".format(metrics['exact_match'],
metrics['f1']))
def train(config):
with open(config.emb_mat_file, 'r') as fh:
emb_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.meta_file, 'r') as fh:
meta = json.load(fh)
train_total = meta["train_total"]
test_total = meta["test_total"]
print("Building model...")
parser = get_record_parser(config)
train_dataset = get_batch_dataset(config.train_record_file, parser, train_total, config)
test_dataset = get_dataset(config.test_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
test_iterator = test_dataset.make_one_shot_iterator()
model = Model(config, iterator, emb_mat)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
lr = config.learning_rate
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
test_handle = sess.run(test_iterator.string_handle())
if os.path.exists(os.path.join(config.save_dir, "checkpoint")):
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
global_step = max(sess.run(model.global_step), 0)
step_per_epoch = train_total // config.batch_size
test_step_per_epoch = test_total // config.batch_size
epoch = global_step // step_per_epoch + 1
epoch_step = global_step % step_per_epoch
print("init epoch: {}, init epoch_step: {}".format(epoch, epoch_step))
for epoch in tqdm(range(epoch, config.epoches + 1), ncols=100):
# for epoch in range(epoch, config.epoches + 1):
losses, accs, tps, fps, fns = 0., 0., 0., 0., 0.
for _ in tqdm(range(global_step % step_per_epoch, step_per_epoch), ncols=100):
# for _ in range(global_step % step_per_epoch, step_per_epoch):
global_step = sess.run(model.global_step) + 1
loss, acc, tp, fp, fn, _ = sess.run(
[model.loss, model.acc, model.tp, model.fp, model.fn, model.train_op],
feed_dict={handle: train_handle, model.dropout: config.dropout, model.lr: lr})
losses += loss
accs += acc
tps += tp
fps += fp
fns += fn
if global_step % 100 == 0:
loss_sum = tf.Summary(value=[tf.Summary.Value(tag="model/loss", simple_value=loss), ])
writer.add_summary(loss_sum, global_step)
loss = losses / step_per_epoch
acc = accs / step_per_epoch
precision = tps / (tps + fps + 1e-6)
recall = tps / (tps + fns + 1e-6)
loss_sum = tf.Summary(value=[tf.Summary.Value(tag="train/loss", simple_value=loss), ])
writer.add_summary(loss_sum, epoch)
acc_sum = tf.Summary(value=[tf.Summary.Value(tag="train/acc", simple_value=acc), ])
writer.add_summary(acc_sum, epoch)
precision_sum = tf.Summary(value=[tf.Summary.Value(tag="train/precision", simple_value=precision), ])
writer.add_summary(precision_sum, epoch)
recall_sum = tf.Summary(value=[tf.Summary.Value(tag="train/recall", simple_value=recall), ])
writer.add_summary(recall_sum, epoch)
print()
print()
print('tps:', tps)
print('fps:', fps)
print('fns:', fns)
print('TRAIN Epoch {}: loss {:.4f} acc {:.4f} precision {:.4f} recall {:.4f}'.format(
epoch, loss, acc, precision, recall))
losses, accs, tps, fps, fns = 0., 0., 0., 0., 0.
# for _ in tqdm(range(test_step_per_epoch), ncols=100):
for _ in range(test_step_per_epoch):
loss, acc, tp, fp, fn = sess.run(
[model.loss, model.acc, model.tp, model.fp, model.fn],
feed_dict={handle: test_handle})
losses += loss
accs += acc
tps += tp
fps += fp
fns += fn
loss = losses / test_step_per_epoch
acc = accs / test_step_per_epoch
precision = tps / (tps + fps + 1e-6)
recall = tps / (tps + fns + 1e-6)
loss_sum = tf.Summary(value=[tf.Summary.Value(tag="test/loss", simple_value=loss), ])
writer.add_summary(loss_sum, epoch)
acc_sum = tf.Summary(value=[tf.Summary.Value(tag="test/acc", simple_value=acc), ])
writer.add_summary(acc_sum, epoch)
precision_sum = tf.Summary(value=[tf.Summary.Value(tag="test/precision", simple_value=precision), ])
writer.add_summary(precision_sum, epoch)
recall_sum = tf.Summary(value=[tf.Summary.Value(tag="test/recall", simple_value=recall), ])
writer.add_summary(recall_sum, epoch)
print('tps:', tps)
print('fps:', fps)
print('fns:', fns)
print('TEST Epoch {}: loss {:.4f} acc {:.4f} precision {:.4f} recall {:.4f}'.format(
epoch, loss, acc, precision, recall))
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
train_dataset = get_batch_dataset(config.train_record_file, parser,
config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
best_f1 = 0.
best_em = 0.
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(config.save_dir, "checkpoint")):
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir))
global_step = max(sess.run(model.global_step), 1)
for _ in (range(global_step, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
print('Step\'{}s loss: {}'.format(global_step, loss))
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
metrics, summ = evaluate_batch(model,
config.val_num_batches,
train_eval_file, sess,
"train", handle,
train_handle)
for s in summ:
writer.add_summary(s, global_step)
print('train em: ', metrics["exact_match"])
print('train f1: ', metrics["f1"])
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1,
dev_eval_file, sess, "dev", handle, dev_handle)
dev_f1 = metrics["f1"]
dev_em = metrics["exact_match"]
print('dev em: ', metrics["exact_match"])
print('dev f1: ', metrics["f1"])
if dev_f1 < best_f1 and dev_em < best_em:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_em = max(best_em, dev_em)
best_f1 = max(best_f1, dev_f1)
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def test(config):
# Load word embedding file
with open(config.word_emb_file, 'r') as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
# Load char embedding file
with open(config.char_emb_file, 'r') as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_meta, 'r') as fh:
test_meta = json.load(fh)
test_total = test_meta['total']
graph = tf.Graph()
print('Loading model...')
with graph.as_default() as g:
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config,
test_batch,
word_mat,
char_mat,
trainable=False,
graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
# if config.decay < 1.0:
# sess.run(model.assign_vars)
total_loss = 0.0
total_corrects = 0
result = {}
for step in tqdm(range(test_total // config.batch_size + 1)):
loss_in_batch, corrects_in_batch = sess.run(
[model.loss, model.correct_prediction])
total_loss += loss_in_batch
total_corrects += corrects_in_batch
loss = total_loss / test_total
acc = total_corrects / test_total
result['loss'] = loss
result['acc'] = acc
with open(config.answer_file, 'w') as fh:
json.dump(result, fh)
print('Loss: {}, Accuracy: {}'.format(loss, acc))
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
train_dataset = get_batch_dataset(config.train_record_file, parser, config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
lr = config.init_lr
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool)))
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
for _ in tqdm(range(1, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op], feed_dict={
handle: train_handle})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[tf.Summary.Value(
tag="model/loss", simple_value=loss), ])
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
sess.run(tf.assign(model.is_train,
tf.constant(False, dtype=tf.bool)))
_, summ = evaluate_batch(
model, config.val_num_batches, train_eval_file, sess, "train", handle, train_handle)
for s in summ:
writer.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1, dev_eval_file, sess, "dev", handle, dev_handle)
sess.run(tf.assign(model.is_train,
tf.constant(True, dtype=tf.bool)))
dev_loss = metrics["loss"]
if dev_loss < loss_save:
loss_save = dev_loss
patience = 0
else:
patience += 1
if patience >= config.patience:
lr /= 2.0
loss_save = dev_loss
patience = 0
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def train(config):
with open(config.word_emb_file, 'r') as fh:
word_mat = np.array(json.load(fh),
dtype=np.float32) # word embedding matrix
with open(config.char_emb_file, 'r') as fh:
char_mat = np.array(json.load(fh),
dtype=np.float32) # char embedding matrix
# total examples number in valid file
with open(config.dev_meta, 'r') as fh:
dev_meta = json.load(fh)
dev_total = dev_meta['total']
print('Building model...')
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
train_dataset = get_batch_dataset(config.train_record_file, parser,
config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat, graph=g)
# model = QANet4CBT(config, iterator, word_mat, char_mat, graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 10.0
patience = 0
best_acc = 0.
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(config.save_dir, 'checkpoint')):
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir))
global_step = max(sess.run(model.global_step), 1)
total_corrects = 0
total_loss = 0.0
# Training
for _ in tqdm(range(global_step, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss_in_batch, corrects_in_batch, train_op = sess.run(
[model.loss, model.correct_prediction, model.train_op],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
total_corrects += corrects_in_batch
total_loss += loss_in_batch * config.batch_size
if global_step % config.period == 0:
acc = total_corrects / (global_step * config.batch_size)
loss = total_loss / (global_step * config.batch_size)
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag='model/loss', simple_value=loss),
])
writer.add_summary(loss_sum, global_step)
acc_sum = tf.Summary(value=[
tf.Summary.Value(tag='model/acc', simple_value=acc),
])
writer.add_summary(acc_sum, global_step)
# Validation and save model
if global_step % config.checkpoint == 0:
val_acc, val_loss, v_acc_sum, v_loss_sum = validate(
config, model, sess, dev_total, 'dev', handle,
dev_handle)
writer.add_summary(v_acc_sum, global_step)
writer.add_summary(v_loss_sum, global_step)
# Early Stopping
if val_acc < best_acc:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_acc = max(best_acc, val_acc)
# Save Model, keep top 5 best models.
filename = os.path.join(
config.save_dir, 'model_{}_val-acc_{}.ckpt'.format(
global_step, best_acc))
saver.save(sess, filename)
writer.flush()
def _test(config, k):
print('-' * 10 + 'cv-{:0>2d}'.format(k) + '-' * 10)
prepro_home = os.path.join(config.prepro_dir, '{:0>2d}'.format(k))
emb_mat_file = os.path.join(prepro_home, 'emb_mat.json')
meta_file = os.path.join(prepro_home, 'meta.json')
idx2token_file = os.path.join(prepro_home, 'idx2token.json')
test_record_file = os.path.join(prepro_home, 'test.tfrecords')
log_dir = os.path.join(config.log_dir, 'ensemble')
save_dir = os.path.join(config.save_dir, '{:0>2d}'.format(k))
answer_dir = os.path.join(config.answer_dir, '{:0>2d}'.format(k))
with open(emb_mat_file, 'r') as fh:
emb_mat = np.array(json.load(fh), dtype=np.float32)
with open(meta_file, 'r') as fh:
meta = json.load(fh)
with open(idx2token_file, 'r') as fh:
idx2token = json.load(fh)
test_total = meta["test_total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
test_dataset = get_dataset(test_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, test_dataset.output_types, test_dataset.output_shapes)
test_iterator = test_dataset.make_one_shot_iterator()
model = Model(config, iterator, emb_mat)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
test_handle = sess.run(test_iterator.string_handle())
if os.path.exists(os.path.join(save_dir, "checkpoint")):
saver.restore(sess, tf.train.latest_checkpoint(save_dir))
global_step = max(sess.run(model.global_step), 0)
test_step_per_epoch = test_total // config.batch_size
cs, qs, preds, labels = [], [], [], []
accs, tps, fps, fns = 0., 0., 0., 0.
for _ in tqdm(range(test_step_per_epoch), ncols=100):
# for _ in range(test_step_per_epoch):
c, q, y, pred, label, acc, tp, fp, fn = sess.run(
[model.c, model.q, model.y, model.pred, model.label,
model.acc, model.tp, model.fp, model.fn],
feed_dict={handle: test_handle})
for i in range(len(c)):
cs.append([idx2token[str(w)] for w in c[i] if w])
qs.append([idx2token[str(w)] for w in q[i] if w])
preds += list(pred)
labels += list(label)
accs += acc
tps += tp
fps += fp
fns += fn
acc = accs / test_step_per_epoch
precision = tps / (tps + fps + 1e-6)
recall = tps / (tps + fns + 1e-6)
print('tps:', tps)
print('fps:', fps)
print('fns:', fns)
print('TEST CV {} acc {:.4f} precision {:.4f} recall {:.4f}'.format(
k, acc, precision, recall))
return cs, qs, preds, labels
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
# 验证集总数据量
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
# tf.data.TFRecordDataset类型
train_dataset = get_batch_dataset(config.train_record_file, parser,
config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
# 单次迭代器
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
# 迭代器句柄占位符
handle = tf.placeholder(tf.string, shape=[])
# 这个迭代器会根据handle不同而使用handle对应的迭代器
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
model = Model(config, iterator, word_mat, char_mat, graph=g)
# 可动态申请显存
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
best_f1 = 0.
best_em = 0.
with tf.Session(config=sess_config) as sess:
# 指定一个文件来保存图，可以调用add_summary()将训练过成数据保存在FileWriter指定的文件中
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
# 各个数据集创建对应的handle
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(config.save_dir, "checkpoint")):
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir))
global_step = max(sess.run(model.global_step), 1)
for _ in tqdm(range(global_step, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
# 每config.period轮记录一次损失batch_loss
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
# 向logs写入模型误差
writer.add_summary(loss_sum, global_step)
# 每config.checkpoint轮评估和保存一次模型
if global_step % config.checkpoint == 0:
# 训练集效果
_, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle)
for s in summ:
# 写入训练效果
writer.add_summary(s, global_step)
# 验证集效果
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1,
dev_eval_file, sess, "dev", handle, dev_handle)
for s in summ:
# 写入验证效果
writer.add_summary(s, global_step)
dev_f1 = metrics["f1"]
dev_em = metrics["exact_match"]
# 如果大于early_stop轮没提升就停止训练
if dev_f1 < best_f1 and dev_em < best_em:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_em = max(best_em, dev_em)
best_f1 = max(best_f1, dev_f1)
# 保存logs
writer.flush()
# 保存模型
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def train(config):
""" Training the network. """
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
# get total number of entries in dev
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
# get the datasets in the form of batches:
train_dataset = get_batch_dataset(config.train_record_file, parser,
config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
print(train_dataset.output_shapes)
#print(train_dataset.shape)
# create the iterators for the training:
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
# create an instance of the model to be trained:
model = Model(config, iterator, word_mat, char_mat, graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
best_f1 = 0.
best_em = 0.
list_losses = []
list_acc = []
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
# initialize all variables:
sess.run(tf.global_variables_initializer())
# instance of the saver to save the model at the end of training:
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(config.save_dir, "checkpoint")):
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir))
print("loaded from save.", config.save_dir)
elif os.path.exists(os.path.join(config.load_dir, "checkpoint")):
# define the pretrained variable scopes
scopes_to_be_loaded = [
'Input_Embedding_Layer', 'Embedding_Encoder_Layer',
'Context_to_Query_Attention_Layer', 'Model_Encoder_Layer',
'Output_Layer'
]
for scope in scopes_to_be_loaded:
variables_can_be_restored = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope=scope)
temp_saver = tf.train.Saver(variables_can_be_restored)
temp_saver.restore(
sess, tf.train.latest_checkpoint(config.load_dir))
print("from: ", config.load_dir, " loaded: ",
scopes_to_be_loaded)
else:
print("training from scratch")
global_step_1 = max(sess.run(model.global_step), 1)
for _ in tqdm(range(global_step_1, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
# one training run:
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
# if in periode to save batch loss
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
writer.add_summary(loss_sum, global_step)
metrics, summ = evaluate_batch(model, 2, train_eval_file,
sess, "train", handle,
train_handle)
# if at a checkpoint to save and evaluate the model
if global_step % config.checkpoint == 0:
print("Evaluating batches...")
metrics, summ = evaluate_batch(model,
config.val_num_batches,
train_eval_file, sess,
"train", handle,
train_handle)
for s in summ:
writer.add_summary(s, global_step)
print("Perf on train sample: " + str(metrics))
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1,
dev_eval_file, sess, "dev", handle, dev_handle)
print("Perf on dev sample: " + str(metrics))
list_losses.append(metrics["loss"])
list_acc.append(metrics["true_acc"])
print("Loss list: ", list_losses)
print("True acc list: ", list_acc)
# early stoping method
dev_f1 = metrics["f1"]
dev_em = metrics["exact_match"]
if dev_f1 < best_f1 and dev_em < best_em:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_em = max(best_em, dev_em)
best_f1 = max(best_f1, dev_f1)
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
# save the model
try:
filename = os.path.join(
config.save_dir,
"model_{}.ckpt".format(global_step))
saver.save(sess, filename)
except:
pass