def _forward(config):
assert config.load
test_data = read_data(config, config.forward_name, True)
update_config(config, [test_data])
_config_debug(config)
if config.use_glove_for_unk:
word2vec_dict = test_data.shared['lower_word2vec'] if config.lower_word else test_data.shared['word2vec']
new_word2idx_dict = test_data.shared['new_word2idx']
idx2vec_dict = {idx: word2vec_dict[word] for word, idx in new_word2idx_dict.items()}
new_emb_mat = np.array([idx2vec_dict[idx] for idx in range(len(idx2vec_dict))], dtype='float32')
config.new_emb_mat = new_emb_mat
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
print("num params: {}".format(get_num_params()))
evaluator = Evaluator(config, model)
graph_handler = GraphHandler(config, model) # controls all tensors and variables in the graph, including loading /saving
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
num_batches = math.ceil(test_data.num_examples / config.batch_size)
if 0 < config.test_num_batches < num_batches:
num_batches = config.test_num_batches
e = evaluator.get_evaluation_from_batches(sess, tqdm(test_data.get_batches(config.batch_size, num_batches=num_batches), total=num_batches))
print(e)
if config.dump_answer:
print("dumping answer ...")
graph_handler.dump_answer(e, path=config.answer_path)
if config.dump_eval:
print("dumping eval ...")
graph_handler.dump_eval(e, path=config.eval_path)
def _train(config):
data_filter = get_squad_data_filter(config)
#train_data = read_data(config, 'train', config.load, data_filter=data_filter)
train_data = read_data(config, 'train', False, data_filter=data_filter)
dev_data = read_data(config, 'dev', True, data_filter=data_filter)
update_config(config, [train_data, dev_data])
_config_debug(config)
word2vec_dict = train_data.shared[
'lower_word2vec'] if config.lower_word else train_data.shared[
'word2vec']
word2idx_dict = train_data.shared['word2idx']
idx2vec_dict = {
word2idx_dict[word]: vec
for word, vec in word2vec_dict.items() if word in word2idx_dict
}
emb_mat = np.array([
idx2vec_dict[idx]
if idx in idx2vec_dict else np.random.multivariate_normal(
np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in range(config.word_vocab_size)
])
config.emb_mat = emb_mat
# construct model graph and variables (using default graph)
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
print("num params: {}".format(get_num_params()))
trainer = MultiGPUTrainer(config, models)
evaluator = MultiGPUF1Evaluator(
config, models, tensor_dict=model.tensor_dict if config.vis else None)
graph_handler = GraphHandler(
config, model
) # controls all tensors and variables in the graph, including loading /saving
# Variables
config_proto = tf.ConfigProto()
config_proto.gpu_options.allow_growth = True
config_proto.allow_soft_placement = True
sess = tf.Session(config=config_proto)
graph_handler.initialize(sess)
# plot weights
for train_var in tf.trainable_variables():
plot_tensor(train_var.eval(session=sess),
train_var.op.name,
plot_weights=config.plot_weights,
hidden_size=config.hidden_size)
# Begin training
num_steps = config.num_steps or int(
math.ceil(train_data.num_examples /
(config.batch_size * config.num_gpus))) * config.num_epochs
global_step = 0
for batches in tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps,
shuffle=True,
cluster=config.cluster),
total=num_steps):
global_step = sess.run(
model.global_step
) + 1 # +1 because all calculations are done after step
get_summary = global_step % config.log_period == 0
loss, summary, train_op = trainer.step(sess,
batches,
get_summary=get_summary)
if get_summary:
graph_handler.add_summary(summary, global_step)
# occasional saving
if global_step % config.save_period == 0:
graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
num_steps = math.ceil(dev_data.num_examples /
(config.batch_size * config.num_gpus))
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
e_train = evaluator.get_evaluation_from_batches(
sess,
tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
graph_handler.add_summaries(e_train.summaries, global_step)
e_dev = evaluator.get_evaluation_from_batches(
sess,
tqdm(dev_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
graph_handler.add_summaries(e_dev.summaries, global_step)
if config.dump_eval:
graph_handler.dump_eval(e_dev)
if config.dump_answer:
graph_handler.dump_answer(e_dev)
if global_step % config.save_period != 0:
graph_handler.save(sess, global_step=global_step)
def _train(config):
data_filter = get_squad_data_filter(config)
train_data = read_data(config,
'train',
config.load,
data_filter=data_filter)
dev_data = read_data(config, config.dev_name, True, data_filter=None)
update_config(config, [train_data, dev_data])
_config_debug(config)
word2vec_dict = train_data.shared[
'lower_word2vec'] if config.lower_word else train_data.shared[
'word2vec']
word2idx_dict = train_data.shared['word2idx']
idx2vec_dict = {
word2idx_dict[word]: vec
for word, vec in word2vec_dict.items() if word in word2idx_dict
}
emb_mat = np.array([
idx2vec_dict[idx]
if idx in idx2vec_dict else np.random.multivariate_normal(
np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in range(config.word_vocab_size)
])
config.emb_mat = emb_mat
# construct model graph and variables (using default graph)
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
print("num params: {}".format(get_num_params()))
trainer = MultiGPUTrainer(config, models)
if config.model_name == 'basic':
ThisEvaluator = MultiGPUF1Evaluator
elif config.model_name in ['basic-class', 'basic-generate', 'baseline']:
ThisEvaluator = MultiGPUClassificationAccuracyEvaluator
elif config.model_name == 'span-gen':
ThisEvaluator = UnionEvaluator
evaluator = ThisEvaluator(
config, models, tensor_dict=model.tensor_dict if config.vis else None)
graph_handler = GraphHandler(
config, model
) # controls all tensors and variables in the graph, including loading /saving
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
# Begin training
num_steps = config.num_steps or int(
math.ceil(train_data.num_examples /
(config.batch_size * config.num_gpus))) * config.num_epochs
global_step = 0
for batches in tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps,
shuffle=True,
cluster=config.cluster),
total=num_steps):
global_step = sess.run(
model.global_step
) + 1 # +1 because all calculations are done after step
get_summary = global_step % config.log_period == 0
loss, summary, train_op = trainer.step(sess,
batches,
get_summary=get_summary)
if get_summary:
graph_handler.add_summary(summary, global_step)
# occasional saving
if global_step % config.save_period == 0:
print("Saving variables on step ", global_step)
graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
num_steps = math.ceil(dev_data.num_examples /
(config.batch_size * config.num_gpus))
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
"""
train_batches = tqdm(train_data.get_multi_batches(config.batch_size, config.num_gpus, num_steps=num_steps), total=num_steps)
e_train = evaluator.get_evaluation_from_batches(
sess, train_batches
)
graph_handler.add_summaries(e_train.summaries, global_step)
"""
e_dev = evaluator.get_evaluation_from_batches(
sess,
tqdm(dev_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
print("Evaluated on dev at step ", global_step, ": ", e_dev)
graph_handler.add_summaries(e_dev.summaries, global_step)
if config.dump_eval:
graph_handler.dump_eval(e_dev)
if config.dump_answer:
graph_handler.dump_answer(e_dev)
if global_step % config.save_period != 0:
print("Final save at step ", global_step)
graph_handler.save(sess, global_step=global_step)
def _train(config):
data_filter = get_squad_data_filter(config)
train_data = read_data(config,
'train',
config.load,
data_filter=data_filter)
dev_data = read_data(config, 'dev', True, data_filter=data_filter)
update_config(config, [train_data])
_config_debug(config)
word2vec_dict = train_data.shared[
'lower_word2vec'] if config.lower_word else train_data.shared[
'word2vec']
word2idx_dict = train_data.shared['word2idx']
idx2vec_dict = {
word2idx_dict[word]: vec
for word, vec in word2vec_dict.items() if word in word2idx_dict
}
emb_mat = np.array([
idx2vec_dict[idx]
if idx in idx2vec_dict else np.random.multivariate_normal(
np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in range(config.word_vocab_size)
])
config.len_new_emb_mat = len(dev_data.shared['idx2word'])
config.emb_mat = emb_mat # INITIALIZE EMB MAT IN CONFIG
#entity_mat = np.array([i for i in range(config.word_vocab_size - config.vw_wo_entity_size)])
# binary encode
#onehot_encoder = OneHotEncoder(categories="auto",sparse=False)
#config.onehot_encoded = onehot_encoder.fit_transform(entity_mat.reshape(-1,1))
# construct model graph and variables (using default graph)
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
print("num params: {}".format(get_num_params()))
trainer = MultiGPUTrainer(config, models)
#evaluator = MultiGPUF1Evaluator(config, models, tensor_dict=model.tensor_dict if config.vis else None) # FIXME: Put this back!
#BLEU evaluator
evaluator = BleuEvaluatorSpan(
config, model, tensor_dict=model.tensor_dict if config.vis else None)
graph_handler = GraphHandler(
config, model
) # controls all tensors and variables in the graph, including loading /saving
# Variables
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
# Begin training
num_steps = config.num_steps or int(
math.ceil(train_data.num_examples /
(config.batch_size * config.num_gpus))) * config.num_epochs
global_step = 0
for batches in tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps,
shuffle=True,
cluster=config.cluster),
total=num_steps):
# batches and models should be of the same length
global_step = sess.run(
model.global_step
) + 1 # +1 because all calculations are done after step
get_summary = global_step % config.log_period == 0
print("TRAINER STEP STARTS!")
loss, summary, train_op = trainer.step(sess,
batches,
get_summary=get_summary)
print("TRAINER STEP DONE!")
#if get_summary:
# graph_handler.add_summary(summary, global_step)
# occasional saving
if global_step % config.save_period == 0:
graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
num_steps = math.ceil(dev_data.num_examples /
(config.batch_size * config.num_gpus))
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
print("train eval started")
e_train = evaluator.get_evaluation_from_batches(
sess,
tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
print("dev eval started")
graph_handler.add_summaries(e_train.summaries, global_step)
e_dev = evaluator.get_evaluation_from_batches(
sess,
tqdm(dev_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
graph_handler.add_summaries(e_dev.summaries, global_step)
print("eval done")
if config.dump_eval:
graph_handler.dump_eval(e_dev)
graph_handler.dump_eval(e_train)
if config.dump_answer:
graph_handler.dump_answer(e_dev)
print("dump eval done")
if global_step % config.save_period != 0:
graph_handler.save(sess, global_step=global_step)
def _train(config):
data_filter = get_squad_data_filter(config)
train_data = read_data(config, 'train', config.load, data_filter=data_filter)
dev_data = read_data(config, 'dev', True, data_filter=data_filter)
update_config(config, [train_data, dev_data])
_config_debug(config)
word2vec_dict = train_data.shared['lower_word2vec'] if config.lower_word else train_data.shared['word2vec']
word2idx_dict = train_data.shared['word2idx']
new_word2idx_dict = train_data.shared['new_word2idx']
print('word2idx len : {}, new_word2idx len : {}'.format(len(word2idx_dict), len(new_word2idx_dict)))
idx2vec_dict = {word2idx_dict[word]: vec for word, vec in word2vec_dict.items() if word in word2idx_dict}
idx2word_dict = {idx: word for word, idx in word2idx_dict.items()}
offset = len(idx2word_dict)
idx2word_dict.update({offset+idx: word for word, idx in new_word2idx_dict.items()})
train_data.shared['idx2word'] = idx2word_dict
emb_mat = np.array([idx2vec_dict[idx] if idx in idx2vec_dict
else np.random.multivariate_normal(np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in range(config.word_vocab_size)])
config.emb_mat = emb_mat
# construct model graph and variables (using default graph)
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
print("num params: {}".format(get_num_params()))
trainer = MultiGPUTrainer(config, models)
evaluator = MultiGPUF1Evaluator(config, models, tensor_dict=model.tensor_dict if config.vis else None)
graph_handler = GraphHandler(config, model) # controls all tensors and variables in the graph, including loading /saving
# Variables
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
# Begin training
num_steps = config.num_steps or int(math.ceil(train_data.num_examples / (config.batch_size * config.num_gpus))) * config.num_epochs
global_step = 0
for batches in tqdm(train_data.get_multi_batches(config.batch_size, config.num_gpus,
num_steps=num_steps, shuffle=True, cluster=config.cluster), total=num_steps):
# QA, QG shared encoder로 학습하는 부분
global_step = sess.run(model.global_step) + 1
#get_summary = global_step % config.log_period == 1
get_summary = True # 너무 답답해서 매 스텝마다 찍어야겠다...
loss, seq2seq_loss, summary, train_op, gen_q_sample = trainer.step(sess, batches, get_summary=get_summary)
config.is_gen = False
print("global step : ", global_step)
print("Loss : ", loss, "|", seq2seq_loss)
print("Generated Question Sample : ", ' '.join([idx2word_dict[w] for w in gen_q_sample[0]]))
"""
config.is_gen = True
for (_, batch) in batches:
batch.data['q'] = ['']*len(gen_q_sample)
batch.data['cq'] = ['']*len(gen_q_sample)
for b_idx in range(len(gen_q_sample)):
batch.data['q'][b_idx] = [idx2word_dict[w] if w in idx2word_dict else "-UNK-" for w in gen_q_sample[b_idx]]
batch.data['cq'][b_idx] = [list(idx2word_dict[w] if w in idx2word_dict else "-UNK-") for w in gen_q_sample[b_idx]]
qa_gen_loss, _, __, train_op, ___ = trainer.step(sess, batch, get_summary=get_summary, is_gen=config.is_gen)
print("QA Gen Loss : ", qa_gen_loss)
"""
if get_summary:
graph_handler.add_summary(summary, global_step)
# occasional saving
if global_step % config.save_period == 0:
graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
num_steps = math.ceil(dev_data.num_examples / (config.batch_size * config.num_gpus))
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
"""
e_train = evaluator.get_evaluation_from_batches(
sess, tqdm(train_data.get_multi_batches(config.batch_size, config.num_gpus, num_steps=num_steps), total=num_steps)
)
graph_handler.add_summaries(e_train.summaries, global_step)
"""
e_dev = evaluator.get_evaluation_from_batches(
sess, tqdm(dev_data.get_multi_batches(config.batch_size, config.num_gpus, num_steps=num_steps), total=num_steps))
graph_handler.add_summaries(e_dev.summaries, global_step)
print(e_dev)
if config.dump_eval:
graph_handler.dump_eval(e_dev)
if config.dump_answer:
graph_handler.dump_answer(e_dev)
if global_step % config.save_period != 0:
graph_handler.save(sess, global_step=global_step)
def _train(config):
train_data = read_data(config, 'val_train', config.load)
dev_data = read_data(config, 'val_val', True)
# test = read_data(config, 'test', True)
update_config(config, [train_data, dev_data])
_config_debug(config)
word2vec_dict = train_data.shared['lower_word2vec'] if config.lower_word else train_data.shared['word2vec']
word2idx_dict = train_data.shared['word2idx']
idx2vec_dict = {word2idx_dict[word]: vec for word, vec in word2vec_dict.items() if word in word2idx_dict}
emb_mat = np.array([idx2vec_dict[idx] if idx in idx2vec_dict
else np.random.multivariate_normal(np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in range(config.word_vocab_size)])
config.emb_mat = emb_mat
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
print("num params: {}".format(get_num_params()))
trainer = MultiGPUTrainer(config, models)
evaluator = AccuracyEvaluator(config.train_num_can, config, model,
tensor_dict=model.tensor_dict if config.vis else None)
graph_handler = GraphHandler(config,
model) # controls all tensors and variables in the graph, including loading /saving
# Variables
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
# Begin training
num_steps = config.num_steps or int(
math.ceil(train_data.num_examples / (config.batch_size * config.num_gpus))) * config.num_epochs
global_step = 0
best_dev=[0,0]
for batches in tqdm(train_data.get_multi_batches(config.batch_size, config.num_gpus,
num_steps=num_steps, shuffle=False, cluster=config.cluster), total=num_steps):
global_step = sess.run(model.global_step) + 1 # +1 because all calculations are done after step
get_summary = global_step % config.log_period == 0
loss, summary, train_op = trainer.step(sess, batches, get_summary=get_summary)
if get_summary:
graph_handler.add_summary(summary, global_step)
if not config.eval:
continue
if global_step % config.eval_period == 0:
num_steps_dev = math.ceil(dev_data.num_examples / (config.batch_size * config.num_gpus))
num_steps_train = math.ceil(train_data.num_examples / (config.batch_size * config.num_gpus))
e_train = evaluator.get_evaluation_from_batches(
sess, tqdm(train_data.get_multi_batches(config.batch_size, config.num_gpus, num_steps=num_steps_train),
total=num_steps_train)
)
# graph_handler.add_summaries(e_test.summaries, global_step)
e_dev = evaluator.get_evaluation_from_batches(
sess, tqdm(dev_data.get_multi_batches(config.batch_size, config.num_gpus, num_steps=num_steps_dev),
total=num_steps_dev))
# graph_handler.dump_eval(e)
# graph_handler.add_summaries(e_dev.summaries, global_step)
print('train step:{} loss:{} acc:{}'.format(global_step, e_train.loss, e_train.acc))
print('val step:{} loss:{} acc:{}'.format(global_step, e_dev.loss, e_dev.acc))
# print('w_s:{}'.format(w_s))
if global_step > 700:
config.save_period = 50
config.eval_period = 50
if best_dev[0] < e_dev.acc:
best_dev=[e_dev.acc,global_step,e_train.acc]
graph_handler.save(sess, global_step=global_step)
# if config.dump_eval:
# graph_handler.dump_eval(e_dev)
if global_step % config.save_period != 0:
graph_handler.save(sess, global_step=global_step)
print (best_dev)
print ("you can test on test data set and set load setp is {}".format(best_dev[1]))
def _train(config):
data_filter = get_squad_data_filter(config)
train_data = read_data(config, config.trainfile, "train", data_filter = data_filter)
dev_data = read_data(config, config.validfile, "valid", data_filter = data_filter)
update_config(config, [train_data, dev_data])
_config_debug(config)
models = get_multi_gpu_models(config)
model = models[0]
print("num params: {}".format(get_num_params()))
trainer = MultiGPUTrainer(config, models)
evaluator = MultiGPUF1Evaluator(config, models, tensor_dict=model.tensor_dict if config.vis else None)
# controls all tensors and variables in the graph, including loading /saving
graph_handler = GraphHandler(config, model)
# Variables
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
# Begin training
num_steps = min(config.num_steps,int(math.ceil(train_data.num_examples /
(config.batch_size * config.num_gpus))) * config.num_epochs)
acc = 0
for batches in tqdm(train_data.get_multi_batches(config.batch_size, config.num_gpus, num_steps=num_steps,
shuffle=True, cluster=config.cluster),
total=num_steps):
global_step = sess.run(model.global_step) + 1 # +1 because all calculations are done after step
get_summary = global_step % config.log_period == 0
loss, summary, train_op = trainer.step(sess, batches, get_summary=get_summary)
if get_summary:
graph_handler.add_summary(summary, global_step)
# Occasional evaluation and saving
if global_step % config.save_period == 0:
num_steps = int(math.ceil(dev_data.num_examples / (config.batch_size * config.num_gpus)))
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
e_train = evaluator.get_evaluation_from_batches(sess, tqdm(train_data.get_multi_batches(
config.batch_size, config.num_gpus, num_steps=num_steps), total=num_steps))
graph_handler.add_summaries(e_train.summaries, global_step)
e_dev = evaluator.get_evaluation_from_batches(
sess, tqdm(dev_data.get_multi_batches(config.batch_size, config.num_gpus, num_steps=num_steps),
total=num_steps))
graph_handler.add_summaries(e_dev.summaries, global_step)
if e_dev.acc > acc:
acc = e_dev.acc
print("begin saving model...")
print(e_dev)
graph_handler.save(sess)
print("end saving model, dumping eval and answer...")
if config.dump_eval:
graph_handler.dump_eval(e_dev)
if config.dump_answer:
graph_handler.dump_answer(e_dev)
print("end dumping")
print("begin freezing model...")
config.clear_device = False
config.input_path = graph_handler.save_path
config.output_path = "model"
config.input_names = None
config.output_names = None
freeze_graph(config)
print("model frozen at {}".format(config.output_path))
def _train(config):
if config.dataset == 'qangaroo':
data_filter = get_qangaroo_data_filter(config)
else:
raise NotImplementedError
train_data = read_data(config,
'train',
config.load,
data_filter=data_filter)
dev_data = read_data(config, 'dev', True, data_filter=data_filter)
update_config(config, [train_data, dev_data])
word2vec_dict = train_data.shared[
'lower_word2vec'] if config.lower_word else train_data.shared[
'word2vec']
word2idx_dict = train_data.shared['word2idx']
idx2vec_dict = {
word2idx_dict[word]: vec
for word, vec in word2vec_dict.items() if word in word2idx_dict
}
emb_mat = np.array([
idx2vec_dict[idx]
if idx in idx2vec_dict else np.random.multivariate_normal(
np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in range(config.word_vocab_size)
])
# construct model graph and variables (using default graph)
pprint(config.__flags, indent=2)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
with sess.as_default():
models = get_multi_gpu_models(config, emb_mat)
model = models[0]
print("num params: {}".format(get_num_params()))
trainer = MultiGPUTrainer(config, models)
if config.reasoning_layer is not None and config.mac_prediction == 'candidates':
evaluator = MultiGPUF1CandidateEvaluator(
config,
models,
tensor_dict=model.tensor_dict if config.vis else None)
else:
evaluator = MultiGPUF1Evaluator(
config,
models,
tensor_dict=model.tensor_dict if config.vis else None)
graph_handler = GraphHandler(
config, model
) # controls all tensors and variables in the graph, including loading /saving
# Variables
#gpu_options = tf.GPUOptions(allow_growth=True)
#sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options))
#sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
# Begin training
num_steps = config.num_steps or int(
math.ceil(train_data.num_examples /
(config.batch_size * config.num_gpus))) * config.num_epochs
global_step = 0
for batches in tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps,
shuffle=True,
cluster=config.cluster),
total=num_steps):
INSUFFICIENT_DATA = False
for batch in batches:
_, ds = batch
if len(ds.data['x']) < config.batch_size:
INSUFFICIENT_DATA = True
break
if INSUFFICIENT_DATA:
continue
global_step = sess.run(
model.global_step
) + 1 # +1 because all calculations are done after step
get_summary = global_step % config.log_period == 0
loss, summary, train_op = trainer.step(sess,
batches,
get_summary=get_summary)
if get_summary:
graph_handler.add_summary(summary, global_step)
# occasional saving
if global_step % config.save_period == 0:
graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
num_steps = math.ceil(dev_data.num_examples /
(config.batch_size * config.num_gpus))
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
e_dev = evaluator.get_evaluation_from_batches(
sess,
tqdm(dev_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
graph_handler.add_summaries(e_dev.summaries, global_step)
e_train = evaluator.get_evaluation_from_batches(
sess,
tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
graph_handler.add_summaries(e_train.summaries, global_step)
if config.dump_eval:
graph_handler.dump_eval(e_dev)
if config.dump_answer:
graph_handler.dump_answer(e_dev)
if global_step % config.save_period != 0:
graph_handler.save(sess, global_step=global_step)
