def _test(config):
test_data = read_data(config, 'test', True)
update_config(config, [test_data])
_config_debug(config)
if config.use_glove_for_unk:
word2vec_dict = test_data.shared[
'lower_word2vec'] if config.lower_word else test_data.shared[
'word2vec']
new_word2idx_dict = test_data.shared['new_word2idx']
idx2vec_dict = {
idx: word2vec_dict[word]
for word, idx in new_word2idx_dict.items()
}
new_emb_mat = np.array(
[idx2vec_dict[idx] for idx in range(len(idx2vec_dict))],
dtype='float32')
config.new_emb_mat = new_emb_mat
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
evaluator = MultiGPUEvaluator(
config,
models,
tensor_dict=models[0].tensor_dict if config.vis else None)
graph_handler = GraphHandler(config, model)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
num_steps = math.ceil(test_data.num_examples /
(config.batch_size * config.num_gpus))
if 0 < config.test_num_batches < num_steps:
num_steps = config.test_num_batches
e = None
for multi_batch in tqdm(test_data.get_multi_batches(
config.batch_size,
config.num_gpus,
num_steps=num_steps,
cluster=config.cluster),
total=num_steps):
ei = evaluator.get_evaluation(sess, multi_batch)
e = ei if e is None else e + ei
if config.vis:
eval_subdir = os.path.join(
config.eval_dir, "{}-{}".format(ei.data_type,
str(ei.global_step).zfill(6)))
if not os.path.exists(eval_subdir):
os.mkdir(eval_subdir)
path = os.path.join(eval_subdir, str(ei.idxs[0]).zfill(8))
graph_handler.dump_eval(ei, path=path)
print("test acc: %f, loss: %f" % (e.acc, e.loss))
if config.dump_answer:
print("dumping answer ...")
graph_handler.dump_answer(e)
if config.dump_eval:
print("dumping eval ...")
graph_handler.dump_eval(e)
def _test(config):
test_data = read_data(config, 'test', True)
update_config(config, [test_data])
_config_debug(config)
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
evaluator = AccuracyEvaluator(config.test_num_can, config, model,
tensor_dict=models[0].tensor_dict if config.vis else None)
graph_handler = GraphHandler(config, model)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
num_steps = math.ceil(test_data.num_examples / (config.batch_size * config.num_gpus))
e = None
tensor=[]
for i, multi_batch in enumerate(tqdm(
test_data.get_multi_batches(config.batch_size, config.num_gpus, num_steps=num_steps,
cluster=config.cluster), total=num_steps)):
ei = evaluator.get_evaluation(sess, multi_batch)
# outfinal=ei.tensor
# tensor.extend(outfinal)
e = ei if e is None else e + ei
# if config.vis:
# eval_subdir = os.path.join(config.eval_dir,
# "{}-{}".format(multi_batch[0][1].data_type, str(ei.global_step).zfill(6)))
# if not os.path.exists(eval_subdir):
# os.mkdir(eval_subdir)
# path = os.path.join(eval_subdir, str(ei.idxs[0]).zfill(8))
# graph_handler.dump_eval(ei, path=path)
print(e.acc)
if config.dump_eval:
print("dumping eval ...")
graph_handler.dump_eval(e)
if config.dump_answer:
print("dumping answers ...")
graph_handler.dump_answer(e)
def _train(config):
np.set_printoptions(threshold=np.inf)
train_data = read_data(config, 'train', config.load)
dev_data = read_data(config, 'dev', True)
update_config(config, [train_data, dev_data])
_config_debug(config)
word2vec_dict = train_data.shared[
'lower_word2vec'] if config.lower_word else train_data.shared[
'word2vec']
word2idx_dict = train_data.shared['word2idx']
idx2vec_dict = {
word2idx_dict[word]: vec
for word, vec in word2vec_dict.items() if word in word2idx_dict
}
emb_mat = np.array([
idx2vec_dict[idx]
if idx in idx2vec_dict else np.random.multivariate_normal(
np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in range(config.word_vocab_size)
])
config.emb_mat = emb_mat
def make_idx2word():
"""
return index of the word from the preprocessed dictionary.
"""
idx2word = {}
d = train_data.shared['word2idx']
for word, idx in d.items():
print(word)
idx2word[idx] = word
if config.use_glove_for_unk:
d2 = train_data.shared['new_word2idx']
for word, idx in d2.items():
print(word)
idx2word[idx + len(d)] = word
return idx2word
idx2word = make_idx2word()
# Save total number of words used in this dictionary: words in GloVe + etc tokens(including UNK, POS, ... etc)
print("size of config.id2word len:", len(idx2word))
print("size of config.total_word_vocab_size:",
config.total_word_vocab_size)
# construct model graph and variables (using default graph)
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
print("num params: {}".format(get_num_params()))
trainer = MultiGPUTrainer(config, models)
evaluator = MultiGPUEvaluator(
config, models, tensor_dict=model.tensor_dict if config.vis else None)
graph_handler = GraphHandler(
config, model
) # controls all tensors and variables in the graph, including loading /saving
# Variables
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
num_steps = config.num_steps or int(
math.ceil(train_data.num_examples /
(config.batch_size * config.num_gpus))) * config.num_epochs
min_val = {}
min_val['loss'] = 100.0
min_val['acc'] = 0
min_val['step'] = 0
min_val['patience'] = 0
for batches in tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps,
shuffle=True,
cluster=config.cluster),
total=num_steps):
global_step = sess.run(
model.global_step
) + 1 # +1 because all calculations are done after step
get_summary = global_step % config.log_period == 0
loss, summary, train_op = trainer.step(sess,
batches,
get_summary=get_summary)
if get_summary:
graph_handler.add_summary(summary, global_step)
# occasional saving
if global_step % config.save_period == 0:
graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
num_steps = math.ceil(dev_data.num_examples /
(config.batch_size * config.num_gpus))
# num_steps: total steps to finish this training session.
# val_num_batches: 100
if 0 < config.val_num_batches < num_steps:
# if config.val_num_batches is less the the actual steps required to run whole dev set. Run evaluation up to the step.
num_steps = config.val_num_batches
# This train loss is calulated from sampling the same number of data size of dev_data.
e_train = evaluator.get_evaluation_from_batches(
sess,
tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
graph_handler.add_summaries(e_train.summaries, global_step)
# This e_dev may differ from the dev_set used in test time because some data is filtered out here.
e_dev = evaluator.get_evaluation_from_batches(
sess,
tqdm(dev_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
graph_handler.add_summaries(e_dev.summaries, global_step)
print("%s e_train: loss=%.4f" % (header, e_train.loss))
print("%s e_dev: loss=%.4f" % (header, e_dev.loss))
print()
if min_val['loss'] > e_dev.loss:
min_val['loss'] = e_dev.loss
min_val['step'] = global_step
min_val['patience'] = 0
else:
min_val['patience'] = min_val['patience'] + 1
if min_val['patience'] >= 1000:
slack.notify(
text="%s patience reached %d. early stopping." %
(header, min_val['patience']))
break
slack.notify(text="%s e_dev: loss=%.4f" % (header, e_dev.loss))
if config.dump_eval:
graph_handler.dump_eval(e_dev)
if config.dump_answer:
graph_handler.dump_answer(e_dev)
slack.notify(
text=
"%s <@U024BE7LH|insikk> Train is finished. e_dev: loss=%.4f at step=%d\nPlease assign another task to get more research result"
% (header, min_val['loss'], min_val['step']))
if global_step % config.save_period != 0:
graph_handler.save(sess, global_step=global_step)