def main(unused_argv):
assert FLAGS.input_file_pattern, "--input_file_pattern is required"
assert FLAGS.train_dir, "--train_dir is required"
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
# Create training directory.
train_dir = FLAGS.train_dir
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
# Build the TensorFlow graph.
g = tf.Graph()
with g.as_default():
# Build the model.
model = auto_encoder_model.Auto_Encoder_Model(model_config,
mode="train")
model.build()
graph_handler = GraphHandler(model_config, model)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
queue_runner = tf.train.start_queue_runners(sess=sess, coord=coord)
graph_handler.initialize(sess)
for _ in tqdm(range(1, model_config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op, output = sess.run(
[model.total_loss, model.train_op, model.outputs_])
if global_step % model_config.period == 0:
summary_op = tf.summary.merge_all()
summaries = sess.run(summary_op)
graph_handler.writer.add_summary(summaries, global_step)
graph_handler.writer.flush()
if global_step % model_config.checkpoint == 0:
filename = os.path.join(
model_config.save_dir,
"{}_{}.ckpt".format(model_config.model_name,
global_step))
graph_handler.save(sess, filename)
coord.join(queue_runner)
def train(config):
gpu_options = tf.GPUOptions(visible_device_list="2")
sess_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
sess_config.gpu_options.allow_growth = True
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
train_dataset = get_batch_dataset(config.train_record_file, parser, config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(handle,
train_dataset.output_types,
train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat)
graph_handler = GraphHandler(
config, model
) # controls all tensors and variables in the graph, including loading /saving
loss_save = 100.0
patience = 0
lr = config.init_lr
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
graph_handler.initialize(sess)
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool)))
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
print("Started training")
for _ in tqdm(range(1, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={handle: train_handle})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
graph_handler.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
sess.run(
tf.assign(model.is_train, tf.constant(False,
dtype=tf.bool)))
_, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle)
for s in summ:
graph_handler.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1, dev_eval_file,
sess, "dev", handle, dev_handle)
sess.run(
tf.assign(model.is_train, tf.constant(True,
dtype=tf.bool)))
dev_loss = metrics["loss"]
if dev_loss < loss_save:
loss_save = dev_loss
patience = 0
else:
patience += 1
if patience >= config.patience:
lr /= 2.0
loss_save = dev_loss
patience = 0
sess.run(tf.assign(model.lr, tf.constant(lr,
dtype=tf.float32)))
graph_handler.add_summaries(summ, global_step)
graph_handler.writer.flush()
filename = os.path.join(
config.save_dir,
"{}_{}.ckpt".format(config.model_name, global_step))
graph_handler.save(sess, filename)
def _train(config):
np.set_printoptions(threshold=np.inf)
train_data = read_data(config, 'train', config.load)
dev_data = read_data(config, 'dev', True)
update_config(config, [train_data, dev_data])
_config_debug(config)
word2vec_dict = train_data.shared[
'lower_word2vec'] if config.lower_word else train_data.shared[
'word2vec']
word2idx_dict = train_data.shared['word2idx']
idx2vec_dict = {
word2idx_dict[word]: vec
for word, vec in word2vec_dict.items() if word in word2idx_dict
}
emb_mat = np.array([
idx2vec_dict[idx]
if idx in idx2vec_dict else np.random.multivariate_normal(
np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in range(config.word_vocab_size)
])
config.emb_mat = emb_mat
def make_idx2word():
"""
return index of the word from the preprocessed dictionary.
"""
idx2word = {}
d = train_data.shared['word2idx']
for word, idx in d.items():
print(word)
idx2word[idx] = word
if config.use_glove_for_unk:
d2 = train_data.shared['new_word2idx']
for word, idx in d2.items():
print(word)
idx2word[idx + len(d)] = word
return idx2word
idx2word = make_idx2word()
# Save total number of words used in this dictionary: words in GloVe + etc tokens(including UNK, POS, ... etc)
print("size of config.id2word len:", len(idx2word))
print("size of config.total_word_vocab_size:",
config.total_word_vocab_size)
# construct model graph and variables (using default graph)
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
print("num params: {}".format(get_num_params()))
trainer = MultiGPUTrainer(config, models)
evaluator = MultiGPUEvaluator(
config, models, tensor_dict=model.tensor_dict if config.vis else None)
graph_handler = GraphHandler(
config, model
) # controls all tensors and variables in the graph, including loading /saving
# Variables
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
num_steps = config.num_steps or int(
math.ceil(train_data.num_examples /
(config.batch_size * config.num_gpus))) * config.num_epochs
min_val = {}
min_val['loss'] = 100.0
min_val['acc'] = 0
min_val['step'] = 0
min_val['patience'] = 0
for batches in tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps,
shuffle=True,
cluster=config.cluster),
total=num_steps):
global_step = sess.run(
model.global_step
) + 1 # +1 because all calculations are done after step
get_summary = global_step % config.log_period == 0
loss, summary, train_op = trainer.step(sess,
batches,
get_summary=get_summary)
if get_summary:
graph_handler.add_summary(summary, global_step)
# occasional saving
if global_step % config.save_period == 0:
graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
num_steps = math.ceil(dev_data.num_examples /
(config.batch_size * config.num_gpus))
# num_steps: total steps to finish this training session.
# val_num_batches: 100
if 0 < config.val_num_batches < num_steps:
# if config.val_num_batches is less the the actual steps required to run whole dev set. Run evaluation up to the step.
num_steps = config.val_num_batches
# This train loss is calulated from sampling the same number of data size of dev_data.
e_train = evaluator.get_evaluation_from_batches(
sess,
tqdm(train_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
graph_handler.add_summaries(e_train.summaries, global_step)
# This e_dev may differ from the dev_set used in test time because some data is filtered out here.
e_dev = evaluator.get_evaluation_from_batches(
sess,
tqdm(dev_data.get_multi_batches(config.batch_size,
config.num_gpus,
num_steps=num_steps),
total=num_steps))
graph_handler.add_summaries(e_dev.summaries, global_step)
print("%s e_train: loss=%.4f" % (header, e_train.loss))
print("%s e_dev: loss=%.4f" % (header, e_dev.loss))
print()
if min_val['loss'] > e_dev.loss:
min_val['loss'] = e_dev.loss
min_val['step'] = global_step
min_val['patience'] = 0
else:
min_val['patience'] = min_val['patience'] + 1
if min_val['patience'] >= 1000:
slack.notify(
text="%s patience reached %d. early stopping." %
(header, min_val['patience']))
break
slack.notify(text="%s e_dev: loss=%.4f" % (header, e_dev.loss))
if config.dump_eval:
graph_handler.dump_eval(e_dev)
if config.dump_answer:
graph_handler.dump_answer(e_dev)
slack.notify(
text=
"%s <@U024BE7LH|insikk> Train is finished. e_dev: loss=%.4f at step=%d\nPlease assign another task to get more research result"
% (header, min_val['loss'], min_val['step']))
if global_step % config.save_period != 0:
graph_handler.save(sess, global_step=global_step)
def _train(config):
word2idx = Counter(json.load(open("../data/{}/word2idx_{}.json".format(config.data_from, config.data_from), "r"))["word2idx"])
idx2word = json.load(open("../data/{}/word2idx_{}.json".format(config.data_from, config.data_from), "r"))["idx2word"]
assert len(word2idx) == len(idx2word)
for i in range(10): assert word2idx[idx2word[i]] == i
vocab_size = len(word2idx)
print("vocab_size", vocab_size, idx2word[:10])
word2vec = Counter(json.load(open("../data/{}/word2vec_{}.json".format(config.data_from, config.pretrain_from), "r"))["word2vec"])
# word2vec = {} if config.debug or config.load else get_word2vec(config, word2idx)
idx2vec = {word2idx[word]: vec for word, vec in word2vec.items() if word in word2idx}
print("no unk words:", len(idx2vec))
unk_embedding = np.random.multivariate_normal(np.zeros(config.word_embedding_size), np.eye(config.word_embedding_size))
config.emb_mat = np.array([idx2vec[idx] if idx in idx2vec else unk_embedding for idx in range(vocab_size)])
config.vocab_size = vocab_size
print("emb_mat:", config.emb_mat.shape)
test_type = "test"
if config.data_from == "ice":
test_type = "dev"
else:
test_type = "test"
train_dict, test_dict = {}, {}
ice_flat = ""
if config.data_from == "ice" and config.model_name.endswith("flat"):
ice_flat = "_flat"
if os.path.exists("../data/{}/{}_{}{}{}.json".format(config.data_from, config.data_from, "train", ice_flat, config.clftype)):
train_dict = json.load(open("../data/{}/{}_{}{}{}.json".format(config.data_from, config.data_from, "train", ice_flat, config.clftype), "r"))
if os.path.exists("../data/{}/{}_{}{}{}.json".format(config.data_from, config.data_from, test_type, ice_flat, config.clftype)):
test_dict = json.load(open("../data/{}/{}_{}{}{}.json".format(config.data_from, config.data_from, test_type, ice_flat, config.clftype), "r"))
# check
for key, val in train_dict.items():
if isinstance(val[0], list) and len(val[0])>10: print(key, val[0][:50])
else: print(key, val[0:4])
print("train:", len(train_dict))
print("test:", len(test_dict))
if config.data_from == "reuters":
train_data = DataSet(train_dict, "train") if len(train_dict)>0 else read_reuters(config, data_type="train", word2idx=word2idx)
dev_data = DataSet(test_dict, "test") if len(test_dict)>0 else read_reuters(config, data_type="test", word2idx=word2idx)
elif config.data_from == "20newsgroup":
train_data = DataSet(train_dict, "train") if len(train_dict)>0 else read_news(config, data_type="train", word2idx=word2idx)
dev_data = DataSet(test_dict, "test") if len(test_dict)>0 else read_news(config, data_type="test", word2idx=word2idx)
elif config.data_from == "ice":
train_data = DataSet(train_dict, "train")
dev_data = DataSet(test_dict, "dev")
config.train_size = train_data.get_data_size()
config.dev_size = dev_data.get_data_size()
print("train/dev:", config.train_size, config.dev_size)
# calculate doc length
# TO CHECK
avg_len = 0
for d_l in train_dict["x_len"]:
avg_len += d_l/config.train_size
print("avg_len at train:", avg_len)
if config.max_docs_length > 2000: config.max_docs_length = 2000
pprint(config.__flags, indent=2)
model = get_model(config)
trainer = Trainer(config, model)
graph_handler = GraphHandler(config, model)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
num_batches = config.num_batches or int(math.ceil(train_data.num_examples / config.batch_size)) * config.num_epochs
global_step = 0
dev_evaluate = Evaluator(config, model)
best_f1 = 0.50
for batch in tqdm(train_data.get_batches(config.batch_size, num_batches=num_batches, shuffle=True, cluster=config.cluster), total=num_batches):
global_step = sess.run(model.global_step) + 1
# print("global_step:", global_step)
get_summary = global_step % config.log_period
loss, summary, train_op = trainer.step(sess, batch, get_summary)
if get_summary:
graph_handler.add_summary(summary, global_step)
# occasional saving
# if global_step % config.save_period == 0 :
# graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
#config.test_batch_size = config.dev_size/3
num_steps = math.ceil(dev_data.num_examples / config.test_batch_size)
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
# print("num_steps:", num_steps)
e_dev = dev_evaluate.get_evaluation_from_batches(
sess, tqdm(dev_data.get_batches(config.test_batch_size, num_batches=num_steps), total=num_steps))
if e_dev.fv > best_f1:
best_f1 = e_dev.fv
#if global_step % config.save_period == 0:
graph_handler.save(sess, global_step=global_step)
graph_handler.add_summaries(e_dev.summaries, global_step)
print("f1:", best_f1)
# model hot start
if (MODE == 'dcgan') or (MODE == 'lsgan'):
disc_iters = 1
else:
disc_iters = CRITIC_ITERS
for i in range(disc_iters):
_disc_cost, _ = session.run([disc_cost, disc_train_op])
if MODE == 'wgan':
_ = session.run([clip_disc_weights])
# if iteration % config.period == 0:
# loss_sum = tf.Summary(value=[tf.Summary.Value(
# tag="model/loss", simple_value=_disc_cost), ])
# graph_handler.add_summary(loss_sum, global_step)
lib.plot.plot('train disc cost', _disc_cost)
lib.plot.plot('time', time.time() - start_time)
if (iteration < 5) or (iteration % 100 == 0):
generate_image(iteration)
lib.plot.flush()
lib.plot.tick()
if iteration % config.checkpoint == 0:
graph_handler.writer.flush()
filename = os.path.join(
config.save_dir,
"{}_{}.ckpt".format(config.model_name, global_step))
graph_handler.save(session, filename)
def _train(config):
train_data = read_data(config, 'val_train', config.load)
dev_data = read_data(config, 'val_val', True)
# test = read_data(config, 'test', True)
update_config(config, [train_data, dev_data])
_config_debug(config)
word2vec_dict = train_data.shared['lower_word2vec'] if config.lower_word else train_data.shared['word2vec']
word2idx_dict = train_data.shared['word2idx']
idx2vec_dict = {word2idx_dict[word]: vec for word, vec in word2vec_dict.items() if word in word2idx_dict}
emb_mat = np.array([idx2vec_dict[idx] if idx in idx2vec_dict
else np.random.multivariate_normal(np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in range(config.word_vocab_size)])
config.emb_mat = emb_mat
pprint(config.__flags, indent=2)
models = get_multi_gpu_models(config)
model = models[0]
print("num params: {}".format(get_num_params()))
trainer = MultiGPUTrainer(config, models)
evaluator = AccuracyEvaluator(config.train_num_can, config, model,
tensor_dict=model.tensor_dict if config.vis else None)
graph_handler = GraphHandler(config,
model) # controls all tensors and variables in the graph, including loading /saving
# Variables
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
# Begin training
num_steps = config.num_steps or int(
math.ceil(train_data.num_examples / (config.batch_size * config.num_gpus))) * config.num_epochs
global_step = 0
best_dev=[0,0]
for batches in tqdm(train_data.get_multi_batches(config.batch_size, config.num_gpus,
num_steps=num_steps, shuffle=False, cluster=config.cluster), total=num_steps):
global_step = sess.run(model.global_step) + 1 # +1 because all calculations are done after step
get_summary = global_step % config.log_period == 0
loss, summary, train_op = trainer.step(sess, batches, get_summary=get_summary)
if get_summary:
graph_handler.add_summary(summary, global_step)
if not config.eval:
continue
if global_step % config.eval_period == 0:
num_steps_dev = math.ceil(dev_data.num_examples / (config.batch_size * config.num_gpus))
num_steps_train = math.ceil(train_data.num_examples / (config.batch_size * config.num_gpus))
e_train = evaluator.get_evaluation_from_batches(
sess, tqdm(train_data.get_multi_batches(config.batch_size, config.num_gpus, num_steps=num_steps_train),
total=num_steps_train)
)
# graph_handler.add_summaries(e_test.summaries, global_step)
e_dev = evaluator.get_evaluation_from_batches(
sess, tqdm(dev_data.get_multi_batches(config.batch_size, config.num_gpus, num_steps=num_steps_dev),
total=num_steps_dev))
# graph_handler.dump_eval(e)
# graph_handler.add_summaries(e_dev.summaries, global_step)
print('train step:{} loss:{} acc:{}'.format(global_step, e_train.loss, e_train.acc))
print('val step:{} loss:{} acc:{}'.format(global_step, e_dev.loss, e_dev.acc))
# print('w_s:{}'.format(w_s))
if global_step > 700:
config.save_period = 50
config.eval_period = 50
if best_dev[0] < e_dev.acc:
best_dev=[e_dev.acc,global_step,e_train.acc]
graph_handler.save(sess, global_step=global_step)
# if config.dump_eval:
# graph_handler.dump_eval(e_dev)
if global_step % config.save_period != 0:
graph_handler.save(sess, global_step=global_step)
print (best_dev)
print ("you can test on test data set and set load setp is {}".format(best_dev[1]))
def _train(config):
word2idx = Counter(
json.load(
open(
"data/{}/word2idx_{}.json".format(config.data_from,
config.data_from),
"r"))["word2idx"])
vocab_size = len(word2idx)
print("vocab_size", vocab_size)
word2vec = Counter(
json.load(
open(
"data/{}/word2vec_{}.json".format(config.data_from,
config.pretrain_from),
"r"))["word2vec"])
# word2vec = {} if config.debug or config.load else get_word2vec(config, word2idx)
idx2vec = {
word2idx[word]: vec
for word, vec in word2vec.items() if word in word2idx and word != "UNK"
}
unk_embedding = np.random.multivariate_normal(
np.zeros(config.word_embedding_size),
np.eye(config.word_embedding_size))
config.emb_mat = np.array([
idx2vec[idx] if idx in idx2vec else unk_embedding
for idx in range(vocab_size)
])
config.vocab_size = vocab_size
print("emb_mat:", config.emb_mat.shape)
train_dict, test_dict = {}, {}
if os.path.exists("data/{}/{}_{}.json".format(config.data_from,
config.data_from, "train")):
train_dict = json.load(
open(
"data/{}/{}_{}.json".format(config.data_from, config.data_from,
"train"), "r"))
if os.path.exists("data/{}/{}_{}.json".format(config.data_from,
config.data_from, "test")):
test_dict = json.load(
open(
"data/{}/{}_{}.json".format(config.data_from, config.data_from,
"test"), "r"))
# check
if config.data_from == "reuters":
train_data = DataSet(train_dict,
"train") if len(train_dict) > 0 else read_reuters(
config, data_type="train", word2idx=word2idx)
dev_data = DataSet(test_dict,
"test") if len(test_dict) > 0 else read_reuters(
config, data_type="test", word2idx=word2idx)
elif config.data_from == "20newsgroup":
train_data = DataSet(train_dict,
"train") if len(train_dict) > 0 else read_news(
config, data_type="train", word2idx=word2idx)
dev_data = DataSet(test_dict,
"test") if len(test_dict) > 0 else read_news(
config, data_type="test", word2idx=word2idx)
config.train_size = train_data.get_data_size()
config.dev_size = dev_data.get_data_size()
print("train/dev:", config.train_size, config.dev_size)
if config.max_docs_length > 2000: config.max_docs_length = 2000
pprint(config.__flags, indent=2)
model = get_model(config)
graph_handler = GraphHandler(config, model)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
num_batches = config.num_batches or int(
math.ceil(
train_data.num_examples / config.batch_size)) * config.num_epochs
global_step = 0
dev_evaluate = Evaluator(config, model)
for batch in tqdm(train_data.get_batches(config.batch_size,
num_batches=num_batches,
shuffle=True,
cluster=config.cluster),
total=num_batches):
batch_idx, batch_ds = batch
'''
if config.debug:
for key, value in batch_ds.data.items():
if not key.startswith("x"):
print(key, value)
continue
'''
global_step = sess.run(model.global_step) + 1
# print("global_step:", global_step)
get_summary = global_step % config.log_period
feed_dict = model.get_feed_dict(batch, config)
logits, y, y_len, loss, summary, train_op = sess.run(
[
model.logits, model.y, model.y_seq_length, model.loss,
model.summary, model.train_op
],
feed_dict=feed_dict)
#print("logits:", logits[0:3], y[0:3], y_len[0:3], logits.shape, y.shape, y_len.shape)
print("loss:", loss)
if get_summary:
graph_handler.add_summary(summary, global_step)
# occasional saving
if global_step % config.save_period == 0:
graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
#config.test_batch_size = config.dev_size/3
num_steps = math.ceil(dev_data.num_examples /
config.test_batch_size)
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
# print("num_steps:", num_steps)
e_dev = dev_evaluate.get_evaluation_from_batches(
sess,
tqdm(dev_data.get_batches(config.test_batch_size,
num_batches=num_steps),
total=num_steps))
graph_handler.add_summaries(e_dev.summaries, global_step)