def force_decode():
# force_decode it: generate a file which contains every score and the final score;
mylog_section("READ DATA")
#读入test数据,test不需要新建立词典，直接调用建立好的词典就可以了。
test_data_bucket, _buckets, test_data_order = read_test(
FLAGS.data_cache_dir, FLAGS.test_path,
get_vocab_path(FLAGS.data_cache_dir), FLAGS.L, FLAGS.n_bucket)
vocab_path = get_vocab_path(FLAGS.data_cache_dir)
real_vocab_size = get_real_vocab_size(vocab_path)
FLAGS._buckets = _buckets
FLAGS.real_vocab_size = real_vocab_size
test_bucket_sizes = [
len(test_data_bucket[b]) for b in range(len(_buckets))
]
test_total_size = int(sum(test_bucket_sizes))
# reports
mylog_section("REPORT")
mylog("real_vocab_size: {}".format(FLAGS.real_vocab_size))
mylog("_buckets:{}".format(FLAGS._buckets))
mylog("FORCE_DECODE:")
mylog("total: {}".format(test_total_size))
mylog("bucket_sizes: {}".format(test_bucket_sizes))
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = FLAGS.allow_growth
mylog_section("IN TENSORFLOW")
with tf.Session(config=config) as sess:
# runtime profile
if FLAGS.profile:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
else:
run_options = None
run_metadata = None
mylog("Creating Model")
model = create_model(sess, run_options, run_metadata)
mylog_section("All Variables")
show_all_variables()
sess.run(model.dropoutRate.assign(1.0))
batch_size = FLAGS.batch_size
mylog_section("Data Iterators")
dite = DataIterator(model,
test_data_bucket,
len(_buckets),
batch_size,
None,
data_order=test_data_order)
ite = dite.next_original()
fdump = open(FLAGS.score_file, 'w')
i_sent = 0
mylog_section("FORCE_DECODING")
for inputs, outputs, weights, bucket_id in ite:
# inputs: [[_GO],[1],[2],[3],[_EOS],[pad_id],[pad_id]]
# positions: [4]
mylog("--- decoding {}/{} sent ---".format(i_sent,
test_total_size))
i_sent += 1
L = model.step(sess,
inputs,
outputs,
weights,
bucket_id,
forward_only=True,
dump_lstm=False)
mylog("LOSS: {}".format(L))
fdump.write("{}\n".format(L))
fdump.close()
def train():
#1.读入train数据和dev数据
mylog_section('READ DATA')
train_data_bucket, dev_data_bucket, _buckets, vocab_path = read_train_dev(
FLAGS.data_cache_dir, FLAGS.train_path, FLAGS.dev_path,
FLAGS.vocab_size, FLAGS.L, FLAGS.n_bucket)
##########以下是打印需要的信息 start #####################
real_vocab_size = get_real_vocab_size(vocab_path)
FLAGS._buckets = _buckets
FLAGS.real_vocab_size = real_vocab_size
# 计算总共要处理的tokens个数
train_n_tokens = np.sum([
np.sum([len(sentence) for sentence in bucket])
for bucket in train_data_bucket
])
# train_data_bucket
train_bucket_sizes = [
len(train_data_bucket[index]) for index in xrange(len(_buckets))
]
train_total_size = float(sum(train_bucket_sizes))
# 计算累计值,用于计算bucket，在 data_iterator中随机生成一个0-1的数，这里的train_buckets_scale根据每个bucket中句子数量的不同，切分成不同的权重[0.1,0.3,0.5,0.8,1]
# 当随机的0-1的数落到上述权重的某个区间，那么就选哪个bucket。
train_buckets_scale = [
sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in xrange(len(train_bucket_sizes))
]
dev_bucket_sizes = [
len(dev_data_bucket[index]) for index in xrange(len(_buckets))
]
dev_total_size = int(sum(dev_bucket_sizes))
mylog_section("REPORT")
# steps
batch_size = FLAGS.batch_size
n_epoch = FLAGS.n_epoch
steps_per_epoch = int(train_total_size / batch_size)
steps_per_checkpoint = int(steps_per_epoch / 2) #每半个epoch 验证一次模型
total_steps = steps_per_epoch * n_epoch
# reports
mylog("real_vocab_size: {}".format(FLAGS.real_vocab_size))
mylog("_buckets: {}".format(FLAGS._buckets))
mylog("Train:")
mylog("total: {}".format(train_total_size))
mylog("bucket sizes: {}".format(train_bucket_sizes))
mylog("Dev:")
mylog("total: {}".format(dev_total_size))
mylog("bucket sizes: {}".format(dev_bucket_sizes))
mylog("Steps_per_epoch: {}".format(steps_per_epoch))
mylog("Total_steps:{}".format(total_steps))
mylog("Steps_per_checkpoint: {}".format(steps_per_checkpoint))
##########打印需要的信息 end #####################
mylog_section("IN TENSORFLOW")
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = FLAGS.allow_growth
with tf.Session(config=config) as sess:
# runtime profile
if FLAGS.profile:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
else:
run_options = None
run_metadata = None
mylog_section("MODEL/SUMMARY/WRITER")
mylog("Creating Model.. (this can take a few minutes)")
model = create_model(sess, run_options, run_metadata)
mylog_section("All Variables")
show_all_variables()
# Data Iterators
mylog_section("Data Iterators")
dite = DataIterator(model, train_data_bucket, len(train_buckets_scale),
batch_size, train_buckets_scale)
iteType = 0
if iteType == 0:
mylog("Itetype: withRandom")
ite = dite.next_random()
elif iteType == 1:
mylog("Itetype: withSequence")
ite = dite.next_sequence()
# statistics during training
step_time, loss = 0.0, 0.0
current_step = 0
low_ppx = float("inf")
steps_per_report = 30
n_targets_report = 0
report_time = 0
n_valid_sents = 0
n_valid_words = 0
patience = FLAGS.patience
mylog_section("TRAIN")
while current_step < total_steps:
# start
start_time = time.time()
# data and train
inputs, outputs, weights, bucket_id = ite.next() #训练数据
L = model.step(sess, inputs, outputs, weights, bucket_id)
# loss and time
step_time += (time.time() - start_time) / steps_per_checkpoint
loss += L
current_step += 1
# 此处 weights 等数据的格式是 len(weights) == 句子长度
# len(weights[0]) 是 batch size
n_valid_sents += np.sum(np.sign(weights[0]))
n_valid_words += np.sum(weights)
# for report
report_time += (time.time() - start_time)
n_targets_report += np.sum(weights)
#显示信息
if current_step % steps_per_report == 0:
sect_name = "STEP {}".format(current_step)
msg = "StepTime: {:.2f} sec Speed: {:.2f} targets/s Total_targets: {}".format(
report_time / steps_per_report,
n_targets_report * 1.0 / report_time, train_n_tokens)
mylog_line(sect_name, msg)
report_time = 0
n_targets_report = 0
# Create the Timeline object, and write it to a json
if FLAGS.profile:
tl = timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
with open('timeline.json', 'w') as f:
f.write(ctf)
exit()
#达到半个epoch，计算ppx(dev)
if current_step % steps_per_checkpoint == 0:
i_checkpoint = int(current_step / steps_per_checkpoint)
# train_ppx
loss = loss / n_valid_words
train_ppx = math.exp(
float(loss)) if loss < 300 else float("inf")
learning_rate = model.learning_rate.eval()
# dev_ppx
dev_loss, dev_ppx = evaluate(sess, model, dev_data_bucket)
# report
sect_name = "CHECKPOINT {} STEP {}".format(
i_checkpoint, current_step)
msg = "Learning_rate: {:.4f} Dev_ppx: {:.2f} Train_ppx: {:.2f}".format(
learning_rate, dev_ppx, train_ppx)
mylog_line(sect_name, msg)
# save model per checkpoint
if FLAGS.saveCheckpoint:
checkpoint_path = os.path.join(FLAGS.saved_model_dir,
"model")
s = time.time()
model.saver.save(sess,
checkpoint_path,
global_step=i_checkpoint,
write_meta_graph=False)
msg = "Model saved using {:.2f} sec at {}".format(
time.time() - s, checkpoint_path)
mylog_line(sect_name, msg)
# save best model
if dev_ppx < low_ppx:
patience = FLAGS.patience
low_ppx = dev_ppx
checkpoint_path = os.path.join(FLAGS.saved_model_dir,
"best")
s = time.time()
model.best_saver.save(sess,
checkpoint_path,
global_step=0,
write_meta_graph=False)
msg = "Model saved using {:.2f} sec at {}".format(
time.time() - s, checkpoint_path)
mylog_line(sect_name, msg)
else:
patience -= 1
#每次当 dev_ppx >= low_ppx时 学习步长减半
sess.run(model.learning_rate_decay_op)
msg = 'dev_ppx:{}, low_ppx:{}'.format(
str(dev_ppx), str(low_ppx))
mylog_line(sect_name, msg)
msg = 'dev_ppx >= low_ppx，patience ={}, learning_reate ={}'.format(
str(patience), str(model.learning_rate.eval()))
mylog_line(sect_name, msg)
if patience <= 0:
mylog("Training finished. Running out of patience.")
break
# Save checkpoint and zero timer and loss.
step_time, loss, n_valid_sents, n_valid_words = 0.0, 0.0, 0, 0
def beam_decode():
# not yet tested:
# known issues:
# should use next_original
mylog("Reading Data...")
test_data_bucket, _buckets, test_data_order = read_test(
FLAGS.data_cache_dir, FLAGS.test_path,
get_vocab_path(FLAGS.data_cache_dir), FLAGS.L, FLAGS.n_bucket)
vocab_path = get_vocab_path(FLAGS.data_cache_dir)
real_vocab_size = get_real_vocab_size(vocab_path)
FLAGS._buckets = _buckets
FLAGS.real_vocab_size = real_vocab_size
test_bucket_sizes = [
len(test_data_bucket[b]) for b in range(len(_buckets))
]
test_total_size = int(sum(test_bucket_sizes))
# reports
mylog("real_vocab_size: {}".format(FLAGS.real_vocab_size))
mylog("_buckets:{}".format(FLAGS._buckets))
mylog("BEAM_DECODE:")
mylog("total: {}".format(test_total_size))
mylog("buckets: {}".format(test_bucket_sizes))
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = FLAGS.allow_growth
with tf.Session(config=config) as sess:
# runtime profile
if FLAGS.profile:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
else:
run_options = None
run_metadata = None
mylog("Creating Model")
model = create_model(sess, run_options, run_metadata)
mylog("before init_beam_decoder()")
show_all_variables()
model.init_beam_decoder(beam_size=FLAGS.beam_size,
max_steps=FLAGS.beam_step)
model.init_beam_variables(sess)
mylog("after init_beam_decoder()")
show_all_variables()
sess.run(model.dropoutRate.assign(1.0))
start_id = 0
n_steps = 0
batch_size = FLAGS.batch_size
dite = DataIterator(model, test_data_bucket, len(_buckets), batch_size,
None)
ite = dite.next_sequence(stop=True, test=True)
i_sent = 0
for inputs, positions, valids, bucket_id in ite:
# user : [0]
# inputs: [[_GO],[1],[2],[3],[_EOS],[pad_id],[pad_id]]
# positions: [4]
print("--- decoding {}/{} sent ---".format(i_sent, n_total_user))
i_sent += 1
# do the following convert:
# inputs: [[pad_id],[1],[2],[pad_id],[pad_id],[pad_id]]
# positions:[2]
PAD_ID = 0
last_history = inputs[positions[0]]
inputs_beam = [last_history * FLAGS.beam_size]
inputs[positions[0]] = list([PAD_ID] * FLAGS.beam_size)
inputs[positions[0] - 1] = list([PAD_ID] * FLAGS.beam_size)
positions[0] = positions[0] - 2 if positions[0] >= 2 else 0
scores = [0.0] * FLAGS.beam_size
sentences = [[] for x in range(FLAGS.beam_size)]
beam_parent = range(FLAGS.beam_size)
for i in range(FLAGS.beam_step):
if i == 0:
top_value, top_index = model.beam_step(
sess,
index=i,
word_inputs_history=inputs,
sequence_length=positions,
word_inputs_beam=inputs_beam)
else:
top_value, top_index = model.beam_step(
sess,
index=i,
word_inputs_beam=inputs_beam,
beam_parent=beam_parent)
# expand
global_queue = []
if i == 0:
nrow = 1
else:
nrow = top_index[0].shape[0]
for row in range(nrow):
for col in range(top_index[0].shape[1]):
score = scores[row] + np.log(top_value[0][row, col])
word_index = top_index[0][row, col]
beam_index = row
if FLAGS.no_repeat:
if not word_index in sentences[beam_index]:
global_queue.append(
(score, beam_index, word_index))
else:
global_queue.append(
(score, beam_index, word_index))
global_queue = sorted(global_queue, key=lambda x: -x[0])
inputs_beam = []
beam_parent = []
scores = []
temp_sentences = []
if FLAGS.print_beam:
print("--------- Step {} --------".format(i))
for j, (score, beam_index, word_index) in enumerate(
global_queue[:FLAGS.beam_size]):
if FLAGS.print_beam:
print("Beam:{} Father:{} word:{} score:{}".format(
j, beam_index, word_index, score))
beam_parent.append(beam_index)
inputs_beam.append(word_index)
scores.append(score)
temp_sentences.append(sentences[beam_index] + [word_index])
inputs_beam = [inputs_beam]
sentences = temp_sentences
if FLAGS.print_beam:
print(sentences)
def dump_lstm():
# dump the hidden states to some where
mylog_section("READ DATA")
test_data_bucket, _buckets, test_data_order = read_test(
FLAGS.data_cache_dir, FLAGS.test_path,
get_vocab_path(FLAGS.data_cache_dir), FLAGS.L, FLAGS.n_bucket)
vocab_path = get_vocab_path(FLAGS.data_cache_dir)
real_vocab_size = get_real_vocab_size(vocab_path)
FLAGS._buckets = _buckets
FLAGS.real_vocab_size = real_vocab_size
test_bucket_sizes = [
len(test_data_bucket[b]) for b in range(len(_buckets))
]
test_total_size = int(sum(test_bucket_sizes))
# reports
mylog_section("REPORT")
mylog("real_vocab_size: {}".format(FLAGS.real_vocab_size))
mylog("_buckets:{}".format(FLAGS._buckets))
mylog("DUMP_LSTM:")
mylog("total: {}".format(test_total_size))
mylog("buckets: {}".format(test_bucket_sizes))
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = FLAGS.allow_growth
with tf.Session(config=config) as sess:
# runtime profile
if FLAGS.profile:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
else:
run_options = None
run_metadata = None
mylog_section("MODEL")
mylog("Creating Model")
model = create_model(sess, run_options, run_metadata)
mylog("Init tensors to dump")
model.init_dump_states()
# dump_graph('graph.txt')
mylog_section("All Variables")
show_all_variables()
sess.run(model.dropoutRate.assign(1.0))
start_id = 0
n_steps = 0
batch_size = FLAGS.batch_size
mylog_section("Data Iterators")
dite = DataIterator(model,
test_data_bucket,
len(_buckets),
batch_size,
None,
data_order=test_data_order)
ite = dite.next_original()
fdump = open(FLAGS.dump_file, 'wb')
mylog_section("DUMP_LSTM")
i_sent = 0
for inputs, outputs, weights, bucket_id in ite:
# inputs: [[_GO],[1],[2],[3],[_EOS],[pad_id],[pad_id]]
# positions: [4]
mylog("--- decoding {}/{} sent ---".format(i_sent,
test_total_size))
i_sent += 1
# print(inputs)
# print(outputs)
# print(weights)
# print(bucket_id)
L, states = model.step(sess,
inputs,
outputs,
weights,
bucket_id,
forward_only=True,
dump_lstm=True)
mylog("LOSS: {}".format(L))
sw = StateWrapper()
sw.create(inputs, outputs, weights, states)
sw.save_to_stream(fdump)
# do the following convert:
# inputs: [[pad_id],[1],[2],[pad_id],[pad_id],[pad_id]]
# positions:[2]
fdump.close()
def train():
# Read Data
mylog_section("READ DATA")
train_data_bucket, dev_data_bucket, _buckets, vocab_path = read_train_dev(
FLAGS.data_cache_dir, FLAGS.train_path, FLAGS.dev_path,
FLAGS.vocab_size, FLAGS.L, FLAGS.n_bucket)
# 执行到此处, train_data_bucket，dev_data_bucket，_buckets 长度相同
# train_data_bucket，dev_data_bucket 都是 [b1,b2,b3, ..., bn] 格式
# 每个 bi 中都是化为数字的 sentence
# _buckets [2,4,5] 类似，是分割的句子长度
real_vocab_size = get_real_vocab_size(vocab_path)
FLAGS._buckets = _buckets
FLAGS.real_vocab_size = real_vocab_size
# 计算总共要处理的tokens个数
train_n_tokens = np.sum(
[np.sum([len(items) for items in x]) for x in train_data_bucket])
# train_data_bucket
train_bucket_sizes = [
len(train_data_bucket[b]) for b in range(len(_buckets))
]
train_total_size = float(sum(train_bucket_sizes))
# 计算累计值
train_buckets_scale = [
sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in range(len(train_bucket_sizes))
]
dev_bucket_sizes = [len(dev_data_bucket[b]) for b in range(len(_buckets))]
dev_total_size = int(sum(dev_bucket_sizes))
mylog_section("REPORT")
# steps
batch_size = FLAGS.batch_size
n_epoch = FLAGS.n_epoch
steps_per_epoch = int(train_total_size / batch_size)
steps_per_dev = int(dev_total_size / batch_size)
steps_per_checkpoint = int(steps_per_epoch / 2)
total_steps = steps_per_epoch * n_epoch
# reports
mylog("real_vocab_size: {}".format(FLAGS.real_vocab_size))
mylog("_buckets: {}".format(FLAGS._buckets))
mylog("Train:")
mylog("total: {}".format(train_total_size))
mylog("bucket sizes: {}".format(train_bucket_sizes))
mylog("Dev:")
mylog("total: {}".format(dev_total_size))
mylog("bucket sizes: {}".format(dev_bucket_sizes))
mylog("Steps_per_epoch: {}".format(steps_per_epoch))
mylog("Total_steps:{}".format(total_steps))
mylog("Steps_per_checkpoint: {}".format(steps_per_checkpoint))
mylog_section("IN TENSORFLOW")
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = FLAGS.allow_growth
with tf.Session(config=config) as sess:
# runtime profile
if FLAGS.profile:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
else:
run_options = None
run_metadata = None
mylog_section("MODEL/SUMMARY/WRITER")
mylog("Creating Model.. (this can take a few minutes)")
model = create_model(sess, run_options, run_metadata)
mylog("Creating ModelSummary")
modelSummary = ModelSummary()
mylog("Creating tf.summary.FileWriter")
summaryWriter = tf.summary.FileWriter(
os.path.join(FLAGS.summary_dir, "train.summary"), sess.graph)
mylog_section("All Variables")
show_all_variables()
# Data Iterators
mylog_section("Data Iterators")
dite = DataIterator(model, train_data_bucket, len(train_buckets_scale),
batch_size, train_buckets_scale)
iteType = 0
if iteType == 0:
mylog("Itetype: withRandom")
ite = dite.next_random()
elif iteType == 1:
mylog("Itetype: withSequence")
ite = dite.next_sequence()
# statistics during training
step_time, loss = 0.0, 0.0
current_step = 0
previous_losses = []
low_ppx = float("inf")
low_ppx_step = 0
steps_per_report = 30
n_targets_report = 0
report_time = 0
n_valid_sents = 0
n_valid_words = 0
patience = FLAGS.patience
mylog_section("TRAIN")
while current_step < total_steps:
# start
start_time = time.time()
# data and train
inputs, outputs, weights, bucket_id = next(ite)
L = model.step(sess, inputs, outputs, weights, bucket_id)
# loss and time
step_time += (time.time() - start_time) / steps_per_checkpoint
loss += L
current_step += 1
# 此处 weights 等数据的格式是 len(weights) == 句子长度
# len(weights[0]) 是 batch size
n_valid_sents += np.sum(np.sign(weights[0]))
n_valid_words += np.sum(weights)
# for report
report_time += (time.time() - start_time)
n_targets_report += np.sum(weights)
if current_step % steps_per_report == 0:
sect_name = "STEP {}".format(current_step)
msg = "StepTime: {:.2f} sec Speed: {:.2f} targets/s Total_targets: {}".format(
report_time / steps_per_report,
n_targets_report * 1.0 / report_time, train_n_tokens)
mylog_line(sect_name, msg)
report_time = 0
n_targets_report = 0
# Create the Timeline object, and write it to a json
if FLAGS.profile:
tl = timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
with open('timeline.json', 'w') as f:
f.write(ctf)
exit()
if current_step % steps_per_checkpoint == 0:
i_checkpoint = int(current_step / steps_per_checkpoint)
# train_ppx
loss = loss / n_valid_words
train_ppx = math.exp(
float(loss)) if loss < 300 else float("inf")
learning_rate = model.learning_rate.eval()
# dev_ppx
dev_loss, dev_ppx = evaluate(sess, model, dev_data_bucket)
# report
sect_name = "CHECKPOINT {} STEP {}".format(
i_checkpoint, current_step)
msg = "Learning_rate: {:.4f} Dev_ppx: {:.2f} Train_ppx: {:.2f}".format(
learning_rate, dev_ppx, train_ppx)
mylog_line(sect_name, msg)
# save summary
_summaries = modelSummary.step_record(sess, train_ppx, dev_ppx)
for _summary in _summaries:
summaryWriter.add_summary(_summary, i_checkpoint)
# save model per checkpoint
if FLAGS.saveCheckpoint:
checkpoint_path = os.path.join(FLAGS.saved_model_dir,
"model")
s = time.time()
model.saver.save(sess,
checkpoint_path,
global_step=i_checkpoint,
write_meta_graph=False)
msg = "Model saved using {:.2f} sec at {}".format(
time.time() - s, checkpoint_path)
mylog_line(sect_name, msg)
# save best model
if dev_ppx < low_ppx:
patience = FLAGS.patience
low_ppx = dev_ppx
low_ppx_step = current_step
checkpoint_path = os.path.join(FLAGS.saved_model_dir,
"best")
s = time.time()
model.best_saver.save(sess,
checkpoint_path,
global_step=0,
write_meta_graph=False)
msg = "Model saved using {:.2f} sec at {}".format(
time.time() - s, checkpoint_path)
mylog_line(sect_name, msg)
else:
patience -= 1
if patience <= 0:
mylog("Training finished. Running out of patience.")
break
# Save checkpoint and zero timer and loss.
step_time, loss, n_valid_sents, n_valid_words = 0.0, 0.0, 0, 0
