def main(_):
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config)
with tf.variable_scope("model", reuse=True, initializer=initializer):
mvalid = PTBModel(is_training=False, config=config)
mtest = PTBModel(is_training=False, config=eval_config)
tf.initialize_all_variables().run()
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session, m, train_data, m.train_op,
verbose=True)
print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
test_perplexity = run_epoch(session, mtest, test_data, tf.no_op())
print("Test Perplexity: %.3f" % test_perplexity)
def load_data(seqlength=20):
raw_data = reader.ptb_raw_data('data/ptb')
train_data, val_data, test_data, _ = raw_data
train_data = indices_to_seq_data(train_data, seqlength)
val_data = indices_to_seq_data(val_data, seqlength)
return {'train': train_data,
'test': val_data}
def main(unused_args):
if not FLAGS.data_path:
raise ValueError("Must specify --data_path to PTB data directory")
if not FLAGS.save_path:
raise ValueError("Must specify --save_path to model directory")
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config)
with tf.variable_scope("model", reuse=True, initializer=initializer):
mvalid = PTBModel(is_training=False, config=config)
mtest = PTBModel(is_training=False, config=eval_config)
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
ckpt=tf.train.get_checkpoint_state(FLAGS.save_path)
if (ckpt):
print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
saver.restore(session, ckpt.model_checkpoint_path)
else:
print("Created model with fresh parameters.")
tf.initialize_all_variables().run()
if not FLAGS.testonly:
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session, m, train_data, m.train_op,
verbose=True)
print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
save_path = saver.save(session, FLAGS.save_path+'/model.ckpt',i)
print("Model saved in: %s" % save_path)
valid_perplexity = run_epoch(session, mvalid, valid_data, tf.no_op())
print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
else:
print("Running only a perplexity test")
test_perplexity = run_epoch(session, mtest, test_data, tf.no_op(),verbose=True)
print("Test Perplexity: %.3f" % test_perplexity)
def testPtbRawData(self):
tmpdir = tf.test.get_temp_dir()
for suffix in "train", "valid", "test":
filename = os.path.join(tmpdir, "ptb.%s.txt" % suffix)
with tf.gfile.GFile(filename, "w") as fh:
fh.write(self._string_data)
# Smoke test
output = reader.ptb_raw_data(tmpdir)
self.assertEqual(len(output), 4)
def main(_):
if not FLAGS.data_path:
# raise ValueError("Must set --data_path to PTB data directory")
FLAGS.data_path = 'data/'
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, My_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config)
# with tf.variable_scope("model", reuse=True, initializer=initializer):
# mvalid = PTBModel(is_training=False, config=config)
# mMy = PTBModel(is_training=False, config=eval_config)
summary_op = tf.merge_all_summaries()
saver = tf.train.Saver(tf.all_variables())
tf.initialize_all_variables().run()
summary_writer = tf.train.SummaryWriter(FLAGS.data_path,
graph_def=session.graph_def)
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session,
m,
train_data,
m.train_op,
summary_writer,
summary_op,
verbose=True)
# print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
# valid_perplexity = run_epoch(session, mvalid, valid_data, tf.no_op())
# print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
#
# My_perplexity = run_epoch(session, mMy, My_data, tf.no_op())
# print("My Perplexity: %.3f" % My_perplexity)
if i % 20 == 0:
print('Now perplexity %.3f' % (train_perplexity))
print("SAVEING:")
checkpoint_path = os.path.join(FLAGS.data_path, 'model.ckpt')
saver.save(sess=session, save_path=checkpoint_path, global_step=i)
print("save model to {}".format(checkpoint_path))
def main(_):
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default():
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.name_scope("Train"):
train_input = PTBInput(config=config, data=train_data, name="TrainInput")
with tf.variable_scope("Model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config, input_=train_input)
tf.scalar_summary("Training Loss", m.cost)
tf.scalar_summary("Learning Rate", m.lr)
with tf.name_scope("Valid"):
valid_input = PTBInput(config=config, data=valid_data, name="ValidInput")
with tf.variable_scope("Model", reuse=True, initializer=initializer):
mvalid = PTBModel(is_training=False, config=config, input_=valid_input)
tf.scalar_summary("Validation Loss", mvalid.cost)
with tf.name_scope("Test"):
test_input = PTBInput(config=eval_config, data=test_data, name="TestInput")
with tf.variable_scope("Model", reuse=True, initializer=initializer):
mtest = PTBModel(is_training=False, config=eval_config,
input_=test_input)
sv = tf.train.Supervisor(logdir=FLAGS.save_path)
with sv.managed_session() as session:
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay ** max(i + 1 - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session, m, eval_op=m.train_op,
verbose=True)
print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
valid_perplexity = run_epoch(session, mvalid)
print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
test_perplexity = run_epoch(session, mtest)
print("Test Perplexity: %.3f" % test_perplexity)
if FLAGS.save_path:
print("Saving model to %s." % FLAGS.save_path)
sv.saver.save(session, FLAGS.save_path, global_step=sv.global_step)
def load_ptb_dataset(data_path):
"""Load the PTB dataset.
You can download the PTB dataset from here:
http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz
:param data_path: path to the data/ dir of the PTB dataset.
:return: train, validation, test data
"""
raw_data = reader.ptb_raw_data(data_path)
trX, vlX, teX, _ = raw_data
return trX, vlX, teX
def main():
data_directory = "data"
word_to_id = reader._build_vocab(os.path.join(data_directory,
"ptb.train.txt"))
train, cv, test, _ = reader.ptb_raw_data(data_directory)
train_batch_size = 128
train_num_steps = len(train) // train_batch_size - 1
train_num_steps = 10
ptb_iterator = reader.ptb_iterator(train, train_batch_size, train_num_steps)
learner = Learner(word_to_id)
learner.Train(ptb_iterator, train_batch_size, train_num_steps)
def main(_):
t0 = time.time()
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
# changed from tensorflow - add peak_wps calculation
peak_wps = 0
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config)
with tf.variable_scope("model", reuse=True, initializer=initializer):
mvalid = PTBModel(is_training=False, config=config)
mtest = PTBModel(is_training=False, config=eval_config)
tf.initialize_all_variables().run()
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity, cur_peak_wps = run_epoch(session, m, peak_wps, train_data, m.train_op,
verbose=True)
if cur_peak_wps > peak_wps:
peak_wps = cur_peak_wps
print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
valid_perplexity, cur_peak_wps = run_epoch(session, mvalid, peak_wps, valid_data, tf.no_op())
print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
test_perplexity, cur_peak_wps = run_epoch(session, mtest, peak_wps, test_data, tf.no_op())
print("Test Perplexity: %.3f" % test_perplexity)
# change form tensorflow - print out timing info
t1 = time.time()
print('total time: ', t1-t0)
print('peak wps:', peak_wps)
tf.app.flags.DEFINE_integer("seed", 12345, "Random seed.")
tf.app.flags.DEFINE_integer("runs", 3, "How many runs.")
tf.app.flags.DEFINE_float("keep_prob", 1.0, "Keep probability for dropout.")
tf.app.flags.DEFINE_string("result_file", None, "Where to write results.")
tf.app.flags.DEFINE_string("moru_ops", 'keep,replace', "operations of moru cell.")
tf.app.flags.DEFINE_string("moru_op_biases", None, "biases of moru operations at beginning of training. "
"Defaults to 0 for each.")
tf.app.flags.DEFINE_integer("moru_op_ctr", None, "Size of op ctr. By default ops are controlled by current input"
"and previous state. Given a positive integer, an additional"
"recurrent op ctr is introduced in MORUCell.")
tf.app.flags.DEFINE_string('device', '/gpu:0', 'device to run on')
FLAGS = tf.app.flags.FLAGS
FLAGS._parse_flags()
raw_data = reader.ptb_raw_data(FLAGS.data)
train_data, valid_data, test_data, _ = raw_data
perplexities = []
batch_size = FLAGS.batch_size
num_steps = FLAGS.num_steps
rng = random.Random(FLAGS.seed)
for run_id in range(FLAGS.runs):
tf.reset_default_graph()
last_valid_perplexities = [float("inf")] * 3
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
tf.set_random_seed(rng.randint(0, 10000))
initializer = tf.random_uniform_initializer(-FLAGS.init_scale,
FLAGS.init_scale)
with tf.device(FLAGS.device):
def main(argv=None):
# 获取数据源
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
#定义初始化函数
initializer = tf.random_uniform_initializer(-0.05, 0.05)
# 计算一个epoch需要训练的次数
train_data_len = len(train_data)
train_batch_len = train_data_len // TRAINING_BATCH_SIZE
train_epoch_size = (train_batch_len - 1) // TRAINING_NUM_STEP
valid_data_len = len(valid_data)
valid_batch_len = valid_data_len // EVAL_BATCH_SIZE
valid_epoch_size = (valid_batch_len - 1) // EVAL_NUM_STEP
test_data_len = len(test_data)
test_batch_len = test_data_len // EVAL_BATCH_SIZE
test_epoch_size = (test_batch_len - 1) // EVAL_NUM_STEP
#定义训练使用的循环神经网络
with tf.variable_scope("language_model",
reuse=None,
initializer=initializer):
train_model = PTBModel(True, TRAINING_BATCH_SIZE, TRAINING_NUM_STEP)
#定义评测用的模型
with tf.variable_scope("language_model",
reuse=True,
initializer=initializer):
eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
with tf.Session() as session:
tf.global_variables_initializer().run()
train_d = reader.ptb_producer(train_data, TRAINING_BATCH_SIZE,
TRAINING_NUM_STEP)
valid_d = reader.ptb_producer(valid_data, EVAL_BATCH_SIZE,
EVAL_NUM_STEP)
test_d = reader.ptb_producer(test_data, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
# 开启多线程
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=session, coord=coord)
#使用训练数据训练模型
for i in range(NUM_EPOCH):
print("in iteration: %d" % (i + 1))
#在所有训练数据上训练循环神经网络
run_epoch(session, train_model, train_d, train_model.train_op,
train_epoch_size, True)
#使用验证数据评测模型效果
valid_perplexity = run_epoch(session, eval_model, valid_d,
tf.no_op(), valid_epoch_size, False)
print("Epoch: %d Validation Perplexity: %.3f" %
(i + 1, valid_perplexity))
#使用测试数据测试模型
test_perplexity = run_epoch(session, eval_model, test_d, tf.no_op(),
test_epoch_size, False)
print("Test Perplexity: %.3f" % test_perplexity)
# 关闭多线程
coord.request_stop()
coord.join(threads)
return MediumConfig()
elif FLAGS.model == "large":
return LargeConfig()
elif FLAGS.model == "test":
return TestConfig()
else:
raise ValueError("Invalid model: %s", FLAGS.model)
# def main(_):
if __name__ == '__main__':
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
print(FLAGS.data_path)
raw_data = reader.ptb_raw_data(FLAGS.data_path) # 获取原始数据
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(
-config.init_scale, # 定义如何对参数变量初始化
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True,
config=config) # 训练模型， is_trainable=True
with tf.variable_scope("model", reuse=True, initializer=initializer):
def main(_):
if FLAGS.rename_variable_prefix:
if not FLAGS.model_path or not FLAGS.new_model_path:
logging.error("Must set --model_path and --new_model_path to rename model variables")
exit(1)
else:
if not FLAGS.train_dir:
logging.error("Must set --train_dir")
exit(1)
if not FLAGS.data_dir and (not FLAGS.train_idx or not FLAGS.dev_idx):
logging.error("Must set --data_dir to PTB data directory or specify data using --train_idx,--dev_idx")
exit(1)
logging.getLogger().setLevel(logging.INFO)
logging.info("Start: {}".format(datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')))
device = "/gpu:0"
log_device_placement = False
allow_soft_placement = True
if FLAGS.device:
device = '/'+FLAGS.device
logging.info("Use device %s" % device)
with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto(allow_soft_placement=allow_soft_placement, log_device_placement=log_device_placement)) \
as session, tf.device(device):
if FLAGS.rename_variable_prefix:
model_utils.rename_variable_prefix(session, FLAGS.config_file, FLAGS.model_path, FLAGS.new_model_path,
FLAGS.variable_prefix, FLAGS.rename_variable_prefix)
elif FLAGS.score:
logging.info("Run model in scoring mode")
use_log_probs = True
train_dir = "train.rnn.de"
model, _ = model_utils.load_model(session, "large50k", train_dir, use_log_probs)
#test_path = os.path.join(FLAGS.data_dir, "test15/test15.ids50003.de")
#test_data = reader.read_indexed_data(test_path)
#test_sentences = [ test_data ]
# Add eos symbol to the beginning to score first word as well
test_sentences = [[2, 5, 3316, 7930, 7, 7312, 9864, 30, 8, 10453, 4, 2],
[2, 7, 5, 30, 8, 10453, 7930, 3316, 7312, 9864, 4, 2],
[2, 5, 8, 30, 7, 4, 9864, 3316, 7312, 7930, 10453, 2],
[2, 8, 10453, 9864, 30, 5, 3316, 7312, 7, 7930, 4]]
for test_data in test_sentences:
# using log probs or cross entropies gives the same perplexities
if use_log_probs:
# Run model as in training, with an iterator over inputs
train_utils.run_epoch_eval(session, model, test_data, tf.no_op(), use_log_probs=use_log_probs)
# Run model step by step (yields the same result)
#score_sentence(session, model, test_data)
else:
train_utils.run_epoch_eval(session, model, test_data, tf.no_op(), use_log_probs=use_log_probs)
else:
logging.info("Run model in training mode")
if FLAGS.fixed_random_seed:
tf.set_random_seed(1234)
if FLAGS.model:
config = model_utils.get_config(FLAGS.model)
eval_config = model_utils.get_config(FLAGS.model)
elif FLAGS.config_file:
config = model_utils.read_config(FLAGS.config_file)
eval_config = copy.copy(config)
else:
logging.error("Must specify either model name or config file.")
exit(1)
eval_config.batch_size = 1
eval_config.num_steps = 1
model, mvalid, mtest = model_utils.create_model(session, config, eval_config, FLAGS.train_dir, FLAGS.optimizer)
# Restore saved train variable
start_epoch = 1
start_idx = 0
start_state = None
tmpfile = FLAGS.train_dir+"/tmp_idx.pkl"
if model.global_step.eval() >= FLAGS.steps_per_checkpoint and \
os.path.isfile(tmpfile):
with open(tmpfile, "rb") as f:
start_epoch, start_idx, start_state = pickle.load(f)
logging.info("Restore saved train variables from %s, resume from epoch=%i and train idx=%i and last state" % (tmpfile, start_epoch, start_idx))
if FLAGS.data_dir:
raw_data = reader.ptb_raw_data(FLAGS.data_dir)
train_data, valid_data, test_data, _ = raw_data
else:
train_data = reader.read_indexed_data(FLAGS.train_idx, FLAGS.max_train_data_size, config.vocab_size)
valid_data = reader.read_indexed_data(FLAGS.dev_idx, vocab_size=config.vocab_size)
if FLAGS.test_idx:
test_data = reader.read_indexed_data(FLAGS.test_idx, vocab_size=config.vocab_size)
for epoch in range(start_epoch, config.max_max_epoch+1):
if not (FLAGS.optimizer == "adadelta" or FLAGS.optimizer == "adam"):
if start_idx == 0:
lr_decay = config.lr_decay ** max(epoch - config.max_epoch+1, 0.0)
model.assign_lr(session, config.learning_rate * lr_decay)
logging.info("Epoch: %d Learning rate: %.3f" % (epoch, session.run(model.lr)))
train_perplexity = train_utils.run_epoch(session, model, train_data, model.train_op, FLAGS.train_dir, FLAGS.steps_per_checkpoint,
train=True, start_idx=start_idx, start_state=start_state, tmpfile=tmpfile, m_valid=mvalid,
valid_data=valid_data, epoch=epoch)
if start_idx == 0:
logging.info("Epoch: %d Train Perplexity: %.3f" % (epoch, train_perplexity))
else:
logging.info("Epoch: %d Train Perplexity: %.3f (incomplete)" % (epoch, train_perplexity))
start_idx = 0
start_state = None
valid_perplexity = train_utils.run_epoch(session, mvalid, valid_data, tf.no_op(), FLAGS.train_dir, FLAGS.steps_per_checkpoint)
logging.info("Epoch: %d Full Valid Perplexity: %.3f" % (epoch, valid_perplexity))
logging.info("Training finished.")
if FLAGS.data_dir or FLAGS.test_idx:
test_perplexity = train_utils.run_epoch(session, mtest, test_data, tf.no_op(), FLAGS.train_dir, FLAGS.steps_per_checkpoint)
logging.info("Test Perplexity: %.3f" % test_perplexity)
checkpoint_path = os.path.join(FLAGS.train_dir, "rnn.ckpt")
logging.info("Save final model to path=%s" % checkpoint_path)
model.saver.save(session, checkpoint_path, global_step=model.global_step)
logging.info("End: {}".format(datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')))
def main(_):
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default():
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.name_scope("Train"):
train_input = PTBInput(config=config,
data=train_data,
name="TrainInput")
with tf.variable_scope("Model",
reuse=None,
initializer=initializer):
m = PTBModel(is_training=True,
config=config,
input_=train_input)
tf.scalar_summary("Training Loss", m.cost)
tf.scalar_summary("Learning Rate", m.lr)
with tf.name_scope("Valid"):
valid_input = PTBInput(config=config,
data=valid_data,
name="ValidInput")
with tf.variable_scope("Model",
reuse=True,
initializer=initializer):
mvalid = PTBModel(is_training=False,
config=config,
input_=valid_input)
tf.scalar_summary("Validation Loss", mvalid.cost)
with tf.name_scope("Test"):
test_input = PTBInput(config=eval_config,
data=test_data,
name="TestInput")
with tf.variable_scope("Model",
reuse=True,
initializer=initializer):
mtest = PTBModel(is_training=False,
config=eval_config,
input_=test_input)
sv = tf.train.Supervisor(logdir=FLAGS.save_path)
with sv.managed_session() as session:
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay**max(i + 1 - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" %
(i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session,
m,
eval_op=m.train_op,
verbose=True)
print("Epoch: %d Train Perplexity: %.3f" %
(i + 1, train_perplexity))
valid_perplexity = run_epoch(session, mvalid)
print("Epoch: %d Valid Perplexity: %.3f" %
(i + 1, valid_perplexity))
test_perplexity = run_epoch(session, mtest)
print("Test Perplexity: %.3f" % test_perplexity)
if FLAGS.save_path:
print("Saving model to %s." % FLAGS.save_path)
sv.saver.save(session,
FLAGS.save_path,
global_step=sv.global_step)
def main(_):
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
# 22222222222222222222222222222222222222222222222222222222222222222222222222222222222222222
ps_hosts = FLAGS.ps_hosts.split(",")
worker_hosts = FLAGS.worker_hosts.split(",")
# 33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333
# Create a cluster from the parameter server and worker hosts.
cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})
# Create and start a server for the local task.
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_index)
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
# 444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444
if FLAGS.job_name == "ps":
server.join()
elif FLAGS.job_name == "worker":
# Assigns ops to the local worker by default.
# 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1 5.1
with tf.Graph().as_default(): #, tf.Session() as session:
initializer = tf.random_uniform_initializer(
-config.init_scale, config.init_scale)
# 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2 5.2
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % FLAGS.task_index,
cluster=cluster)):
with tf.variable_scope("model",
reuse=None,
initializer=initializer):
m = PTBModel(is_training=True, config=config)
with tf.variable_scope("model",
reuse=True,
initializer=initializer):
mvalid = PTBModel(is_training=False, config=config)
mtest = PTBModel(is_training=False, config=eval_config)
#tf.initialize_all_variables().run()
init_op = tf.initialize_all_variables()
# GlobalStep-GlobalStep-GlobalStep-GlobalStep-GlobalStep-GlobalStep-GlobalStep-GlobalStep-
global_step = tf.Variable(0)
saver = tf.train.Saver()
summary_op = tf.merge_all_summaries()
init_op = tf.initialize_all_variables()
# 6666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666
# Ehtemalan managed session bayad biad inja
sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0),
logdir="/tmp/train_logs",
init_op=init_op,
summary_op=summary_op,
saver=saver,
global_step=global_step,
save_model_secs=600)
# 777777777777777777777777777777777777777777777777777777777777777777777777777777777777777777
with sv.managed_session(server.target) as session:
# 888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay**max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" %
(i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session,
m,
train_data,
m.train_op,
verbose=True)
print("Epoch: %d Train Perplexity: %.3f" %
(i + 1, train_perplexity))
valid_perplexity = run_epoch(session, mvalid, valid_data,
tf.no_op())
print("Epoch: %d Valid Perplexity: %.3f" %
(i + 1, valid_perplexity))
test_perplexity = run_epoch(session, mtest, test_data,
tf.no_op())
print("Test Perplexity: %.3f" % test_perplexity)
# Ask for all the services to stop.
sv.stop()
# -*- coding:utf8 -*- import tensorflow as tf from tensorflow.models.rnn.ptb import reader DATA_PATH = "/usr/bigdata/data/PTB/simple-examples/data/" train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH) print(len(train_data)) print(train_data[:100])
def main(_):
if not FLAGS.data_path:
raise ValueError(
"Paramétrer --datapath à l'emplacement du dossier de test")
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
start_time = time.time()
print("Type de neurones: {}".format(FLAGS.cell))
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config, cell_type=FLAGS.cell)
with tf.variable_scope("model", reuse=True, initializer=initializer):
mvalid = PTBModel(is_training=False,
config=config,
cell_type=FLAGS.cell)
mtest = PTBModel(is_training=False,
config=eval_config,
cell_type=FLAGS.cell)
tf.initialize_all_variables().run()
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay**max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Période: %d Vitesse d'apprentissage: %.3f" %
(i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session,
m,
train_data,
m.train_op,
verbose=True)
print("Période: %d Perplexité sur données d'apprentissage: %.3f" %
(i + 1, train_perplexity))
# Test si la perplexité de validation a augmenté ou stagné, si oui on divise le lr par 1.5
# Non utilisé mais correspond mieux à l'article
'''
valid_old = np.inf
if i > 0:
valid_old = valid_perplexity
'''
valid_perplexity = run_epoch(session, mvalid, valid_data,
tf.no_op())
print("Période: %d Perplexité sur données de validation: %.3f" %
(i + 1, valid_perplexity))
'''
if i > 0 and valid_old <= valid_perplexity:
print('Diminution du LR: %.4f' % m.lr)
m.lr_decay /=1.5
'''
test_perplexity = run_epoch(session, mtest, test_data, tf.no_op())
print("Perplexité sur données de test: %.3f" % test_perplexity)
print("Temps d'exécution: %.3f" % (time.time() - start_time))
def main(_):
if FLAGS.rename_variable_prefix:
if not FLAGS.model_path or not FLAGS.new_model_path:
logging.error(
"Must set --model_path and --new_model_path to rename model variables"
)
exit(1)
else:
if not FLAGS.train_dir:
logging.error("Must set --train_dir")
exit(1)
if not FLAGS.data_dir and (not FLAGS.train_idx or not FLAGS.dev_idx):
logging.error(
"Must set --data_dir to PTB data directory or specify data using --train_idx,--dev_idx"
)
exit(1)
logging.getLogger().setLevel(logging.INFO)
logging.info("Start: {}".format(
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S')))
device = "/gpu:0"
log_device_placement = False
allow_soft_placement = True
if FLAGS.device:
device = '/' + FLAGS.device
logging.info("Use device %s" % device)
with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto(allow_soft_placement=allow_soft_placement, log_device_placement=log_device_placement)) \
as session, tf.device(device):
if FLAGS.rename_variable_prefix:
model_utils.rename_variable_prefix(session, FLAGS.config_file,
FLAGS.model_path,
FLAGS.new_model_path,
FLAGS.variable_prefix,
FLAGS.rename_variable_prefix)
elif FLAGS.score:
logging.info("Run model in scoring mode")
use_log_probs = True
logging.info("Run model in scoring mode")
train_dir = "train.rnn.de"
model, _ = load_model(session, FLAGS.config_file, train_dir,
use_log_probs)
#test_path = os.path.join(FLAGS.data_dir, "test15/test15.ids50003.de")
#test_data = reader.read_indexed_data(test_path)
#test_sentences = [ test_data ]
# Add eos symbol to the beginning to score first word as well
test_sentences = [
[2, 5, 3316, 7930, 7, 7312, 9864, 30, 8, 10453, 4, 2],
[2, 7, 5, 30, 8, 10453, 7930, 3316, 7312, 9864, 4, 2],
[2, 5, 8, 30, 7, 4, 9864, 3316, 7312, 7930, 10453, 2],
[2, 8, 10453, 9864, 30, 5, 3316, 7312, 7, 7930, 4]
]
for test_data in test_sentences:
# using log probs or cross entropies gives the same perplexities
if use_log_probs:
# Run model as in training, with an iterator over inputs
train_utils.run_epoch_eval(session,
model,
test_data,
tf.no_op(),
use_log_probs=use_log_probs)
# Run model step by step (yields the same result)
#score_sentence(session, model, test_data)
else:
train_utils.run_epoch_eval(session,
model,
test_data,
tf.no_op(),
use_log_probs=use_log_probs)
else:
logging.info("Run model in training mode")
if FLAGS.model:
config = model_utils.get_config(FLAGS.model)
eval_config = model_utils.get_config(FLAGS.model)
elif FLAGS.config_file:
config = model_utils.read_config(FLAGS.config_file)
eval_config = copy.copy(config)
else:
logging.error("Must specify either model name or config file.")
sys.exit(1)
eval_config.batch_size = 1
eval_config.num_steps = 1
#raw_data = reader.ptb_raw_data(FLAGS.data_dir)
#train_data, valid_data, test_data, _ = raw_data
train_data = reader.read_indexed_data(FLAGS.train_idx,
FLAGS.max_train_data_size,
config.vocab_size)
valid_data = reader.read_indexed_data(FLAGS.dev_idx,
vocab_size=config.vocab_size)
test_data = reader.read_indexed_data(FLAGS.test_idx,
vocab_size=config.vocab_size)
if FLAGS.use_adagrad:
config.learning_rate = 0.5
if FLAGS.fixed_random_seed:
tf.set_random_seed(1234)
initializer = tf.random_uniform_initializer(
-config.init_scale, config.init_scale)
with tf.variable_scope("model",
reuse=None,
initializer=initializer):
model = RNNLMModel(is_training=True, config=config)
with tf.variable_scope("model",
reuse=True,
initializer=initializer):
mvalid = RNNLMModel(is_training=False, config=config)
mtest = RNNLMModel(is_training=False, config=eval_config)
ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path):
logging.info("Reading model parameters from %s" %
ckpt.model_checkpoint_path)
model.saver.restore(session, ckpt.model_checkpoint_path)
else:
logging.info("Created model with fresh parameters.")
session.run(tf.initialize_all_variables())
# Restore saved train variable
start_idx = 0
tmpfile = FLAGS.train_dir + "/tmp_idx.pkl"
if model.global_step.eval() >= FLAGS.steps_per_checkpoint and \
os.path.isfile(tmpfile):
with open(tmpfile, "rb") as f:
start_idx = pickle.load(f)
logging.info(
"Restore saved train variable from %s, resume from train idx=%i"
% (tmpfile, start_idx))
checkpoint_path = os.path.join(FLAGS.train_dir, "rnn.ppx.ckpt")
best_valid = np.inf
for i in range(config.max_max_epoch):
if not (FLAGS.use_adadelta or FLAGS.use_adam):
lr_decay = config.lr_decay**max(i - config.max_epoch, 0.0)
model.assign_lr(session, config.learning_rate * lr_decay)
if FLAGS.data_dir:
raw_data = reader.ptb_raw_data(FLAGS.data_dir)
train_data, valid_data, test_data, _ = raw_data
else:
train_data = reader.read_indexed_data(
FLAGS.train_idx, FLAGS.max_train_data_size,
config.vocab_size)
valid_data = reader.read_indexed_data(
FLAGS.dev_idx, vocab_size=config.vocab_size)
if FLAGS.test_idx:
test_data = reader.read_indexed_data(
FLAGS.test_idx, vocab_size=config.vocab_size)
for epoch in range(start_epoch, config.max_max_epoch + 1):
if not (FLAGS.optimizer == "adadelta"
or FLAGS.optimizer == "adam"):
if start_idx == 0:
lr_decay = config.lr_decay**max(
epoch - config.max_epoch + 1, 0.0)
model.assign_lr(session,
config.learning_rate * lr_decay)
logging.info("Epoch: %d Learning rate: %.3f" %
(epoch, session.run(model.lr)))
train_perplexity = train_utils.run_epoch(
session,
model,
train_data,
model.train_op,
FLAGS.train_dir,
FLAGS.steps_per_checkpoint,
train=True,
start_idx=start_idx,
tmpfile=tmpfile,
m_valid=mvalid,
valid_data=valid_data)
start_idx = 0
logging.info("Epoch: %d Train Perplexity: %.3f" %
(i + 1, train_perplexity))
valid_perplexity = train_utils.run_epoch(
session, mvalid, valid_data, tf.no_op(), FLAGS.train_dir,
FLAGS.steps_per_checkpoint)
if valid_perplexity < best_valid:
best_valid = valid_perplexity
model.saver.save(session,
checkpoint_path,
global_step=model.global_step)
logging.info(
"Model achieves new best dev perplexity %.3f - saving to path=%s"
.format(valid_perplexity, checkpoint_path))
logging.info("Epoch: %d Full Valid Perplexity: %.3f" %
(i + 1, valid_perplexity))
logging.info("Training finished.")
test_perplexity = train_utils.run_epoch(session, mtest, test_data,
tf.no_op(),
FLAGS.train_dir,
FLAGS.steps_per_checkpoint)
final_checkpoint_path = os.path.join(FLAGS.train_dir,
"rnn.final.ckpt")
logging.info("Test Perplexity: %.3f" % test_perplexity)
logging.info("Save final model to path=%s" % final_checkpoint_path)
model.saver.save(session,
final_checkpoint_path,
global_step=model.global_step)
logging.info("End: {}".format(
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S')))
handler.setFormatter(formatter)
logger.addHandler(handler)
if DEBUG:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
from tensorflow.models.rnn.ptb import reader
import numpy as np
DATAURL = "https://github.com/mil-tokyo/neural_network/tree/master/saito/sample/simple-examples/data"
data_path = "simple-examples/data/"
#"../mldatasets/simple-examples/data/"
try:
raw_data = reader.ptb_raw_data( data_path)
except:
# wget http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz
# tar xvf simple-examples.tgz
logging.warn("download data from\t%s\tto\t%s" % (DATAURL , data_path) )
train_data, valid_data, test_data, _ = raw_data
"define shapes and generate data"
vocab_size = max(train_data)
emb_size = 120
num_steps = 17
batch_size = 20
logging.debug("batch_size\t%s" % batch_size )
def main(_):
# 获取原始数据
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
# 计算一个epoch需要训练的次数
train_data_len = len(train_data)
train_batch_len = train_data_len // TRAIN_BATCH_SIZE
train_epoch_size = (train_batch_len - 1) // TRAIN_NUM_STEP
valid_data_len = len(valid_data)
valid_batch_len = valid_data_len // EVAL_BATCH_SIZE
valid_epoch_size = (valid_batch_len - 1) // EVAL_NUM_STEP
test_data_len = len(test_data)
test_batch_len = test_data_len // EVAL_BATCH_SIZE
test_epoch_size = (test_batch_len - 1) // EVAL_NUM_STEP
# 定义初始化函数
initializer = tf.random_uniform_initializer(-0.05, 0.05)
# 定义训练用的循环神经网络模型
with tf.variable_scope("language_model",
reuse=None,
initializer=initializer):
train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)
# 定义评测用的循环神经网络模型
with tf.variable_scope("language_model",
reuse=True,
initializer=initializer):
eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
with tf.Session() as session:
tf.global_variables_initializer().run()
train_queue = reader.ptb_producer(train_data, train_model.batch_size,
train_model.num_steps)
eval_queue = reader.ptb_producer(valid_data, eval_model.batch_size,
eval_model.num_steps)
test_queue = reader.ptb_producer(test_data, eval_model.batch_size,
eval_model.num_steps)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=session, coord=coord)
# 使用训练数据训练模型
for i in range(NUM_EPOCH):
print "In iteration: {}".format(i + 1)
run_epoch(session, train_model, train_queue, train_model.train_op,
True, train_epoch_size)
valid_perplexity = run_epoch(session, eval_model, eval_queue,
tf.no_op(), False, valid_epoch_size)
print "Epoch: {} Validation Perplexity: {:.3f}".format(
i + 1, valid_perplexity)
# 最后使用测试数据测试模型效果
test_perplexity = run_epoch(session, eval_model, test_queue,
tf.no_op(), False, test_epoch_size)
print "Test Perplexity: {:.3f}".format(test_perplexity)
coord.request_stop()
coord.join(threads)
def main(_):
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default(), tf.Session() as session:
if FLAGS.importmodeldir is not None:
# Import a model instance
# Find last executed epoch
from glob import glob
history = list(map(lambda x: int(x.split('-')[1][:-5]), glob(FLAGS.importmodeldir+'/model/model.ckpt-*.meta')))
last_epoch = np.max(history)
# Recreate model
with tf.variable_scope("model", reuse=None):
m = PTBModel(is_training=True, config=config)
# merged_summaries_for_training = tf.merge_all_summaries() # use this operation to merge summaries attached so far
with tf.variable_scope("model", reuse=True):
mtest = PTBModel(is_training=False, config=eval_config)
merged_summaries_for_test = tf.merge_all_summaries() # use this operation to merge summaries attached so far
mvalid = PTBModel(is_training=False, config=config)
# merged_summaries_for_valid = tf.merge_all_summaries() # use this operation to merge summaries attached so far
# Fill model variables with trained values
tf.train.Saver().restore(session, FLAGS.importmodeldir+'/model/model.ckpt-{}'.format(last_epoch))
initial_epoch = last_epoch + 1
else:
# Create a model instance
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config)
# merged_summaries_for_training = tf.merge_all_summaries() # use this operation to merge summaries attached so far
with tf.variable_scope("model", reuse=True, initializer=initializer):
mtest = PTBModel(is_training=False, config=eval_config)
merged_summaries_for_test = tf.merge_all_summaries() # use this operation to merge summaries attached so far
mvalid = PTBModel(is_training=False, config=config)
# merged_summaries_for_valid = tf.merge_all_summaries() # use this operation to merge summaries attached so far
tf.initialize_all_variables().run()
initial_epoch = 0
init_logs()
init_model_persistance()
train_writer = tf.train.SummaryWriter(FLAGS.logdir + "/train", session.graph)
valid_writer = tf.train.SummaryWriter(FLAGS.logdir + "/valid", session.graph)
test_writer = tf.train.SummaryWriter(FLAGS.logdir + "/test", session.graph)
for i in range(initial_epoch, config.max_max_epoch):
lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session, m, train_data, m.train_op,
verbose=True, summary_op=None, summary_writer=train_writer)
print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
valid_perplexity = run_epoch(session, mvalid, valid_data, tf.no_op(), summary_op=None,summary_writer=None)
print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
if FLAGS.exportmodeldir is not None:
tf.train.Saver().save(session,FLAGS.exportmodeldir+"/model/model.ckpt",global_step=i)
test_perplexity = run_epoch(session, mtest, test_data, tf.no_op(), summary_op=merged_summaries_for_test, summary_writer=test_writer)
if FLAGS.exportmodeldir is not None:
tf.train.Saver().save(session,FLAGS.exportmodeldir+"/model/model.ckpt",global_step=config.max_max_epoch)
print("Test Perplexity: %.3f" % test_perplexity)
