def main(unused_argv):
# Set up the model config.
model_config = configuration.model_config(
input_file_pattern=FLAGS.input_file_pattern,
input_queue_capacity=FLAGS.num_eval_examples,
shuffle_input_data=False)
if FLAGS.model_config_overrides:
model_config.parse_json(FLAGS.model_config_overrides)
config_json = json.dumps(model_config.values(), indent=2)
tf.logging.info("model_config: %s", config_json)
with tf.Graph().as_default():
# Build the model for evaluation.
model = skip_thoughts_model.SkipThoughtsModel(model_config,
mode="eval")
model.build()
evaluation.evaluate_repeatedly(
model=model,
checkpoint_dir=FLAGS.checkpoint_dir,
eval_dir=FLAGS.eval_dir,
num_eval_examples=FLAGS.num_eval_examples,
min_global_step_for_perplexity=FLAGS.min_global_step,
master=FLAGS.master,
eval_interval_secs=FLAGS.eval_interval_secs)
def build_graph_from_config(self, model_config, checkpoint_path):
"""Builds the inference graph from a configuration object.
Args:
model_config: Object containing configuration for building the model.
checkpoint_path: Checkpoint file or a directory containing a checkpoint
file.
Returns:
restore_fn: A function such that restore_fn(sess) loads model variables
from the checkpoint file.
"""
tf.logging.info("Building model.")
model = skip_thoughts_model.SkipThoughtsModel(model_config,
mode="encode")
model.build()
variables = tf.global_variables()
variables_to_restore = []
restore_names = get_trainable_vars_fromchpt(checkpoint_path)
for v in variables:
if v.name in restore_names:
variables_to_restore += [v]
print(v.name, v.name in restore_names)
saver = tf.train.Saver(variables_to_restore)
return self._create_restore_fn(checkpoint_path, saver)
def main(unused_argv):
if not FLAGS.input_file_pattern:
raise ValueError("--input_file_pattern is required.")
if not FLAGS.checkpoint_dir:
raise ValueError("--checkpoint_dir is required.")
if not FLAGS.eval_dir:
raise ValueError("--eval_dir is required.")
# Create the evaluation directory if it doesn't exist.
eval_dir = FLAGS.eval_dir
if not tf.gfile.IsDirectory(eval_dir):
tf.logging.info("Creating eval directory: %s", eval_dir)
tf.gfile.MakeDirs(eval_dir)
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.model_config(
input_file_pattern=FLAGS.input_file_pattern,
input_queue_capacity=FLAGS.num_eval_examples,
shuffle_input_data=False)
model = skip_thoughts_model.SkipThoughtsModel(model_config,
mode="eval")
model.build()
losses = tf.concat(model.target_cross_entropy_losses, 0)
weights = tf.concat(model.target_cross_entropy_loss_weights, 0)
# Create the Saver to restore model Variables.
saver = tf.train.Saver()
# Create the summary operation and the summary writer.
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(eval_dir)
g.finalize()
# Run a new evaluation run every eval_interval_secs.
while True:
start = time.time()
tf.logging.info(
"Starting evaluation at " +
time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, losses, weights, saver, summary_writer, summary_op)
time_to_next_eval = start + FLAGS.eval_interval_secs - time.time()
if time_to_next_eval > 0:
time.sleep(time_to_next_eval)
def build_graph_from_config(self, model_config, checkpoint_path):
"""Builds the inference graph from a configuration object.
Args:
model_config: Object containing configuration for building the model.
checkpoint_path: Checkpoint file or a directory containing a checkpoint
file.
Returns:
restore_fn: A function such that restore_fn(sess) loads model variables
from the checkpoint file.
"""
tf.logging.info("Building model.")
model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="encode")
model.build()
saver = tf.train.Saver()
return self._create_restore_fn(checkpoint_path, saver)
def main(unused_argv):
if not FLAGS.input_file_pattern:
raise ValueError("--input_file_pattern is required.")
if not FLAGS.train_dir:
raise ValueError("--train_dir is required.")
model_config = configuration.model_config(
input_file_pattern=FLAGS.input_file_pattern)
training_config = configuration.training_config()
tf.logging.info("Building training graph.")
g = tf.Graph()
with g.as_default():
model = skip_thoughts_model.SkipThoughtsModel(model_config,
mode="train")
model.build()
learning_rate = _setup_learning_rate(training_config,
model.global_step)
optimizer = tf.train.AdamOptimizer(learning_rate)
train_tensor = tf.contrib.slim.learning.create_train_op(
total_loss=model.total_loss,
optimizer=optimizer,
global_step=model.global_step,
clip_gradient_norm=training_config.clip_gradient_norm)
saver = tf.train.Saver()
tf.contrib.slim.learning.train(
train_op=train_tensor,
logdir=FLAGS.train_dir,
graph=g,
global_step=model.global_step,
number_of_steps=training_config.number_of_steps,
save_summaries_secs=training_config.save_summaries_secs,
saver=saver,
save_interval_secs=training_config.save_model_secs)
def main(unused_argv):
if not FLAGS.train_dir:
raise ValueError("--train_dir is required.")
#read_vocab(FLAGS.vocab)
model_config = configuration.model_config()
training_config = configuration.training_config()
################ define discriminator model ################
disc_model = Discriminator(sequence_length=MAXLEN,
num_classes=1,
vocab_size=model_config.vocab_size,
embedding_size=model_config.word_embedding_dim,
filter_sizes=[1, 2, 3, 4, 5, 7, 10],
num_filters=[100, 100, 100, 100, 100, 100, 100])
################# define training model #################
model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="train")
model.build()
learning_rate = _setup_learning_rate(training_config, model.global_step)
optimizer = tf.train.AdamOptimizer(learning_rate)
variables_to_train = [v for v in tf.trainable_variables()]
variables_to_restore = [
v for v in tf.all_variables() if ('discriminator' not in v.name)
]
print(len(variables_to_train))
train_tensor = tf.contrib.slim.learning.create_train_op(
total_loss=model.total_loss,
optimizer=optimizer,
clip_gradient_norm=training_config.clip_gradient_norm,
variables_to_train=variables_to_train)
######################define target lstm ####################
#target_lstm = skip_thoughts_model.TargetLSTM(config=model_config)
#synthesized = True
target_lstm = None
synthesized = False
################ define testing model ################
#model_config_test = configuration.model_config()
#model_test = skip_thoughts_model.SkipThoughtsModel(model_config_test, mode="eval")
#model_test.build(is_testing=True)
################ define savers ################
reloader = tf.train.Saver(var_list=variables_to_restore)
reloader_all = tf.train.Saver()
saver = tf.train.Saver(max_to_keep=1000)
gpu_config = tf.ConfigProto(gpu_options=tf.GPUOptions(
per_process_gpu_memory_fraction=1.0, allow_growth=True),
allow_soft_placement=True,
log_device_placement=False)
init_op = tf.global_variables_initializer()
sess = tf.Session(config=gpu_config)
run_metadata = tf.RunMetadata()
sess.run(init_op,
options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE,
output_partition_graphs=True),
run_metadata=run_metadata)
with open("/tmp/meta.txt", 'w') as f:
f.write(str(run_metadata))
if FLAGS.reload_model:
reloader.restore(sess, FLAGS.reload_model)
if FLAGS.reload_model_all:
reloader_all.restore(sess, FLAGS.reload_model_all)
################ load training data ##############
train_data_loader = DataLoader(128)
train_data_loader.load(FLAGS.train_corpus_en, FLAGS.train_corpus_fr)
total_loss_sup_list = []
total_loss_rl_list = []
bleu_list = []
fake_list, real_list, neglikely_list = [], [], []
outf = open(os.path.join(FLAGS.train_dir, 'log.txt'), 'a')
logf = open(os.path.join(FLAGS.train_dir, 'debug_log.txt'), 'w')
############### run training and testing #############
for i in xrange(1000000):
model_prefix = ""
if i < FLAGS.pretrain_G_steps:
model_prefix = "preG_"
np_global_step, total_loss_sup, total_loss_rl, avg_bleu, avg_fake, avg_real, avg_neglikely = my_train_step(
sess,
train_tensor,
model,
train_data_loader,
logf,
train_sup=True,
train_rl=False,
disc_model=disc_model,
adjustD=False,
adjustG=True,
given_num=MAXLEN)
elif i < FLAGS.pretrain_G_steps + FLAGS.pretrain_D_steps:
model_prefix = "preD_"
np_global_step, total_loss_sup, total_loss_rl, avg_bleu, avg_fake, avg_real, avg_neglikely = my_train_step(
sess,
train_tensor,
model,
train_data_loader,
logf,
train_sup=False,
train_rl=True,
disc_model=disc_model,
adjustD=True,
adjustG=False,
given_num=0)
elif FLAGS.mixer_period and FLAGS.mixer_step and FLAGS.mixer_period > 0:
gn = default_given_num - (
i - FLAGS.pretrain_G_steps - FLAGS.pretrain_D_steps
) // FLAGS.mixer_period * FLAGS.mixer_step
if gn < 0: gn = 0
model_prefix = "mixGN" + str(gn) + "_"
if i % 10 == 0:
adjustD = FLAGS.adjustD
else:
adjustD = False
if i % 200 == 0:
print("gn=", gn)
np_global_step, total_loss_sup, total_loss_rl, avg_bleu, avg_fake, avg_real, avg_neglikely = my_train_step( \
sess, train_tensor, model, train_data_loader, logf, train_sup=False, train_rl=True, \
disc_model=disc_model, adjustD=adjustD, adjustG=FLAGS.adjustG, given_num=gn)
else:
model_prefix = ""
np_global_step, total_loss_sup, total_loss_rl, avg_bleu, avg_fake, avg_real, avg_neglikely = my_train_step(
sess,
train_tensor,
model,
train_data_loader,
logf,
train_sup=False,
train_rl=True,
disc_model=disc_model,
adjustD=FLAGS.adjustD,
adjustG=FLAGS.adjustG)
total_loss_sup_list.append(total_loss_sup)
total_loss_rl_list.append(total_loss_rl)
fake_list.append(avg_fake)
real_list.append(avg_real)
bleu_list.append(avg_bleu)
neglikely_list.append(avg_neglikely)
if np_global_step % 2000 == 0:
saver.save(
sess,
os.path.join(FLAGS.train_dir,
model_prefix + "model-" + str(np_global_step)))
if np_global_step % 20 == 0:
# my_test_step(sess, model_test, FLAGS.test_result+'-'+str(np_global_step))
print(np_global_step, np.mean(total_loss_sup_list),
np.mean(total_loss_rl_list))
print(np.mean(bleu_list), np.mean(fake_list), np.mean(real_list))
print(np.mean(neglikely_list))
outf.write(
str(np_global_step) + " " + str(np.mean(total_loss_sup_list)) +
" " + str(np.mean(total_loss_rl_list)) + " " +
str(np.mean(bleu_list)) + " " + str(np.mean(fake_list)) + " " +
str(np.mean(real_list)) + " " + str(np.mean(neglikely_list)) +
"\n")
total_loss_sup_list, total_loss_rl_list, bleu_list, fake_list, real_list, neglikely_list = [],[],[],[],[],[]
def main(unused_argv):
# Create training directory if it doesn't already exist.
if not tf.gfile.IsDirectory(FLAGS.train_dir):
tf.logging.info("Creating training directory: %s", FLAGS.train_dir)
tf.gfile.MakeDirs(FLAGS.train_dir)
# Set up the model config.
model_config = configuration.model_config(
input_file_pattern=FLAGS.input_file_pattern)
if FLAGS.model_config_overrides:
model_config.parse_json(FLAGS.model_config_overrides)
_log_config(model_config, "model_config")
# Set up the training config.
training_config = configuration.training_config()
if FLAGS.training_config_overrides:
training_config.parse_json(FLAGS.training_config_overrides)
_log_config(training_config, "training_config")
tf.logging.info("Building training graph.")
g = tf.Graph()
with g.as_default(), g.device(
tf.train.replica_device_setter(FLAGS.ps_tasks)):
# Build the model.
model = skip_thoughts_model.SkipThoughtsModel(model_config,
mode="train")
model.build()
_log_variable_device_placement()
hooks = [
# Stop training if loss is NaN.
tf.train.NanTensorHook(model.total_loss),
# Log every training step.
tf.train.LoggingTensorHook(
{
"global_step": model.global_step,
"total_loss": model.total_loss
},
every_n_iter=1)
]
# Set up the learning rate and optimizer.
learning_rate = training.create_learning_rate(training_config,
model.global_step)
optimizer = training.create_optimizer(training_config, learning_rate)
# Set up distributed sync or async training.
is_chief = (FLAGS.task == 0)
if FLAGS.sync_replicas:
optimizer = tf.SyncReplicasOptimizer(
optimizer,
replicas_to_aggregate=FLAGS.replicas_to_aggregate,
total_num_replicas=FLAGS.total_num_replicas)
hooks.append(optimizer.make_session_run_hook(is_chief))
else:
# Startup delay for non-chief asynchronous workers.
if not is_chief and training_config.startup_delay_steps:
hooks.append(
tf.train.GlobalStepWaiterHook(
training_config.startup_delay_steps))
train_tensor = training.create_train_op(training_config, optimizer,
model)
keep_every_n = training_config.keep_checkpoint_every_n_hours
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep,
keep_checkpoint_every_n_hours=keep_every_n,
save_relative_paths=True)
scaffold = tf.train.Scaffold(saver=saver)
# Possibly set a step limit.
if training_config.number_of_steps:
hooks.append(
tf.train.StopAtStepHook(
last_step=training_config.number_of_steps))
# Create the TensorFlow session.
with tf.train.MonitoredTrainingSession(
master=FLAGS.master,
is_chief=is_chief,
checkpoint_dir=FLAGS.train_dir,
scaffold=scaffold,
hooks=hooks,
save_checkpoint_secs=training_config.save_model_secs,
save_summaries_steps=None,
save_summaries_secs=training_config.save_summaries_secs
) as sess:
# Run training.
while not sess.should_stop():
sess.run(train_tensor)
def main(unused_argv):
if not FLAGS.input_file_pattern:
raise ValueError("--input_file_pattern is required.")
if not FLAGS.train_dir:
raise ValueError("--train_dir is required.")
model_config = configuration.model_config(
input_file_pattern=FLAGS.input_file_pattern, bidirectional_encoder=True)
training_config = configuration.training_config()
tf.logging.info("Building training graph.")
g = tf.Graph()
with g.as_default():
model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="train")
model.build()
encoder_variables = [v for v in tf.global_variables()
if v.name.startswith("encoder") and "Adam" not in v.name]
embedding_variables = [v for v in tf.global_variables()
if v.name.startswith("word_embedding") and "Adam" not in v.name]
print([v.name for v in (encoder_variables+embedding_variables)])
learning_rate = _setup_learning_rate(training_config, model.global_step)
optimizer = tf.train.AdamOptimizer(learning_rate)
encoder_mult = 0.1
embedding_mult = 0.01
multiply = dict([(v, encoder_mult) for v in encoder_variables] + [(v, embedding_mult) for v in embedding_variables])
train_tensor = tf.contrib.slim.learning.create_train_op(
total_loss=model.total_loss,
optimizer=optimizer,
gradient_multipliers=multiply,
global_step=model.global_step,
clip_gradient_norm=training_config.clip_gradient_norm)
saver = tf.train.Saver()
model_path = tf.train.latest_checkpoint(FLAGS.train_dir)
pretrain_saver = tf.train.Saver(encoder_variables+embedding_variables)
print(model_path)
if model_path:
def restore_fn(sess):
tf.logging.info(
"Restoring SA&T variables from checkpoint file")
saver.restore(sess, model_path)
else:
def restore_fn(sess):
tf.logging.info(
"Restoring SA&T variables from pretrained model")
#saver.restore(sess, "/home/ubuntu/code/A_skip_thoughts_2/skip_thoughts/model/backup/run1/model.ckpt-2111")
pretrain_saver.restore(sess, "/home/ubuntu/code/pretrained/bi/model.ckpt-500008")
tf.contrib.slim.learning.train(
train_op=train_tensor,
logdir=FLAGS.train_dir,
graph=g,
global_step=model.global_step,
number_of_steps=training_config.number_of_steps,
save_summaries_secs=training_config.save_summaries_secs,
saver=saver,
save_interval_secs=training_config.save_model_secs,
init_fn = restore_fn)
def main(unused_argv):
if not FLAGS.input_file_pattern:
raise ValueError("--input_file_pattern is required.")
if not FLAGS.train_dir:
raise ValueError("--train_dir is required.")
model_config = configuration.model_config(
input_file_pattern=FLAGS.input_file_pattern)
training_config = configuration.training_config()
tf.logging.info("Building training graph.")
g = tf.Graph()
with g.as_default():
grads_tower = []
for dev_ind in range(4):
with tf.device('/gpu:%d' % dev_ind):
model = skip_thoughts_model.SkipThoughtsModel(model_config,
mode="train")
model.build()
learning_rate = _setup_learning_rate(training_config,
model.global_step)
optimizer = tf.train.AdamOptimizer(learning_rate)
total_loss = model.total_loss
# Update ops use GraphKeys.UPDATE_OPS collection if update_ops is None.
update_ops = set(ops.get_collection(ops.GraphKeys.UPDATE_OPS))
# Make sure update_ops are computed before total_loss.
if update_ops:
with ops.control_dependencies(update_ops):
barrier = control_flow_ops.no_op(name='update_barrier')
total_loss = control_flow_ops.with_dependencies([barrier],
total_loss)
variables_to_train = tf_variables.trainable_variables()
assert variables_to_train
gate_gradients = tf_optimizer.Optimizer.GATE_OP
# Create the gradients. Note that apply_gradients adds the gradient
# computation to the current graph.
grads = optimizer.compute_gradients(
total_loss,
variables_to_train,
gate_gradients=gate_gradients,
aggregation_method=None,
colocate_gradients_with_ops=False)
grads = tf.contrib.slim.learning.clip_gradient_norms(
grads, training_config.clip_gradient_norm)
grads_tower.append(grads)
avg_grads = average_gradients.average_gradients(grads_tower)
# Create gradient updates.
grad_updates = optimizer.apply_gradients(avg_grads,
global_step=model.global_step)
with ops.name_scope('train_op'):
# Make sure total_loss is valid.
total_loss = array_ops.check_numerics(total_loss,
'LossTensor is inf or nan')
# Ensure the train_tensor computes grad_updates.
train_op = control_flow_ops.with_dependencies([grad_updates],
total_loss)
# Add the operation used for training to the 'train_op' collection
train_ops = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
if train_op not in train_ops:
train_ops.append(train_op)
saver = tf.train.Saver()
tf.contrib.slim.learning.train(
train_op=train_op,
logdir=FLAGS.train_dir,
graph=g,
global_step=model.global_step,
number_of_steps=training_config.number_of_steps,
save_summaries_secs=training_config.save_summaries_secs,
saver=saver,
save_interval_secs=training_config.save_model_secs)
def main(unused_argv):
if not FLAGS.input_file_pattern:
raise ValueError("--input_file_pattern is required.")
if not FLAGS.run_dir:
raise ValueError("--run_dir is required.")
if not FLAGS.decoder:
raise ValueError("--decoder is required.")
if not FLAGS.train_dir:
train_dir = os.path.join(FLAGS.run_dir,
'run_{t}'.format(t=time.time()))
tf.logging.info(
"No specified --train_dir. Creating {d}.".format(d=train_dir))
os.makedirs(train_dir)
write_config(train_dir=train_dir, flags=FLAGS)
else:
tf.logging.info("Specified --train_dir {d}; Not autocreating.".format(
d=FLAGS.train_dir))
train_dir = FLAGS.train_dir
decoder_config = experiments.get_decoder_config(flags=FLAGS)
model_config = configuration.model_config(
input_file_pattern=FLAGS.input_file_pattern,
vocab_size=FLAGS.vocab_size,
batch_size=FLAGS.batch_size,
word_embedding_dim=FLAGS.word_dim,
pretrained_word_emb_file=FLAGS.pretrained_word_emb_file,
word_emb_trainable=FLAGS.word_emb_trainable,
encoder_dim=FLAGS.encoder_dim,
skipgram_encoder=FLAGS.skipgram_encoder,
sequence_decoder_pre=decoder_config.sequence_decoder_pre,
sequence_decoder_cur=decoder_config.sequence_decoder_cur,
sequence_decoder_post=decoder_config.sequence_decoder_post,
skipgram_decoder_pre=decoder_config.skipgram_decoder_pre,
skipgram_decoder_cur=decoder_config.skipgram_decoder_cur,
skipgram_decoder_post=decoder_config.skipgram_decoder_post,
share_weights_logits=FLAGS.share_weights_logits,
normalise_decoder_losses=FLAGS.normalise_decoder_losses,
skipgram_prefactor=FLAGS.skipgram_prefactor,
sequence_prefactor=FLAGS.sequence_prefactor)
training_config = configuration.training_config(
number_of_steps=FLAGS.number_of_steps)
tf.logging.info("Building training graph.")
g = tf.Graph()
with g.as_default():
tf.set_random_seed(1234)
model = skip_thoughts_model.SkipThoughtsModel(model_config,
mode="train")
model.build()
learning_rate = _setup_learning_rate(training_config,
model.global_step)
optimizer = tf.train.AdamOptimizer(learning_rate)
train_tensor = tf.contrib.slim.learning.create_train_op(
total_loss=model.total_loss,
optimizer=optimizer,
global_step=model.global_step,
clip_gradient_norm=training_config.clip_gradient_norm,
summarize_gradients=True,
check_numerics=True)
saver = tf.train.Saver()
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_fraction)
tf.contrib.slim.learning.train(
train_op=train_tensor,
logdir=train_dir,
graph=g,
global_step=model.global_step,
number_of_steps=training_config.number_of_steps,
session_config=tf.ConfigProto(gpu_options=gpu_options),
save_summaries_secs=training_config.save_summaries_secs,
saver=saver,
save_interval_secs=training_config.save_model_secs)
