def rollout_generator(hparams, inputs, input_present, is_training, is_validating, reuse=None): """Define the Generator graph which does rollouts. G will now impute tokens that have been masked from the input seqeunce. """ rollouts = [] with tf.variable_scope('gen_rollout'): for n in xrange(FLAGS.num_rollouts): if n > 0: # TODO(liamfedus): Why is it necessary here to manually set reuse? reuse = True tf.get_variable_scope().reuse_variables() [sequence, logits, log_probs] = model_construction.create_generator(hparams, inputs, input_present, is_training, is_validating, reuse=reuse) rollouts.append([sequence, logits, log_probs]) # Length assertion. assert len(rollouts) == FLAGS.num_rollouts return rollouts
def rollout_generator(hparams, inputs, input_present, is_training, is_validating, reuse=None): """Define the Generator graph which does rollouts. G will now impute tokens that have been masked from the input seqeunce. """ rollouts = [] with tf.variable_scope('gen_rollout'): for n in xrange(FLAGS.num_rollouts): if n > 0: # TODO(liamfedus): Why is it necessary here to manually set reuse? reuse = True tf.get_variable_scope().reuse_variables() [sequence, logits, log_probs] = model_construction.create_generator( hparams, inputs, input_present, is_training, is_validating, reuse=reuse) rollouts.append([sequence, logits, log_probs]) # Length assertion. assert len(rollouts) == FLAGS.num_rollouts return rollouts
def create_rollout_MaskGAN(hparams, is_training): """Create the MaskGAN model. Args: hparams: Hyperparameters for the MaskGAN. is_training: Boolean indicating operational mode (train/inference). evaluated with a teacher forcing regime. Return: model: Namedtuple for specifying the MaskGAN.""" global_step = tf.Variable(0, name='global_step', trainable=False) new_learning_rate = tf.placeholder(tf.float32, [], name='new_learning_rate') learning_rate = tf.Variable(0.0, name='learning_rate', trainable=False) learning_rate_update = tf.assign(learning_rate, new_learning_rate) new_rate = tf.placeholder(tf.float32, [], name='new_rate') percent_real_var = tf.Variable(0.0, trainable=False) percent_real_update = tf.assign(percent_real_var, new_rate) ## Placeholders. inputs = tf.placeholder(tf.int32, shape=[FLAGS.batch_size, FLAGS.sequence_length]) present = tf.placeholder(tf.bool, shape=[FLAGS.batch_size, FLAGS.sequence_length]) inv_present = tf.placeholder( tf.bool, shape=[FLAGS.batch_size, FLAGS.sequence_length]) ## Rollout Generator. fwd_gen_rollouts = rollout_generator(hparams, inputs, present, is_training=is_training, is_validating=False) inv_gen_rollouts = rollout_generator(hparams, inputs, inv_present, is_training=is_training, is_validating=False, reuse=True) ## Rollout Discriminator. fwd_dis_rollouts = rollout_discriminator(hparams, fwd_gen_rollouts, is_training=is_training) inv_dis_rollouts = rollout_discriminator(hparams, inv_gen_rollouts, is_training=is_training, reuse=True) ## Discriminator Loss. [dis_loss, dis_loss_pred, dis_loss_inv_pred] = rollout_discriminator_loss(fwd_dis_rollouts, present, inv_dis_rollouts, inv_present) ## Average log-perplexity for only missing words. However, to do this, # the logits are still computed using teacher forcing, that is, the ground # truth tokens are fed in at each time point to be valid. # TODO(liamfedus): Fix the naming convention. with tf.variable_scope('gen_rollout'): _, fwd_eval_logits, _ = model_construction.create_generator( hparams, inputs, present, is_training=False, is_validating=True, reuse=True) avg_log_perplexity = model_losses.calculate_log_perplexity( fwd_eval_logits, inputs, present) ## Generator Loss. # 1. Cross Entropy losses on missing tokens. [fwd_cross_entropy_losses, inv_cross_entropy_losses ] = rollout_masked_cross_entropy_loss(inputs, present, inv_present, fwd_gen_rollouts, inv_gen_rollouts) # 2. GAN losses on missing tokens. [fwd_RL_loss, fwd_RL_statistics, fwd_averages_op] = rollout_reinforce_objective(hparams, fwd_gen_rollouts, fwd_dis_rollouts, present) [inv_RL_loss, inv_RL_statistics, inv_averages_op] = rollout_reinforce_objective(hparams, inv_gen_rollouts, inv_dis_rollouts, inv_present) # TODO(liamfedus): Generalize this to use all logs. [fwd_sequence, fwd_logits, fwd_log_probs] = fwd_gen_rollouts[-1] [inv_sequence, inv_logits, inv_log_probs] = inv_gen_rollouts[-1] # TODO(liamfedus): Generalize this to use all logs. fwd_predictions = fwd_dis_rollouts[-1] inv_predictions = inv_dis_rollouts[-1] # TODO(liamfedus): Generalize this to use all logs. [fwd_log_probs, fwd_rewards, fwd_advantages, fwd_baselines] = fwd_RL_statistics[-1] [inv_log_probs, inv_rewards, inv_advantages, inv_baselines] = inv_RL_statistics[-1] ## Pre-training. if FLAGS.gen_pretrain_steps: # TODO(liamfedus): Rewrite this. fwd_cross_entropy_loss = tf.reduce_mean(fwd_cross_entropy_losses) gen_pretrain_op = model_optimization.create_gen_pretrain_op( hparams, fwd_cross_entropy_loss, global_step) else: gen_pretrain_op = tf.no_op('gen_pretrain_no_op') if FLAGS.dis_pretrain_steps: dis_pretrain_op = model_optimization.create_dis_pretrain_op( hparams, dis_loss, global_step) else: dis_pretrain_op = tf.no_op('dis_pretrain_no_op') ## Generator Train Op. # 1. Cross-Entropy. if FLAGS.gen_training_strategy == 'cross_entropy': gen_loss = tf.reduce_mean(fwd_cross_entropy_losses + inv_cross_entropy_losses) / 2. [gen_train_op, gen_grads, gen_vars] = model_optimization.create_gen_train_op(hparams, learning_rate, gen_loss, global_step, mode='MINIMIZE') # 2. GAN (REINFORCE) elif FLAGS.gen_training_strategy == 'reinforce': gen_loss = (fwd_RL_loss + inv_RL_loss) / 2. [gen_train_op, gen_grads, gen_vars] = model_optimization.create_reinforce_gen_train_op( hparams, learning_rate, gen_loss, fwd_averages_op, inv_averages_op, global_step) else: raise NotImplementedError ## Discriminator Train Op. dis_train_op, dis_grads, dis_vars = model_optimization.create_dis_train_op( hparams, dis_loss, global_step) ## Summaries. with tf.name_scope('general'): tf.summary.scalar('percent_real', percent_real_var) tf.summary.scalar('learning_rate', learning_rate) with tf.name_scope('generator_losses'): tf.summary.scalar('gen_loss', tf.reduce_mean(gen_loss)) tf.summary.scalar('gen_loss_fwd_cross_entropy', tf.reduce_mean(fwd_cross_entropy_losses)) tf.summary.scalar('gen_loss_inv_cross_entropy', tf.reduce_mean(inv_cross_entropy_losses)) with tf.name_scope('REINFORCE'): with tf.name_scope('objective'): tf.summary.scalar('fwd_RL_loss', tf.reduce_mean(fwd_RL_loss)) tf.summary.scalar('inv_RL_loss', tf.reduce_mean(inv_RL_loss)) with tf.name_scope('rewards'): helper.variable_summaries(fwd_rewards, 'fwd_rewards') helper.variable_summaries(inv_rewards, 'inv_rewards') with tf.name_scope('advantages'): helper.variable_summaries(fwd_advantages, 'fwd_advantages') helper.variable_summaries(inv_advantages, 'inv_advantages') with tf.name_scope('baselines'): helper.variable_summaries(fwd_baselines, 'fwd_baselines') helper.variable_summaries(inv_baselines, 'inv_baselines') with tf.name_scope('log_probs'): helper.variable_summaries(fwd_log_probs, 'fwd_log_probs') helper.variable_summaries(inv_log_probs, 'inv_log_probs') with tf.name_scope('discriminator_losses'): tf.summary.scalar('dis_loss', dis_loss) tf.summary.scalar('dis_loss_fwd_sequence', dis_loss_pred) tf.summary.scalar('dis_loss_inv_sequence', dis_loss_inv_pred) with tf.name_scope('logits'): helper.variable_summaries(fwd_logits, 'fwd_logits') helper.variable_summaries(inv_logits, 'inv_logits') for v, g in zip(gen_vars, gen_grads): helper.variable_summaries(v, v.op.name) helper.variable_summaries(g, 'grad/' + v.op.name) for v, g in zip(dis_vars, dis_grads): helper.variable_summaries(v, v.op.name) helper.variable_summaries(g, 'grad/' + v.op.name) merge_summaries_op = tf.summary.merge_all() # Model saver. saver = tf.train.Saver(keep_checkpoint_every_n_hours=1, max_to_keep=5) # Named tuple that captures elements of the MaskGAN model. Model = collections.namedtuple('Model', [ 'inputs', 'present', 'inv_present', 'percent_real_update', 'new_rate', 'fwd_sequence', 'fwd_logits', 'fwd_rewards', 'fwd_advantages', 'fwd_log_probs', 'fwd_predictions', 'fwd_cross_entropy_losses', 'inv_sequence', 'inv_logits', 'inv_rewards', 'inv_advantages', 'inv_log_probs', 'inv_predictions', 'inv_cross_entropy_losses', 'avg_log_perplexity', 'dis_loss', 'gen_loss', 'dis_train_op', 'gen_train_op', 'gen_pretrain_op', 'dis_pretrain_op', 'merge_summaries_op', 'global_step', 'new_learning_rate', 'learning_rate_update', 'saver' ]) model = Model(inputs, present, inv_present, percent_real_update, new_rate, fwd_sequence, fwd_logits, fwd_rewards, fwd_advantages, fwd_log_probs, fwd_predictions, fwd_cross_entropy_losses, inv_sequence, inv_logits, inv_rewards, inv_advantages, inv_log_probs, inv_predictions, inv_cross_entropy_losses, avg_log_perplexity, dis_loss, gen_loss, dis_train_op, gen_train_op, gen_pretrain_op, dis_pretrain_op, merge_summaries_op, global_step, new_learning_rate, learning_rate_update, saver) return model
def create_MaskGAN(hparams, is_training): """Create the MaskGAN model. Args: hparams: Hyperparameters for the MaskGAN. is_training: Boolean indicating operational mode (train/inference). evaluated with a teacher forcing regime. Return: model: Namedtuple for specifying the MaskGAN. """ global_step = tf.Variable(0, name='global_step', trainable=False) new_learning_rate = tf.placeholder(tf.float32, [], name='new_learning_rate') learning_rate = tf.Variable(0.0, name='learning_rate', trainable=False) learning_rate_update = tf.assign(learning_rate, new_learning_rate) new_rate = tf.placeholder(tf.float32, [], name='new_rate') percent_real_var = tf.Variable(0.0, trainable=False) percent_real_update = tf.assign(percent_real_var, new_rate) ## Placeholders. inputs = tf.placeholder( tf.int32, shape=[FLAGS.batch_size, FLAGS.sequence_length]) targets = tf.placeholder( tf.int32, shape=[FLAGS.batch_size, FLAGS.sequence_length]) present = tf.placeholder( tf.bool, shape=[FLAGS.batch_size, FLAGS.sequence_length]) # TODO(adai): Placeholder for IMDB label. ## Real Sequence is the targets. real_sequence = targets ## Fakse Sequence from the Generator. # TODO(adai): Generator must have IMDB labels placeholder. (fake_sequence, fake_logits, fake_log_probs, fake_gen_initial_state, fake_gen_final_state, _) = model_construction.create_generator( hparams, inputs, targets, present, is_training=is_training, is_validating=False) (_, eval_logits, _, eval_initial_state, eval_final_state, _) = model_construction.create_generator( hparams, inputs, targets, present, is_training=False, is_validating=True, reuse=True) ## Discriminator. fake_predictions = model_construction.create_discriminator( hparams, fake_sequence, is_training=is_training, inputs=inputs, present=present) real_predictions = model_construction.create_discriminator( hparams, real_sequence, is_training=is_training, reuse=True, inputs=inputs, present=present) ## Critic. # The critic will be used to estimate the forward rewards to the Generator. if FLAGS.baseline_method == 'critic': est_state_values = model_construction.create_critic( hparams, fake_sequence, is_training=is_training) else: est_state_values = None ## Discriminator Loss. [dis_loss, dis_loss_fake, dis_loss_real] = model_losses.create_dis_loss( fake_predictions, real_predictions, present) ## Average log-perplexity for only missing words. However, to do this, # the logits are still computed using teacher forcing, that is, the ground # truth tokens are fed in at each time point to be valid. avg_log_perplexity = model_losses.calculate_log_perplexity( eval_logits, targets, present) ## Generator Objective. # 1. Cross Entropy losses on missing tokens. fake_cross_entropy_losses = model_losses.create_masked_cross_entropy_loss( targets, present, fake_logits) # 2. GAN REINFORCE losses. [ fake_RL_loss, fake_log_probs, fake_rewards, fake_advantages, fake_baselines, fake_averages_op, critic_loss, cumulative_rewards ] = model_losses.calculate_reinforce_objective( hparams, fake_log_probs, fake_predictions, present, est_state_values) ## Pre-training. if FLAGS.gen_pretrain_steps: raise NotImplementedError # # TODO(liamfedus): Rewrite this. # fwd_cross_entropy_loss = tf.reduce_mean(fwd_cross_entropy_losses) # gen_pretrain_op = model_optimization.create_gen_pretrain_op( # hparams, fwd_cross_entropy_loss, global_step) else: gen_pretrain_op = None if FLAGS.dis_pretrain_steps: dis_pretrain_op = model_optimization.create_dis_pretrain_op( hparams, dis_loss, global_step) else: dis_pretrain_op = None ## Generator Train Op. # 1. Cross-Entropy. if FLAGS.gen_training_strategy == 'cross_entropy': gen_loss = tf.reduce_mean(fake_cross_entropy_losses) [gen_train_op, gen_grads, gen_vars] = model_optimization.create_gen_train_op( hparams, learning_rate, gen_loss, global_step, mode='MINIMIZE') # 2. GAN (REINFORCE) elif FLAGS.gen_training_strategy == 'reinforce': gen_loss = fake_RL_loss [gen_train_op, gen_grads, gen_vars] = model_optimization.create_reinforce_gen_train_op( hparams, learning_rate, gen_loss, fake_averages_op, global_step) else: raise NotImplementedError ## Discriminator Train Op. dis_train_op, dis_grads, dis_vars = model_optimization.create_dis_train_op( hparams, dis_loss, global_step) ## Critic Train Op. if critic_loss is not None: [critic_train_op, _, _] = model_optimization.create_critic_train_op( hparams, critic_loss, global_step) dis_train_op = tf.group(dis_train_op, critic_train_op) ## Summaries. with tf.name_scope('general'): tf.summary.scalar('percent_real', percent_real_var) tf.summary.scalar('learning_rate', learning_rate) with tf.name_scope('generator_objectives'): tf.summary.scalar('gen_objective', tf.reduce_mean(gen_loss)) tf.summary.scalar('gen_loss_cross_entropy', tf.reduce_mean(fake_cross_entropy_losses)) with tf.name_scope('REINFORCE'): with tf.name_scope('objective'): tf.summary.scalar('fake_RL_loss', tf.reduce_mean(fake_RL_loss)) with tf.name_scope('rewards'): helper.variable_summaries(cumulative_rewards, 'rewards') with tf.name_scope('advantages'): helper.variable_summaries(fake_advantages, 'advantages') with tf.name_scope('baselines'): helper.variable_summaries(fake_baselines, 'baselines') with tf.name_scope('log_probs'): helper.variable_summaries(fake_log_probs, 'log_probs') with tf.name_scope('discriminator_losses'): tf.summary.scalar('dis_loss', dis_loss) tf.summary.scalar('dis_loss_fake_sequence', dis_loss_fake) tf.summary.scalar('dis_loss_prob_fake_sequence', tf.exp(-dis_loss_fake)) tf.summary.scalar('dis_loss_real_sequence', dis_loss_real) tf.summary.scalar('dis_loss_prob_real_sequence', tf.exp(-dis_loss_real)) if critic_loss is not None: with tf.name_scope('critic_losses'): tf.summary.scalar('critic_loss', critic_loss) with tf.name_scope('logits'): helper.variable_summaries(fake_logits, 'fake_logits') for v, g in zip(gen_vars, gen_grads): helper.variable_summaries(v, v.op.name) helper.variable_summaries(g, 'grad/' + v.op.name) for v, g in zip(dis_vars, dis_grads): helper.variable_summaries(v, v.op.name) helper.variable_summaries(g, 'grad/' + v.op.name) merge_summaries_op = tf.summary.merge_all() text_summary_placeholder = tf.placeholder(tf.string) text_summary_op = tf.summary.text('Samples', text_summary_placeholder) # Model saver. saver = tf.train.Saver(keep_checkpoint_every_n_hours=1, max_to_keep=5) # Named tuple that captures elements of the MaskGAN model. Model = collections.namedtuple('Model', [ 'inputs', 'targets', 'present', 'percent_real_update', 'new_rate', 'fake_sequence', 'fake_logits', 'fake_rewards', 'fake_baselines', 'fake_advantages', 'fake_log_probs', 'fake_predictions', 'real_predictions', 'fake_cross_entropy_losses', 'fake_gen_initial_state', 'fake_gen_final_state', 'eval_initial_state', 'eval_final_state', 'avg_log_perplexity', 'dis_loss', 'gen_loss', 'critic_loss', 'cumulative_rewards', 'dis_train_op', 'gen_train_op', 'gen_pretrain_op', 'dis_pretrain_op', 'merge_summaries_op', 'global_step', 'new_learning_rate', 'learning_rate_update', 'saver', 'text_summary_op', 'text_summary_placeholder' ]) model = Model( inputs, targets, present, percent_real_update, new_rate, fake_sequence, fake_logits, fake_rewards, fake_baselines, fake_advantages, fake_log_probs, fake_predictions, real_predictions, fake_cross_entropy_losses, fake_gen_initial_state, fake_gen_final_state, eval_initial_state, eval_final_state, avg_log_perplexity, dis_loss, gen_loss, critic_loss, cumulative_rewards, dis_train_op, gen_train_op, gen_pretrain_op, dis_pretrain_op, merge_summaries_op, global_step, new_learning_rate, learning_rate_update, saver, text_summary_op, text_summary_placeholder) return model
def create_rollout_MaskGAN(hparams, is_training): """Create the MaskGAN model. Args: hparams: Hyperparameters for the MaskGAN. is_training: Boolean indicating operational mode (train/inference). evaluated with a teacher forcing regime. Return: model: Namedtuple for specifying the MaskGAN.""" global_step = tf.Variable(0, name='global_step', trainable=False) new_learning_rate = tf.placeholder(tf.float32, [], name='new_learning_rate') learning_rate = tf.Variable(0.0, name='learning_rate', trainable=False) learning_rate_update = tf.assign(learning_rate, new_learning_rate) new_rate = tf.placeholder(tf.float32, [], name='new_rate') percent_real_var = tf.Variable(0.0, trainable=False) percent_real_update = tf.assign(percent_real_var, new_rate) ## Placeholders. inputs = tf.placeholder( tf.int32, shape=[FLAGS.batch_size, FLAGS.sequence_length]) present = tf.placeholder( tf.bool, shape=[FLAGS.batch_size, FLAGS.sequence_length]) inv_present = tf.placeholder( tf.bool, shape=[FLAGS.batch_size, FLAGS.sequence_length]) ## Rollout Generator. fwd_gen_rollouts = rollout_generator( hparams, inputs, present, is_training=is_training, is_validating=False) inv_gen_rollouts = rollout_generator( hparams, inputs, inv_present, is_training=is_training, is_validating=False, reuse=True) ## Rollout Discriminator. fwd_dis_rollouts = rollout_discriminator( hparams, fwd_gen_rollouts, is_training=is_training) inv_dis_rollouts = rollout_discriminator( hparams, inv_gen_rollouts, is_training=is_training, reuse=True) ## Discriminator Loss. [dis_loss, dis_loss_pred, dis_loss_inv_pred] = rollout_discriminator_loss( fwd_dis_rollouts, present, inv_dis_rollouts, inv_present) ## Average log-perplexity for only missing words. However, to do this, # the logits are still computed using teacher forcing, that is, the ground # truth tokens are fed in at each time point to be valid. # TODO(liamfedus): Fix the naming convention. with tf.variable_scope('gen_rollout'): _, fwd_eval_logits, _ = model_construction.create_generator( hparams, inputs, present, is_training=False, is_validating=True, reuse=True) avg_log_perplexity = model_losses.calculate_log_perplexity( fwd_eval_logits, inputs, present) ## Generator Loss. # 1. Cross Entropy losses on missing tokens. [fwd_cross_entropy_losses, inv_cross_entropy_losses] = rollout_masked_cross_entropy_loss( inputs, present, inv_present, fwd_gen_rollouts, inv_gen_rollouts) # 2. GAN losses on missing tokens. [fwd_RL_loss, fwd_RL_statistics, fwd_averages_op] = rollout_reinforce_objective( hparams, fwd_gen_rollouts, fwd_dis_rollouts, present) [inv_RL_loss, inv_RL_statistics, inv_averages_op] = rollout_reinforce_objective( hparams, inv_gen_rollouts, inv_dis_rollouts, inv_present) # TODO(liamfedus): Generalize this to use all logs. [fwd_sequence, fwd_logits, fwd_log_probs] = fwd_gen_rollouts[-1] [inv_sequence, inv_logits, inv_log_probs] = inv_gen_rollouts[-1] # TODO(liamfedus): Generalize this to use all logs. fwd_predictions = fwd_dis_rollouts[-1] inv_predictions = inv_dis_rollouts[-1] # TODO(liamfedus): Generalize this to use all logs. [fwd_log_probs, fwd_rewards, fwd_advantages, fwd_baselines] = fwd_RL_statistics[-1] [inv_log_probs, inv_rewards, inv_advantages, inv_baselines] = inv_RL_statistics[-1] ## Pre-training. if FLAGS.gen_pretrain_steps: # TODO(liamfedus): Rewrite this. fwd_cross_entropy_loss = tf.reduce_mean(fwd_cross_entropy_losses) gen_pretrain_op = model_optimization.create_gen_pretrain_op( hparams, fwd_cross_entropy_loss, global_step) else: gen_pretrain_op = tf.no_op('gen_pretrain_no_op') if FLAGS.dis_pretrain_steps: dis_pretrain_op = model_optimization.create_dis_pretrain_op( hparams, dis_loss, global_step) else: dis_pretrain_op = tf.no_op('dis_pretrain_no_op') ## Generator Train Op. # 1. Cross-Entropy. if FLAGS.gen_training_strategy == 'cross_entropy': gen_loss = tf.reduce_mean( fwd_cross_entropy_losses + inv_cross_entropy_losses) / 2. [gen_train_op, gen_grads, gen_vars] = model_optimization.create_gen_train_op( hparams, learning_rate, gen_loss, global_step, mode='MINIMIZE') # 2. GAN (REINFORCE) elif FLAGS.gen_training_strategy == 'reinforce': gen_loss = (fwd_RL_loss + inv_RL_loss) / 2. [gen_train_op, gen_grads, gen_vars] = model_optimization.create_reinforce_gen_train_op( hparams, learning_rate, gen_loss, fwd_averages_op, inv_averages_op, global_step) else: raise NotImplementedError ## Discriminator Train Op. dis_train_op, dis_grads, dis_vars = model_optimization.create_dis_train_op( hparams, dis_loss, global_step) ## Summaries. with tf.name_scope('general'): tf.summary.scalar('percent_real', percent_real_var) tf.summary.scalar('learning_rate', learning_rate) with tf.name_scope('generator_losses'): tf.summary.scalar('gen_loss', tf.reduce_mean(gen_loss)) tf.summary.scalar('gen_loss_fwd_cross_entropy', tf.reduce_mean(fwd_cross_entropy_losses)) tf.summary.scalar('gen_loss_inv_cross_entropy', tf.reduce_mean(inv_cross_entropy_losses)) with tf.name_scope('REINFORCE'): with tf.name_scope('objective'): tf.summary.scalar('fwd_RL_loss', tf.reduce_mean(fwd_RL_loss)) tf.summary.scalar('inv_RL_loss', tf.reduce_mean(inv_RL_loss)) with tf.name_scope('rewards'): helper.variable_summaries(fwd_rewards, 'fwd_rewards') helper.variable_summaries(inv_rewards, 'inv_rewards') with tf.name_scope('advantages'): helper.variable_summaries(fwd_advantages, 'fwd_advantages') helper.variable_summaries(inv_advantages, 'inv_advantages') with tf.name_scope('baselines'): helper.variable_summaries(fwd_baselines, 'fwd_baselines') helper.variable_summaries(inv_baselines, 'inv_baselines') with tf.name_scope('log_probs'): helper.variable_summaries(fwd_log_probs, 'fwd_log_probs') helper.variable_summaries(inv_log_probs, 'inv_log_probs') with tf.name_scope('discriminator_losses'): tf.summary.scalar('dis_loss', dis_loss) tf.summary.scalar('dis_loss_fwd_sequence', dis_loss_pred) tf.summary.scalar('dis_loss_inv_sequence', dis_loss_inv_pred) with tf.name_scope('logits'): helper.variable_summaries(fwd_logits, 'fwd_logits') helper.variable_summaries(inv_logits, 'inv_logits') for v, g in zip(gen_vars, gen_grads): helper.variable_summaries(v, v.op.name) helper.variable_summaries(g, 'grad/' + v.op.name) for v, g in zip(dis_vars, dis_grads): helper.variable_summaries(v, v.op.name) helper.variable_summaries(g, 'grad/' + v.op.name) merge_summaries_op = tf.summary.merge_all() # Model saver. saver = tf.train.Saver(keep_checkpoint_every_n_hours=1, max_to_keep=5) # Named tuple that captures elements of the MaskGAN model. Model = collections.namedtuple('Model', [ 'inputs', 'present', 'inv_present', 'percent_real_update', 'new_rate', 'fwd_sequence', 'fwd_logits', 'fwd_rewards', 'fwd_advantages', 'fwd_log_probs', 'fwd_predictions', 'fwd_cross_entropy_losses', 'inv_sequence', 'inv_logits', 'inv_rewards', 'inv_advantages', 'inv_log_probs', 'inv_predictions', 'inv_cross_entropy_losses', 'avg_log_perplexity', 'dis_loss', 'gen_loss', 'dis_train_op', 'gen_train_op', 'gen_pretrain_op', 'dis_pretrain_op', 'merge_summaries_op', 'global_step', 'new_learning_rate', 'learning_rate_update', 'saver' ]) model = Model( inputs, present, inv_present, percent_real_update, new_rate, fwd_sequence, fwd_logits, fwd_rewards, fwd_advantages, fwd_log_probs, fwd_predictions, fwd_cross_entropy_losses, inv_sequence, inv_logits, inv_rewards, inv_advantages, inv_log_probs, inv_predictions, inv_cross_entropy_losses, avg_log_perplexity, dis_loss, gen_loss, dis_train_op, gen_train_op, gen_pretrain_op, dis_pretrain_op, merge_summaries_op, global_step, new_learning_rate, learning_rate_update, saver) return model
def create_MaskGAN(hparams, is_training): """Create the MaskGAN model. Args: hparams: Hyperparameters for the MaskGAN. is_training: Boolean indicating operational mode (train/inference). evaluated with a teacher forcing regime. Return: model: Namedtuple for specifying the MaskGAN. """ global_step = tf.Variable(0, name='global_step', trainable=False) new_learning_rate = tf.placeholder(tf.float32, [], name='new_learning_rate') learning_rate = tf.Variable(0.0, name='learning_rate', trainable=False) learning_rate_update = tf.assign(learning_rate, new_learning_rate) new_rate = tf.placeholder(tf.float32, [], name='new_rate') percent_real_var = tf.Variable(0.0, trainable=False) percent_real_update = tf.assign(percent_real_var, new_rate) ## Placeholders. inputs = tf.placeholder( tf.int32, shape=[FLAGS.batch_size, FLAGS.sequence_length]) targets = tf.placeholder( tf.int32, shape=[FLAGS.batch_size, FLAGS.sequence_length]) present = tf.placeholder( tf.bool, shape=[FLAGS.batch_size, FLAGS.sequence_length]) # TODO(adai): Placeholder for IMDB label. ## Real Sequence is the targets. real_sequence = targets ## Fakse Sequence from the Generator. # TODO(adai): Generator must have IMDB labels placeholder. (fake_sequence, fake_logits, fake_log_probs, fake_gen_initial_state, fake_gen_final_state, _) = model_construction.create_generator( hparams, inputs, targets, present, is_training=is_training, is_validating=False) (_, eval_logits, _, eval_initial_state, eval_final_state, _) = model_construction.create_generator( hparams, inputs, targets, present, is_training=False, is_validating=True, reuse=True) ## Discriminator. fake_predictions = model_construction.create_discriminator( hparams, fake_sequence, is_training=is_training, inputs=inputs, present=present) real_predictions = model_construction.create_discriminator( hparams, real_sequence, is_training=is_training, reuse=True, inputs=inputs, present=present) ## Critic. # The critic will be used to estimate the forward rewards to the Generator. if FLAGS.baseline_method == 'critic': est_state_values = model_construction.create_critic( hparams, fake_sequence, is_training=is_training) else: est_state_values = None ## Discriminator Loss. [dis_loss, dis_loss_fake, dis_loss_real] = model_losses.create_dis_loss( fake_predictions, real_predictions, present) ## Average log-perplexity for only missing words. However, to do this, # the logits are still computed using teacher forcing, that is, the ground # truth tokens are fed in at each time point to be valid. avg_log_perplexity = model_losses.calculate_log_perplexity( eval_logits, targets, present) ## Generator Objective. # 1. Cross Entropy losses on missing tokens. fake_cross_entropy_losses = model_losses.create_masked_cross_entropy_loss( targets, present, fake_logits) # 2. GAN REINFORCE losses. [ fake_RL_loss, fake_log_probs, fake_rewards, fake_advantages, fake_baselines, fake_averages_op, critic_loss, cumulative_rewards ] = model_losses.calculate_reinforce_objective( hparams, fake_log_probs, fake_predictions, present, est_state_values) ## Pre-training. if FLAGS.gen_pretrain_steps: raise NotImplementedError # # TODO(liamfedus): Rewrite this. # fwd_cross_entropy_loss = tf.reduce_mean(fwd_cross_entropy_losses) # gen_pretrain_op = model_optimization.create_gen_pretrain_op( # hparams, fwd_cross_entropy_loss, global_step) else: gen_pretrain_op = None if FLAGS.dis_pretrain_steps: dis_pretrain_op = model_optimization.create_dis_pretrain_op( hparams, dis_loss, global_step) else: dis_pretrain_op = None ## Generator Train Op. # 1. Cross-Entropy. if FLAGS.gen_training_strategy == 'cross_entropy': gen_loss = tf.reduce_mean(fake_cross_entropy_losses) [gen_train_op, gen_grads, gen_vars] = model_optimization.create_gen_train_op( hparams, learning_rate, gen_loss, global_step, mode='MINIMIZE') # 2. GAN (REINFORCE) elif FLAGS.gen_training_strategy == 'reinforce': gen_loss = fake_RL_loss [gen_train_op, gen_grads, gen_vars] = model_optimization.create_reinforce_gen_train_op( hparams, learning_rate, gen_loss, fake_averages_op, global_step) else: raise NotImplementedError ## Discriminator Train Op. dis_train_op, dis_grads, dis_vars = model_optimization.create_dis_train_op( hparams, dis_loss, global_step) ## Critic Train Op. if critic_loss is not None: [critic_train_op, _, _] = model_optimization.create_critic_train_op( hparams, critic_loss, global_step) dis_train_op = tf.group(dis_train_op, critic_train_op) ## Summaries. with tf.name_scope('general'): tf.summary.scalar('percent_real', percent_real_var) tf.summary.scalar('learning_rate', learning_rate) with tf.name_scope('generator_objectives'): tf.summary.scalar('gen_objective', tf.reduce_mean(gen_loss)) tf.summary.scalar('gen_loss_cross_entropy', tf.reduce_mean(fake_cross_entropy_losses)) with tf.name_scope('REINFORCE'): with tf.name_scope('objective'): tf.summary.scalar('fake_RL_loss', tf.reduce_mean(fake_RL_loss)) with tf.name_scope('rewards'): helper.variable_summaries(cumulative_rewards, 'rewards') with tf.name_scope('advantages'): helper.variable_summaries(fake_advantages, 'advantages') with tf.name_scope('baselines'): helper.variable_summaries(fake_baselines, 'baselines') with tf.name_scope('log_probs'): helper.variable_summaries(fake_log_probs, 'log_probs') with tf.name_scope('discriminator_losses'): tf.summary.scalar('dis_loss', dis_loss) tf.summary.scalar('dis_loss_fake_sequence', dis_loss_fake) tf.summary.scalar('dis_loss_prob_fake_sequence', tf.exp(-dis_loss_fake)) tf.summary.scalar('dis_loss_real_sequence', dis_loss_real) tf.summary.scalar('dis_loss_prob_real_sequence', tf.exp(-dis_loss_real)) if critic_loss is not None: with tf.name_scope('critic_losses'): tf.summary.scalar('critic_loss', critic_loss) with tf.name_scope('logits'): helper.variable_summaries(fake_logits, 'fake_logits') for v, g in zip(gen_vars, gen_grads): helper.variable_summaries(v, v.op.name) helper.variable_summaries(g, 'grad/' + v.op.name) for v, g in zip(dis_vars, dis_grads): helper.variable_summaries(v, v.op.name) helper.variable_summaries(g, 'grad/' + v.op.name) merge_summaries_op = tf.summary.merge_all() text_summary_placeholder = tf.placeholder(tf.string) text_summary_op = tf.summary.text('Samples', text_summary_placeholder) # Model saver. saver = tf.train.Saver(keep_checkpoint_every_n_hours=1, max_to_keep=5) # Named tuple that captures elements of the MaskGAN model. Model = collections.namedtuple('Model', [ 'inputs', 'targets', 'present', 'percent_real_update', 'new_rate', 'fake_sequence', 'fake_logits', 'fake_rewards', 'fake_baselines', 'fake_advantages', 'fake_log_probs', 'fake_predictions', 'real_predictions', 'fake_cross_entropy_losses', 'fake_gen_initial_state', 'fake_gen_final_state', 'eval_initial_state', 'eval_final_state', 'avg_log_perplexity', 'dis_loss', 'gen_loss', 'critic_loss', 'cumulative_rewards', 'dis_train_op', 'gen_train_op', 'gen_pretrain_op', 'dis_pretrain_op', 'merge_summaries_op', 'global_step', 'new_learning_rate', 'learning_rate_update', 'saver', 'text_summary_op', 'text_summary_placeholder' ]) model = Model( inputs, targets, present, percent_real_update, new_rate, fake_sequence, fake_logits, fake_rewards, fake_baselines, fake_advantages, fake_log_probs, fake_predictions, real_predictions, fake_cross_entropy_losses, fake_gen_initial_state, fake_gen_final_state, eval_initial_state, eval_final_state, avg_log_perplexity, dis_loss, gen_loss, critic_loss, cumulative_rewards, dis_train_op, gen_train_op, gen_pretrain_op, dis_pretrain_op, merge_summaries_op, global_step, new_learning_rate, learning_rate_update, saver, text_summary_op, text_summary_placeholder) return model