def build_graph(self): """Constructs the portion of the graph that belongs to this model.""" tf.logging.info('Initializing melody RNN graph for scope %s', self.scope) with self.graph.as_default(): with tf.device(lambda op: ''): with tf.variable_scope(self.scope): # Make an LSTM cell with the number and size of layers specified in # hparams. if self.note_rnn_type == 'basic_rnn': self.cell = events_rnn_graph.make_rnn_cell( self.hparams.rnn_layer_sizes) else: self.cell = rl_tuner_ops.make_rnn_cell( self.hparams.rnn_layer_sizes) # Shape of melody_sequence is batch size, melody length, number of # output note actions. self.melody_sequence = tf.placeholder( tf.float32, [None, None, self.hparams.one_hot_length], name='melody_sequence') self.lengths = tf.placeholder(tf.int32, [None], name='lengths') self.initial_state = tf.placeholder( tf.float32, [None, self.cell.state_size], name='initial_state') if self.training_file_list is not None: # Set up a tf queue to read melodies from the training data tfrecord (self.train_sequence, self.train_labels, self.train_lengths ) = sequence_example_lib.get_padded_batch( self.training_file_list, self.hparams.batch_size, self.hparams.one_hot_length) # Closure function is used so that this part of the graph can be # re-run in multiple places, such as __call__. def run_network_on_melody(m_seq, lens, initial_state, swap_memory=True, parallel_iterations=1): """Internal function that defines the RNN network structure. Args: m_seq: A batch of melody sequences of one-hot notes. lens: Lengths of the melody_sequences. initial_state: Vector representing the initial state of the RNN. swap_memory: Uses more memory and is faster. parallel_iterations: Argument to tf.nn.dynamic_rnn. Returns: Output of network (either softmax or logits) and RNN state. """ outputs, final_state = tf.nn.dynamic_rnn( self.cell, m_seq, sequence_length=lens, initial_state=initial_state, swap_memory=swap_memory, parallel_iterations=parallel_iterations) outputs_flat = tf.reshape( outputs, [-1, self.hparams.rnn_layer_sizes[-1]]) if self.note_rnn_type == 'basic_rnn': linear_layer = tf.contrib.layers.linear else: linear_layer = tf.contrib.layers.legacy_linear logits_flat = linear_layer(outputs_flat, self.hparams.one_hot_length) return logits_flat, final_state (self.logits, self.state_tensor) = run_network_on_melody( self.melody_sequence, self.lengths, self.initial_state) self.softmax = tf.nn.softmax(self.logits) self.run_network_on_melody = run_network_on_melody if self.training_file_list is not None: # Does not recreate the model architecture but rather uses it to feed # data from the training queue through the model. with tf.variable_scope(self.scope, reuse=True): zero_state = self.cell.zero_state( batch_size=self.hparams.batch_size, dtype=tf.float32) (self.train_logits, self.train_state) = run_network_on_melody( self.train_sequence, self.train_lengths, zero_state) self.train_softmax = tf.nn.softmax(self.train_logits)
def build_graph(mode, config, sequence_example_file_paths=None): """Builds the TensorFlow graph. Args: mode: 'train', 'eval', or 'generate'. Only mode related ops are added to the graph. config: An EventSequenceRnnConfig containing the encoder/decoder and HParams to use. sequence_example_file_paths: A list of paths to TFRecord files containing tf.train.SequenceExample protos. Only needed for training and evaluation. May be a sharded file of the form. Returns: A tf.Graph instance which contains the TF ops. Raises: ValueError: If mode is not 'train', 'eval', or 'generate'. """ if mode not in ('train', 'eval', 'generate'): raise ValueError("The mode parameter must be 'train', 'eval', " "or 'generate'. The mode parameter was: %s" % mode) hparams = config.hparams encoder_decoder = config.encoder_decoder tf.logging.info('hparams = %s', hparams.values()) input_size = encoder_decoder.input_size with tf.Graph().as_default() as graph: inputs, lengths = None, None if mode == 'train' or mode == 'eval': inputs, _, lengths = magenta.common.get_padded_batch( sequence_example_file_paths, hparams.batch_size, input_size, shuffle=mode == 'train') elif mode == 'generate': inputs = tf.placeholder(tf.float32, [hparams.batch_size, None, input_size]) cell = events_rnn_graph.make_rnn_cell( hparams.rnn_layer_sizes, dropout_keep_prob=hparams.dropout_keep_prob if mode == 'train' else 1.0, attn_length=( hparams.attn_length if hasattr(hparams, 'attn_length') else 0)) rnn_nade = RnnNade( cell, num_dims=input_size, num_hidden=hparams.nade_hidden_units) if mode == 'train' or mode == 'eval': log_probs, cond_probs = rnn_nade.log_prob(inputs, lengths) inputs_flat = tf.to_float( magenta.common.flatten_maybe_padded_sequences(inputs, lengths)) predictions_flat = tf.to_float(tf.greater_equal(cond_probs, .5)) if mode == 'train': loss = tf.reduce_mean(-log_probs) perplexity = tf.reduce_mean(tf.exp(log_probs)) correct_predictions = tf.to_float( tf.equal(inputs_flat, predictions_flat)) accuracy = tf.reduce_mean(correct_predictions) precision = (tf.reduce_sum(inputs_flat * predictions_flat) / tf.reduce_sum(predictions_flat)) recall = (tf.reduce_sum(inputs_flat * predictions_flat) / tf.reduce_sum(inputs_flat)) optimizer = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate) train_op = tf.contrib.slim.learning.create_train_op( loss, optimizer, clip_gradient_norm=hparams.clip_norm) tf.add_to_collection('train_op', train_op) vars_to_summarize = { 'loss': loss, 'metrics/perplexity': perplexity, 'metrics/accuracy': accuracy, 'metrics/precision': precision, 'metrics/recall': recall, } elif mode == 'eval': vars_to_summarize, update_ops = tf.contrib.metrics.aggregate_metric_map( { 'loss': tf.metrics.mean(-log_probs), 'metrics/perplexity': tf.metrics.mean(tf.exp(log_probs)), 'metrics/accuracy': tf.metrics.accuracy( inputs_flat, predictions_flat), 'metrics/precision': tf.metrics.precision( inputs_flat, predictions_flat), 'metrics/recall': tf.metrics.recall( inputs_flat, predictions_flat), }) for updates_op in update_ops.values(): tf.add_to_collection('eval_ops', updates_op) precision = vars_to_summarize['metrics/precision'] recall = vars_to_summarize['metrics/precision'] f1_score = tf.where( tf.greater(precision + recall, 0), 2 * ( (precision * recall) / (precision + recall)), 0) vars_to_summarize['metrics/f1_score'] = f1_score for var_name, var_value in vars_to_summarize.iteritems(): tf.summary.scalar(var_name, var_value) tf.add_to_collection(var_name, var_value) elif mode == 'generate': initial_state = rnn_nade.zero_state(hparams.batch_size) final_state = rnn_nade.steps(inputs, initial_state) samples, log_prob = rnn_nade.sample_single(initial_state) tf.add_to_collection('inputs', inputs) tf.add_to_collection('sample', samples) tf.add_to_collection('log_prob', log_prob) # Flatten state tuples for metagraph compatibility. for state in tf_nest.flatten(initial_state): tf.add_to_collection('initial_state', state) for state in tf_nest.flatten(final_state): tf.add_to_collection('final_state', state) return graph
def build(): """Builds the Tensorflow graph.""" inputs, lengths = None, None if mode in ('train', 'eval'): inputs, _, lengths = magenta.common.get_padded_batch( sequence_example_file_paths, hparams.batch_size, input_size, shuffle=mode == 'train') elif mode == 'generate': inputs = tf.placeholder(tf.float32, [hparams.batch_size, None, input_size]) cell = events_rnn_graph.make_rnn_cell( hparams.rnn_layer_sizes, dropout_keep_prob=hparams.dropout_keep_prob if mode == 'train' else 1.0, attn_length=hparams.attn_length, residual_connections=hparams.residual_connections) rnn_nade = RnnNade(cell, num_dims=input_size, num_hidden=hparams.nade_hidden_units) if mode in ('train', 'eval'): log_probs, cond_probs = rnn_nade.log_prob(inputs, lengths) inputs_flat = tf.to_float( magenta.common.flatten_maybe_padded_sequences(inputs, lengths)) predictions_flat = tf.to_float(tf.greater_equal(cond_probs, .5)) if mode == 'train': loss = tf.reduce_mean(-log_probs) perplexity = tf.reduce_mean(tf.exp(log_probs)) correct_predictions = tf.to_float( tf.equal(inputs_flat, predictions_flat)) accuracy = tf.reduce_mean(correct_predictions) precision = (tf.reduce_sum(inputs_flat * predictions_flat) / tf.reduce_sum(predictions_flat)) recall = (tf.reduce_sum(inputs_flat * predictions_flat) / tf.reduce_sum(inputs_flat)) optimizer = tf.train.AdamOptimizer( learning_rate=hparams.learning_rate) train_op = tf.contrib.slim.learning.create_train_op( loss, optimizer, clip_gradient_norm=hparams.clip_norm) tf.add_to_collection('train_op', train_op) vars_to_summarize = { 'loss': loss, 'metrics/perplexity': perplexity, 'metrics/accuracy': accuracy, 'metrics/precision': precision, 'metrics/recall': recall, } elif mode == 'eval': vars_to_summarize, update_ops = tf.contrib.metrics.aggregate_metric_map( { 'loss': tf.metrics.mean(-log_probs), 'metrics/perplexity': tf.metrics.mean(tf.exp(log_probs)), 'metrics/accuracy': tf.metrics.accuracy(inputs_flat, predictions_flat), 'metrics/precision': tf.metrics.precision(inputs_flat, predictions_flat), 'metrics/recall': tf.metrics.recall(inputs_flat, predictions_flat), }) for updates_op in update_ops.values(): tf.add_to_collection('eval_ops', updates_op) precision = vars_to_summarize['metrics/precision'] recall = vars_to_summarize['metrics/precision'] f1_score = tf.where( tf.greater(precision + recall, 0), 2 * ((precision * recall) / (precision + recall)), 0) vars_to_summarize['metrics/f1_score'] = f1_score for var_name, var_value in vars_to_summarize.iteritems(): tf.summary.scalar(var_name, var_value) tf.add_to_collection(var_name, var_value) elif mode == 'generate': initial_state = rnn_nade.zero_state(hparams.batch_size) final_state = rnn_nade.steps(inputs, initial_state) samples, log_prob = rnn_nade.sample_single(initial_state) tf.add_to_collection('inputs', inputs) tf.add_to_collection('sample', samples) tf.add_to_collection('log_prob', log_prob) # Flatten state tuples for metagraph compatibility. for state in tf_nest.flatten(initial_state): tf.add_to_collection('initial_state', state) for state in tf_nest.flatten(final_state): tf.add_to_collection('final_state', state)
def build_graph(mode, config, sequence_example_file_paths=None): """Builds the TensorFlow graph. Args: mode: 'train', 'eval', or 'generate'. Only mode related ops are added to the graph. config: An EventSequenceRnnConfig containing the encoder/decoder and HParams to use. sequence_example_file_paths: A list of paths to TFRecord files containing tf.train.SequenceExample protos. Only needed for training and evaluation. May be a sharded file of the form. Returns: A tf.Graph instance which contains the TF ops. Raises: ValueError: If mode is not 'train', 'eval', or 'generate'. """ if mode not in ('train', 'eval', 'generate'): raise ValueError("The mode parameter must be 'train', 'eval', " "or 'generate'. The mode parameter was: %s" % mode) hparams = config.hparams encoder_decoder = config.encoder_decoder tf.logging.info('hparams = %s', hparams.values()) input_size = encoder_decoder.input_size with tf.Graph().as_default() as graph: inputs, lengths = None, None if mode == 'train' or mode == 'eval': inputs, _, lengths = magenta.common.get_padded_batch( sequence_example_file_paths, hparams.batch_size, input_size) elif mode == 'generate': inputs = tf.placeholder(tf.float32, [hparams.batch_size, None, input_size]) cell = events_rnn_graph.make_rnn_cell( hparams.rnn_layer_sizes, dropout_keep_prob=hparams.dropout_keep_prob if mode == 'train' else 1.0, attn_length=hparams.attn_length) rnn_nade = RnnNade(cell, num_dims=input_size, num_hidden=hparams.nade_hidden_units) if mode == 'train' or mode == 'eval': log_probs, cond_probs = rnn_nade.log_prob(inputs, lengths) inputs_flat = tf.to_float( magenta.common.flatten_maybe_padded_sequences(inputs, lengths)) predictions_flat = tf.to_float(tf.greater_equal(cond_probs, .5)) if mode == 'train': loss = tf.reduce_mean(-log_probs) perplexity = tf.reduce_mean(tf.exp(log_probs)) correct_predictions = tf.to_float( tf.equal(inputs_flat, predictions_flat)) accuracy = tf.reduce_mean(correct_predictions) precision = (tf.reduce_sum(inputs_flat * predictions_flat) / tf.reduce_sum(predictions_flat)) recall = (tf.reduce_sum(inputs_flat * predictions_flat) / tf.reduce_sum(inputs_flat)) optimizer = tf.train.AdamOptimizer( learning_rate=hparams.learning_rate) train_op = tf.contrib.slim.learning.create_train_op( loss, optimizer, clip_gradient_norm=hparams.clip_norm) tf.add_to_collection('train_op', train_op) vars_to_summarize = { 'loss': loss, 'metrics/perplexity': perplexity, 'metrics/accuracy': accuracy, 'metrics/precision': precision, 'metrics/recall': recall, } elif mode == 'eval': vars_to_summarize, update_ops = tf.contrib.metrics.aggregate_metric_map( { 'loss': tf.metrics.mean(-log_probs), 'metrics/perplexity': tf.metrics.mean(tf.exp(log_probs)), 'metrics/accuracy': tf.metrics.accuracy(inputs_flat, predictions_flat), 'metrics/precision': tf.metrics.precision(inputs_flat, predictions_flat), 'metrics/recall': tf.metrics.recall(inputs_flat, predictions_flat), }) for updates_op in update_ops.values(): tf.add_to_collection('eval_ops', updates_op) precision = vars_to_summarize['metrics/precision'] recall = vars_to_summarize['metrics/precision'] f1_score = tf.where( tf.greater(precision + recall, 0), 2 * ((precision * recall) / (precision + recall)), 0) vars_to_summarize['metrics/f1_score'] = f1_score for var_name, var_value in vars_to_summarize.iteritems(): tf.summary.scalar(var_name, var_value) tf.add_to_collection(var_name, var_value) elif mode == 'generate': initial_state = rnn_nade.zero_state(hparams.batch_size) final_state = rnn_nade.steps(inputs, initial_state) samples, log_prob = rnn_nade.sample_single(initial_state) tf.add_to_collection('inputs', inputs) tf.add_to_collection('sample', samples) tf.add_to_collection('log_prob', log_prob) # Flatten state tuples for metagraph compatibility. for state in tf_nest.flatten(initial_state): tf.add_to_collection('initial_state', state) for state in tf_nest.flatten(final_state): tf.add_to_collection('final_state', state) return graph
def build(): """Builds the Tensorflow graph.""" inputs, lengths = None, None if mode == 'train' or mode == 'eval': inputs, _, lengths = magenta.common.get_padded_batch( sequence_example_file_paths, hparams.batch_size, input_size, shuffle=mode == 'train') elif mode == 'generate': inputs = tf.placeholder(tf.float32, [hparams.batch_size, None, input_size]) cell = events_rnn_graph.make_rnn_cell( hparams.rnn_layer_sizes, dropout_keep_prob=hparams.dropout_keep_prob if mode == 'train' else 1.0, attn_length=( hparams.attn_length if hasattr(hparams, 'attn_length') else 0)) rnn_nade = RnnNade( cell, num_dims=input_size, num_hidden=hparams.nade_hidden_units) if mode == 'train' or mode == 'eval': log_probs, cond_probs = rnn_nade.log_prob(inputs, lengths) inputs_flat = tf.to_float( magenta.common.flatten_maybe_padded_sequences(inputs, lengths)) predictions_flat = tf.to_float(tf.greater_equal(cond_probs, .5)) if mode == 'train': loss = tf.reduce_mean(-log_probs) perplexity = tf.reduce_mean(tf.exp(log_probs)) correct_predictions = tf.to_float( tf.equal(inputs_flat, predictions_flat)) accuracy = tf.reduce_mean(correct_predictions) precision = (tf.reduce_sum(inputs_flat * predictions_flat) / tf.reduce_sum(predictions_flat)) recall = (tf.reduce_sum(inputs_flat * predictions_flat) / tf.reduce_sum(inputs_flat)) optimizer = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate) train_op = tf.contrib.slim.learning.create_train_op( loss, optimizer, clip_gradient_norm=hparams.clip_norm) tf.add_to_collection('train_op', train_op) vars_to_summarize = { 'loss': loss, 'metrics/perplexity': perplexity, 'metrics/accuracy': accuracy, 'metrics/precision': precision, 'metrics/recall': recall, } elif mode == 'eval': vars_to_summarize, update_ops = tf.contrib.metrics.aggregate_metric_map( { 'loss': tf.metrics.mean(-log_probs), 'metrics/perplexity': tf.metrics.mean(tf.exp(log_probs)), 'metrics/accuracy': tf.metrics.accuracy( inputs_flat, predictions_flat), 'metrics/precision': tf.metrics.precision( inputs_flat, predictions_flat), 'metrics/recall': tf.metrics.recall( inputs_flat, predictions_flat), }) for updates_op in update_ops.values(): tf.add_to_collection('eval_ops', updates_op) precision = vars_to_summarize['metrics/precision'] recall = vars_to_summarize['metrics/precision'] f1_score = tf.where( tf.greater(precision + recall, 0), 2 * ( (precision * recall) / (precision + recall)), 0) vars_to_summarize['metrics/f1_score'] = f1_score for var_name, var_value in vars_to_summarize.iteritems(): tf.summary.scalar(var_name, var_value) tf.add_to_collection(var_name, var_value) elif mode == 'generate': initial_state = rnn_nade.zero_state(hparams.batch_size) final_state = rnn_nade.steps(inputs, initial_state) samples, log_prob = rnn_nade.sample_single(initial_state) tf.add_to_collection('inputs', inputs) tf.add_to_collection('sample', samples) tf.add_to_collection('log_prob', log_prob) # Flatten state tuples for metagraph compatibility. for state in tf_nest.flatten(initial_state): tf.add_to_collection('initial_state', state) for state in tf_nest.flatten(final_state): tf.add_to_collection('final_state', state)
def build_graph(mode, config, sequence_example_file_paths=None): """Builds the TensorFlow graph. Args: mode: 'train', 'eval', or 'generate'. Only mode related ops are added to the graph. config: An EventSequenceRnnConfig containing the encoder/decoder and HParams to use. sequence_example_file_paths: A list of paths to TFRecord files containing tf.train.SequenceExample protos. Only needed for training and evaluation. May be a sharded file of the form. Returns: A tf.Graph instance which contains the TF ops. Raises: ValueError: If mode is not 'train', 'eval', or 'generate'. """ if mode not in ('train', 'eval', 'generate'): raise ValueError("The mode parameter must be 'train', 'eval', " "or 'generate'. The mode parameter was: %s" % mode) hparams = config.hparams encoder_decoder = config.encoder_decoder tf.logging.info('hparams = %s', hparams.values()) input_size = encoder_decoder.input_size with tf.Graph().as_default() as graph: inputs, lengths = None, None if mode == 'train' or mode == 'eval': inputs, _, lengths = magenta.common.get_padded_batch( sequence_example_file_paths, hparams.batch_size, input_size) elif mode == 'generate': inputs = tf.placeholder(tf.float32, [hparams.batch_size, None, input_size]) cell = events_rnn_graph.make_rnn_cell( hparams.rnn_layer_sizes, dropout_keep_prob=hparams.dropout_keep_prob if mode == 'train' else 1.0, attn_length=hparams.attn_length) rnn_nade = RnnNade( cell, num_dims=input_size, num_hidden=hparams.nade_hidden_units) if mode == 'train' or mode == 'eval': log_probs, cond_probs = rnn_nade.log_prob(inputs, lengths) inputs_flat = flatten_maybe_padded_sequences(inputs, lengths) predictions_flat = tf.cast(tf.greater_equal(cond_probs, .5), tf.float32) loss = tf.reduce_mean(-log_probs) perplexity = tf.reduce_mean(tf.exp(log_probs)) * 100 accuracy = tf.reduce_mean( tf.to_float(tf.equal(inputs_flat, predictions_flat))) * 100 accuracy_without_true_negatives = ( tf.reduce_sum(inputs_flat * tf.to_float(tf.equal(predictions_flat, inputs_flat))) / tf.reduce_sum(inputs_flat) * 100) global_step = tf.contrib.framework.get_or_create_global_step() tf.add_to_collection('loss', loss) tf.add_to_collection('perplexity', perplexity) tf.add_to_collection('accuracy', accuracy) tf.add_to_collection('accuracy_without_true_negatives', accuracy_without_true_negatives) summaries = [ tf.summary.scalar('loss', loss), tf.summary.scalar('perplexity', perplexity), tf.summary.scalar('accuracy', accuracy), tf.summary.scalar('accuracy_without_true_negatives', accuracy_without_true_negatives), ] if mode == 'train': learning_rate = tf.train.exponential_decay( hparams.initial_learning_rate, global_step, hparams.decay_steps, hparams.decay_rate, staircase=True, name='learning_rate') opt = tf.train.AdamOptimizer(learning_rate) params = tf.trainable_variables() gradients = tf.gradients(loss, params) clipped_gradients, _ = tf.clip_by_global_norm(gradients, hparams.clip_norm) train_op = opt.apply_gradients(zip(clipped_gradients, params), global_step) tf.add_to_collection('learning_rate', learning_rate) tf.add_to_collection('train_op', train_op) summaries.append(tf.summary.scalar( 'learning_rate', learning_rate)) if mode == 'eval': summary_op = tf.summary.merge(summaries) tf.add_to_collection('summary_op', summary_op) elif mode == 'generate': initial_state = rnn_nade.zero_state(hparams.batch_size) final_state = rnn_nade.steps(inputs, initial_state) samples, log_prob = rnn_nade.sample_single(initial_state) tf.add_to_collection('inputs', inputs) tf.add_to_collection('sample', samples) tf.add_to_collection('log_prob', log_prob) # Flatten state tuples for metagraph compatibility. for state in tf_nest.flatten(initial_state): tf.add_to_collection('initial_state', state) for state in tf_nest.flatten(final_state): tf.add_to_collection('final_state', state) return graph
def build_graph(mode, config, sequence_example_file_paths=None): """Builds the TensorFlow graph. Args: mode: 'train', 'eval', or 'generate'. Only mode related ops are added to the graph. config: An EventSequenceRnnConfig containing the encoder/decoder and HParams to use. sequence_example_file_paths: A list of paths to TFRecord files containing tf.train.SequenceExample protos. Only needed for training and evaluation. May be a sharded file of the form. Returns: A tf.Graph instance which contains the TF ops. Raises: ValueError: If mode is not 'train', 'eval', or 'generate'. """ if mode not in ('train', 'eval', 'generate'): raise ValueError("The mode parameter must be 'train', 'eval', " "or 'generate'. The mode parameter was: %s" % mode) hparams = config.hparams encoder_decoder = config.encoder_decoder tf.logging.info('hparams = %s', hparams.values()) input_size = encoder_decoder.input_size with tf.Graph().as_default() as graph: inputs, lengths = None, None if mode == 'train' or mode == 'eval': inputs, _, lengths = magenta.common.get_padded_batch( sequence_example_file_paths, hparams.batch_size, input_size) elif mode == 'generate': inputs = tf.placeholder(tf.float32, [hparams.batch_size, None, input_size]) cell = events_rnn_graph.make_rnn_cell( hparams.rnn_layer_sizes, dropout_keep_prob=hparams.dropout_keep_prob if mode == 'train' else 1.0, attn_length=hparams.attn_length) rnn_nade = RnnNade(cell, num_dims=input_size, num_hidden=hparams.nade_hidden_units) if mode == 'train' or mode == 'eval': log_probs, cond_probs = rnn_nade.log_prob(inputs, lengths) inputs_flat = flatten_maybe_padded_sequences(inputs, lengths) predictions_flat = tf.cast(tf.greater_equal(cond_probs, .5), tf.float32) loss = tf.reduce_mean(-log_probs) perplexity = tf.reduce_mean(tf.exp(log_probs)) * 100 accuracy = tf.reduce_mean( tf.to_float(tf.equal(inputs_flat, predictions_flat))) * 100 accuracy_without_true_negatives = (tf.reduce_sum( inputs_flat * tf.to_float(tf.equal(predictions_flat, inputs_flat))) / tf.reduce_sum(inputs_flat) * 100) global_step = tf.contrib.framework.get_or_create_global_step() tf.add_to_collection('loss', loss) tf.add_to_collection('perplexity', perplexity) tf.add_to_collection('accuracy', accuracy) tf.add_to_collection('accuracy_without_true_negatives', accuracy_without_true_negatives) summaries = [ tf.summary.scalar('loss', loss), tf.summary.scalar('perplexity', perplexity), tf.summary.scalar('accuracy', accuracy), tf.summary.scalar('accuracy_without_true_negatives', accuracy_without_true_negatives), ] if mode == 'train': learning_rate = tf.train.exponential_decay( hparams.initial_learning_rate, global_step, hparams.decay_steps, hparams.decay_rate, staircase=True, name='learning_rate') opt = tf.train.AdamOptimizer(learning_rate) params = tf.trainable_variables() gradients = tf.gradients(loss, params) clipped_gradients, _ = tf.clip_by_global_norm( gradients, hparams.clip_norm) train_op = opt.apply_gradients(zip(clipped_gradients, params), global_step) tf.add_to_collection('learning_rate', learning_rate) tf.add_to_collection('train_op', train_op) summaries.append( tf.summary.scalar('learning_rate', learning_rate)) if mode == 'eval': summary_op = tf.summary.merge(summaries) tf.add_to_collection('summary_op', summary_op) elif mode == 'generate': initial_state = rnn_nade.zero_state(hparams.batch_size) final_state = rnn_nade.steps(inputs, initial_state) samples, log_prob = rnn_nade.sample_single(initial_state) tf.add_to_collection('inputs', inputs) tf.add_to_collection('sample', samples) tf.add_to_collection('log_prob', log_prob) # Flatten state tuples for metagraph compatibility. for state in tf_nest.flatten(initial_state): tf.add_to_collection('initial_state', state) for state in tf_nest.flatten(final_state): tf.add_to_collection('final_state', state) return graph
def build_graph(self): """Constructs the portion of the graph that belongs to this model.""" tf.logging.info('Initializing melody RNN graph for scope %s', self.scope) with self.graph.as_default(): with tf.device(lambda op: ''): with tf.variable_scope(self.scope): # Make an LSTM cell with the number and size of layers specified in # hparams. if self.note_rnn_type == 'basic_rnn': self.cell = events_rnn_graph.make_rnn_cell( self.hparams.rnn_layer_sizes) else: self.cell = rl_tuner_ops.make_rnn_cell(self.hparams.rnn_layer_sizes) # Shape of melody_sequence is batch size, melody length, number of # output note actions. self.melody_sequence = tf.placeholder(tf.float32, [None, None, self.hparams.one_hot_length], name='melody_sequence') self.lengths = tf.placeholder(tf.int32, [None], name='lengths') self.initial_state = tf.placeholder(tf.float32, [None, self.cell.state_size], name='initial_state') if self.training_file_list is not None: # Set up a tf queue to read melodies from the training data tfrecord (self.train_sequence, self.train_labels, self.train_lengths) = sequence_example_lib.get_padded_batch( self.training_file_list, self.hparams.batch_size, self.hparams.one_hot_length) # Closure function is used so that this part of the graph can be # re-run in multiple places, such as __call__. def run_network_on_melody(m_seq, lens, initial_state, swap_memory=True, parallel_iterations=1): """Internal function that defines the RNN network structure. Args: m_seq: A batch of melody sequences of one-hot notes. lens: Lengths of the melody_sequences. initial_state: Vector representing the initial state of the RNN. swap_memory: Uses more memory and is faster. parallel_iterations: Argument to tf.nn.dynamic_rnn. Returns: Output of network (either softmax or logits) and RNN state. """ outputs, final_state = tf.nn.dynamic_rnn( self.cell, m_seq, sequence_length=lens, initial_state=initial_state, swap_memory=swap_memory, parallel_iterations=parallel_iterations) outputs_flat = tf.reshape(outputs, [-1, self.hparams.rnn_layer_sizes[-1]]) if self.note_rnn_type == 'basic_rnn': linear_layer = tf.contrib.layers.linear else: linear_layer = tf.contrib.layers.legacy_linear logits_flat = linear_layer( outputs_flat, self.hparams.one_hot_length) return logits_flat, final_state (self.logits, self.state_tensor) = run_network_on_melody( self.melody_sequence, self.lengths, self.initial_state) self.softmax = tf.nn.softmax(self.logits) self.run_network_on_melody = run_network_on_melody if self.training_file_list is not None: # Does not recreate the model architecture but rather uses it to feed # data from the training queue through the model. with tf.variable_scope(self.scope, reuse=True): zero_state = self.cell.zero_state( batch_size=self.hparams.batch_size, dtype=tf.float32) (self.train_logits, self.train_state) = run_network_on_melody( self.train_sequence, self.train_lengths, zero_state) self.train_softmax = tf.nn.softmax(self.train_logits)