def testMultiEvalStepIncrements(self):
checkpoint_dir = os.path.join(self.get_temp_dir(), 'eval_ops_and_final_ops')
# Train a model for a single step to get a checkpoint.
self._train_model(checkpoint_dir, num_steps=1)
checkpoint_path = saver.latest_checkpoint(checkpoint_dir)
# Create the model so we have something to restore.
inputs = constant_op.constant(self._inputs, dtype=dtypes.float32)
logistic_classifier(inputs)
num_evals = 6
my_var = local_variable(0.0, name='MyVar')
# In eval ops, we also increase the eval step one more time.
eval_ops = [state_ops.assign_add(my_var, 1.0),
state_ops.assign_add(
evaluation._get_or_create_eval_step(), 1, use_locking=True)]
expect_eval_update_counts = num_evals // 2
final_ops = array_ops.identity(my_var)
final_ops_values = evaluation._evaluate_once(
checkpoint_path=checkpoint_path,
eval_ops=eval_ops,
final_ops={'value': final_ops},
hooks=[evaluation._StopAfterNEvalsHook(num_evals),])
self.assertEqual(final_ops_values['value'], expect_eval_update_counts)
def testMultiEvalStepIncrements(self):
checkpoint_dir = os.path.join(self.get_temp_dir(),
'eval_ops_and_final_ops')
# Train a model for a single step to get a checkpoint.
self._train_model(checkpoint_dir, num_steps=1)
checkpoint_path = saver.latest_checkpoint(checkpoint_dir)
# Create the model so we have something to restore.
inputs = constant_op.constant(self._inputs, dtype=dtypes.float32)
logistic_classifier(inputs)
num_evals = 6
my_var = local_variable(0.0, name='MyVar')
# In eval ops, we also increase the eval step one more time.
eval_ops = [
state_ops.assign_add(my_var, 1.0),
state_ops.assign_add(evaluation._get_or_create_eval_step(),
1,
use_locking=True)
]
expect_eval_update_counts = num_evals // 2
final_ops = array_ops.identity(my_var)
final_ops_values = evaluation._evaluate_once(
checkpoint_path=checkpoint_path,
eval_ops=eval_ops,
final_ops={'value': final_ops},
hooks=[
evaluation._StopAfterNEvalsHook(num_evals),
])
self.assertEqual(final_ops_values['value'], expect_eval_update_counts)
def testEvaluateWithFiniteInputs(self):
checkpoint_dir = os.path.join(self.get_temp_dir(),
'evaluate_with_finite_inputs')
# Train a Model to completion:
self._train_model(checkpoint_dir, num_steps=300)
# Run evaluation. Inputs are fed through input producer for one epoch.
all_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32)
all_labels = constant_op.constant(self._labels, dtype=dtypes.float32)
single_input, single_label = training.slice_input_producer(
[all_inputs, all_labels], num_epochs=1)
inputs, labels = training.batch([single_input, single_label], batch_size=6,
allow_smaller_final_batch=True)
logits = logistic_classifier(inputs)
predictions = math_ops.round(logits)
accuracy, update_op = metrics.accuracy(
predictions=predictions, labels=labels)
checkpoint_path = saver.latest_checkpoint(checkpoint_dir)
final_ops_values = evaluation._evaluate_once(
checkpoint_path=checkpoint_path,
eval_ops=update_op,
final_ops={'accuracy': accuracy,
'eval_steps': evaluation._get_or_create_eval_step()},
hooks=[evaluation._StopAfterNEvalsHook(None),])
self.assertTrue(final_ops_values['accuracy'] > .99)
# Runs evaluation for 4 iterations. First 2 evaluate full batch of 6 inputs
# each; the 3rd iter evaluates the remaining 4 inputs, and the last one
# triggers an error which stops evaluation.
self.assertEqual(final_ops_values['eval_steps'], 4)
def after_run(self, run_context, run_values):
# To avoid race condition between the eval step read and increment in
# evaluation graph, we read the value explicitly here.
eval_steps = run_context.session.run(
evaluation._get_or_create_eval_step())
self._test_case.assertEqual(expected_eval_steps, eval_steps)
self._test_case.assertFalse(self._invoked)
self._invoked = True
def begin(self):
if self._summary_writer is None and self._output_dir:
self._summary_writer = SummaryWriterCache.get(self._output_dir)
self._next_step = None
self._global_step_tensor = evaluation._get_or_create_eval_step() # pylint: disable=protected-access
if self._global_step_tensor is None:
raise RuntimeError(
"Global step should be created to use SummarySaverHook.")
def testEvaluateWithFiniteInputs(self):
checkpoint_dir = os.path.join(self.get_temp_dir(),
'evaluate_with_finite_inputs')
# Train a Model to completion:
self._train_model(checkpoint_dir, num_steps=300)
# Run evaluation. Inputs are fed through input producer for one epoch.
all_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32)
all_labels = constant_op.constant(self._labels, dtype=dtypes.float32)
single_input, single_label = training.slice_input_producer(
[all_inputs, all_labels], num_epochs=1)
inputs, labels = training.batch([single_input, single_label],
batch_size=6,
allow_smaller_final_batch=True)
logits = logistic_classifier(inputs)
predictions = math_ops.round(logits)
accuracy, update_op = metrics.accuracy(predictions=predictions,
labels=labels)
checkpoint_path = saver.latest_checkpoint(checkpoint_dir)
final_ops_values = evaluation._evaluate_once(
checkpoint_path=checkpoint_path,
eval_ops=update_op,
final_ops={
'accuracy': accuracy,
'eval_steps': evaluation._get_or_create_eval_step()
},
hooks=[
evaluation._StopAfterNEvalsHook(None),
])
self.assertTrue(final_ops_values['accuracy'] > .99)
# Runs evaluation for 4 iterations. First 2 evaluate full batch of 6 inputs
# each; the 3rd iter evaluates the remaining 4 inputs, and the last one
# triggers an error which stops evaluation.
self.assertEqual(final_ops_values['eval_steps'], 4)
def before_run(self, run_context):
return session_run_hook.SessionRunArgs({
'eval_steps': evaluation._get_or_create_eval_step()
})
def build_graph():
encoder_decoder = config.encoder_decoder
input_size = encoder_decoder.input_size
num_classes = encoder_decoder.num_classes
default_event_label = encoder_decoder.default_event_label
batch_size = config.batch_size
label_shape = []
learning_rate = config.learning_rate
inputs, labels, lengths, composers = None, None, None, None
if mode == 'train' or mode == 'eval':
if config.label_classifier_units:
inputs, labels, lengths, composers = mg.common.get_padded_batch_metadata(
examples_path,
batch_size,
input_size,
label_shape=label_shape,
shuffle=mode == 'train',
composer_shape=config.label_classifier_units,
num_enqueuing_threads=config.threads)
inputs = tf.debugging.check_numerics(
inputs, "Inputs invalid")
labels = tf.cast(
tf.debugging.check_numerics(
tf.cast(labels, tf.float32), "Labels invalid"),
tf.int64)
lengths = tf.cast(
tf.debugging.check_numerics(
tf.cast(lengths, tf.float32), "Lengths invalid"),
tf.int32)
composers = tf.cast(
tf.debugging.check_numerics(
tf.cast(composers, tf.float32),
"Composers invalid"), tf.int64)
else:
inputs, labels, lengths = mg.common.get_padded_batch(
examples_path,
batch_size,
input_size,
label_shape=label_shape,
shuffle=mode == 'train',
num_enqueuing_threads=config.threads)
# assert not tf.debugging.is_nan(inputs)
# assert not tf.debugging.is_nan(labels)
# assert not tf.debugging.is_nan(lengths)
else:
inputs = tf.placeholder(tf.float32,
[batch_size, None, input_size])
config.dropout = 1.0 if mode == 'generate' else config.dropout
outputs, initial_state, final_state = None, None, None
if config.gpu:
tf.logging.info("Using CudNN")
outputs, initial_state, final_state = get_cudnn(
inputs, config.rnn_layers, config.dropout, batch_size,
mode)
else:
cell = get_deep_lstm(config.rnn_layers, config.dropout)
initial_state = cell.zero_state(batch_size, tf.float32)
outputs, final_state = tf.nn.dynamic_rnn(
cell,
inputs,
sequence_length=lengths,
initial_state=initial_state,
swap_memory=True)
outputs = tf.debugging.check_numerics(outputs, "outputs invalid")
outputs_flat = mg.common.flatten_maybe_padded_sequences(
outputs, lengths)
outputs_flat = tf.debugging.check_numerics(outputs_flat,
"outputs_flat invalid")
num_logits = num_classes
logits_flat = tf.contrib.layers.linear(outputs_flat, num_logits)
if config.label_classifier_weight:
composer_logits = tf.layers.dense(
final_state[-1].h, config.label_classifier_units)
composer_predictions = tf.argmax(composer_logits, axis=1)
if mode in ('train', 'eval'):
labels_flat = mg.common.flatten_maybe_padded_sequences(
labels, lengths)
labels_flat = tf.cast(
tf.debugging.check_numerics(
tf.cast(labels_flat, tf.float32),
"labels_flat invalid"), tf.int64)
logits_flat = tf.debugging.check_numerics(
logits_flat, "logits_flat invalid")
softmax_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels_flat, logits=logits_flat)
softmax_cross_entropy = tf.debugging.check_numerics(
softmax_cross_entropy, "softmax_cross_entropy invalid")
if config.label_classifier_weight:
composer_softmax_cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=composers, logits=composer_logits)
predictions_flat = tf.argmax(logits_flat, axis=1)
correct_predictions = tf.to_float(
tf.equal(labels_flat, predictions_flat))
loss = None
# Predict our composer to enforce structure on embeddings
if mode == 'train':
global_step = tf.Variable(-1, trainable=False)
if config.label_classifier_weight:
tf.add_to_collection('composer_logits',
tf.nn.softmax(composer_logits))
composer_loss = tf.reduce_mean(
composer_softmax_cross_entropy)
lstm_loss = tf.reduce_mean(softmax_cross_entropy)
tf.add_to_collection('composer_loss', composer_loss)
tf.add_to_collection('lstm_loss', lstm_loss)
tf.summary.scalar('composer_loss', composer_loss)
tf.summary.scalar('lstm_loss', lstm_loss)
decay_steps = config.decay_steps
classifier_weight = tf.Variable(
config.label_classifier_weight, trainable=False)
classifier_weight = classifier_weight - tf.train.polynomial_decay(
config.label_classifier_weight,
global_step,
decay_steps,
0.0,
power=0.2)
composer_loss = classifier_weight * composer_loss
lstm_loss = (1 - classifier_weight) * lstm_loss
tf.add_to_collection('composer_weighting',
classifier_weight)
tf.summary.scalar('composer_weight', classifier_weight)
# composer_loss = tf.maximum(tf.Variable(1e-07), composer_loss)
composer_loss = tf.debugging.check_numerics(
composer_loss, "composer_loss invalid")
lstm_loss = tf.debugging.check_numerics(
lstm_loss, "lstm_loss invalid")
loss = tf.add(lstm_loss, composer_loss)
loss = tf.debugging.check_numerics(
loss, "loss invalid")
tf.add_to_collection('loss', loss)
tf.summary.scalar('loss', loss)
else:
tf.logging.info("Building normal graph.")
loss = tf.reduce_mean(softmax_cross_entropy)
tf.add_to_collection('loss', loss)
tf.summary.scalar('loss', loss)
perplexity = tf.exp(loss)
accuracy = tf.reduce_mean(correct_predictions)
tf.add_to_collection('perplexity', perplexity)
tf.add_to_collection('accuracy', accuracy)
tf.summary.scalar('perplexity', perplexity)
tf.summary.scalar('accuracy', accuracy)
optimizer = config.optimizer(learning_rate=learning_rate,
momentum=config.momentum)
train_op = tf.contrib.slim.learning.create_train_op(
loss,
optimizer,
global_step=global_step,
clip_gradient_norm=config.norm)
tf.add_to_collection('global_step', global_step)
tf.add_to_collection('train_op', train_op)
tf.add_to_collection('optimizer', optimizer)
elif mode == 'eval':
global_step = evaluation._get_or_create_eval_step()
metric_map = {
'loss':
tf.metrics.mean(softmax_cross_entropy),
'metrics/accuracy':
tf.metrics.accuracy(labels_flat, predictions_flat),
'metrics/per_class_accuracy':
tf.metrics.mean_per_class_accuracy(
labels_flat, predictions_flat, num_classes)
}
if config.label_classifier_weight:
metric_map['composer_loss'] = tf.metrics.mean(
composer_softmax_cross_entropy)
metric_map[
'metrics/composer_accuracy'] = tf.metrics.accuracy(
composers, composer_logits)
metric_map[
'metrics/composer_per_class_accuracy'] = tf.metrics.mean_per_class_accuracy(
composers, composer_logits,
config.label_classifier_units)
vars_to_summarize, update_ops = tf.contrib.metrics.aggregate_metric_map(
metric_map)
for updates_op in update_ops.values():
tf.add_to_collection('eval_ops', updates_op)
vars_to_summarize['metrics/perplexity'] = tf.exp(
vars_to_summarize['loss'])
if config.label_classifier_weight:
vars_to_summarize[
'metrics/composer_perplexity'] = tf.exp(
vars_to_summarize['composer_loss'])
for var_name, var_value in six.iteritems(
vars_to_summarize):
tf.summary.scalar(var_name, var_value)
tf.add_to_collection(var_name, var_value)
tf.add_to_collection('global_step', global_step)
elif mode == 'generate':
if config.label_classifier_weight:
composer_softmax = tf.nn.softmax(composer_logits)
tf.add_to_collection("composer_softmax", composer_softmax)
temperature = tf.placeholder(tf.float32, [])
softmax_flat = tf.nn.softmax(
tf.div(logits_flat, tf.fill([num_classes], temperature)))
softmax = tf.reshape(softmax_flat,
[batch_size, -1, num_classes])
# if config.label_classifier_weight:
tf.add_to_collection('inputs', inputs)
tf.add_to_collection('temperature', temperature)
tf.add_to_collection('softmax', softmax)
for state in tf_nest.flatten(initial_state):
tf.add_to_collection('initial_state', state)
for state in tf_nest.flatten(final_state):
tf.add_to_collection('final_state', state)
def before_run(self, run_context):
return tf.compat.v1.train.SessionRunArgs(
{'eval_steps': evaluation._get_or_create_eval_step()})
