def estimator_fn():
return dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=self.NUM_RNN_CELL_UNITS,
sequence_feature_columns=self.sequence_feature_columns,
context_feature_columns=self.context_feature_columns,
predict_probabilities=True,
model_dir=model_dir)
def estimator_fn():
return dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=self.NUM_RNN_CELL_UNITS,
sequence_feature_columns=self.sequence_feature_columns,
context_feature_columns=self.context_feature_columns,
predict_probabilities=True,
model_dir=model_dir)
def testLearnShiftByOne(self):
"""Tests that `_MultiValueRNNEstimator` can learn a 'shift-by-one' example.
Each label sequence consists of the input sequence 'shifted' by one place.
The RNN must learn to 'remember' the previous input.
"""
batch_size = 16
sequence_length = 32
train_steps = 200
eval_steps = 20
cell_size = 4
learning_rate = 0.3
accuracy_threshold = 0.9
def get_shift_input_fn(batch_size, sequence_length, seed=None):
def input_fn():
random_sequence = tf.random_uniform(
[batch_size, sequence_length + 1],
0,
2,
dtype=tf.int32,
seed=seed)
labels = tf.slice(random_sequence, [0, 0],
[batch_size, sequence_length])
inputs = tf.expand_dims(
tf.to_float(
tf.slice(random_sequence, [0, 1],
[batch_size, sequence_length])), 2)
return {'inputs': inputs}, labels
return input_fn
seq_columns = [
tf.contrib.layers.real_valued_column('inputs', dimension=cell_size)
]
config = tf.contrib.learn.RunConfig(tf_random_seed=21212)
sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=cell_size,
sequence_feature_columns=seq_columns,
learning_rate=learning_rate,
config=config)
train_input_fn = get_shift_input_fn(batch_size,
sequence_length,
seed=12321)
eval_input_fn = get_shift_input_fn(batch_size,
sequence_length,
seed=32123)
sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
evaluation = sequence_estimator.evaluate(input_fn=eval_input_fn,
steps=eval_steps)
accuracy = evaluation['accuracy']
self.assertGreater(
accuracy, accuracy_threshold,
'Accuracy should be higher than {}; got {}'.format(
accuracy_threshold, accuracy))
def testMultiRNNState(self):
"""Test that state flattening/reconstruction works for `MultiRNNCell`."""
batch_size = 11
sequence_length = 16
train_steps = 5
cell_sizes = [4, 8, 7]
learning_rate = 0.1
def get_shift_input_fn(batch_size, sequence_length, seed=None):
def input_fn():
random_sequence = random_ops.random_uniform(
[batch_size, sequence_length + 1],
0,
2,
dtype=dtypes.int32,
seed=seed)
labels = array_ops.slice(random_sequence, [0, 0],
[batch_size, sequence_length])
inputs = array_ops.expand_dims(
math_ops.to_float(
array_ops.slice(random_sequence, [0, 1],
[batch_size, sequence_length])), 2)
input_dict = {
dynamic_rnn_estimator._get_state_name(i): random_ops.random_uniform(
[batch_size, cell_size], seed=((i + 1) * seed))
for i, cell_size in enumerate([4, 4, 8, 8, 7, 7])
}
input_dict['inputs'] = inputs
return input_dict, labels
return input_fn
seq_columns = [feature_column.real_valued_column('inputs', dimension=1)]
config = run_config.RunConfig(tf_random_seed=21212)
cell = core_rnn_cell_impl.MultiRNNCell(
[core_rnn_cell_impl.BasicLSTMCell(size) for size in cell_sizes])
sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=None,
sequence_feature_columns=seq_columns,
cell_type=cell,
learning_rate=learning_rate,
config=config,
predict_probabilities=True)
train_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=12321)
eval_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=32123)
sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
prediction_dict = sequence_estimator.predict(
input_fn=eval_input_fn, as_iterable=False)
for i, state_size in enumerate([4, 4, 8, 8, 7, 7]):
state_piece = prediction_dict[dynamic_rnn_estimator._get_state_name(i)]
self.assertListEqual(list(state_piece.shape), [batch_size, state_size])
def testMultiRNNState(self):
"""Test that state flattening/reconstruction works for `MultiRNNCell`."""
batch_size = 11
sequence_length = 16
train_steps = 5
cell_sizes = [4, 8, 7]
learning_rate = 0.1
def get_shift_input_fn(batch_size, sequence_length, seed=None):
def input_fn():
random_sequence = random_ops.random_uniform(
[batch_size, sequence_length + 1],
0,
2,
dtype=dtypes.int32,
seed=seed)
labels = array_ops.slice(random_sequence, [0, 0],
[batch_size, sequence_length])
inputs = array_ops.expand_dims(
math_ops.to_float(
array_ops.slice(random_sequence, [0, 1],
[batch_size, sequence_length])), 2)
input_dict = {
dynamic_rnn_estimator._get_state_name(i): random_ops.random_uniform(
[batch_size, cell_size], seed=((i + 1) * seed))
for i, cell_size in enumerate([4, 4, 8, 8, 7, 7])
}
input_dict['inputs'] = inputs
return input_dict, labels
return input_fn
seq_columns = [feature_column.real_valued_column('inputs', dimension=1)]
config = run_config.RunConfig(tf_random_seed=21212)
cell = core_rnn_cell_impl.MultiRNNCell(
[core_rnn_cell_impl.BasicLSTMCell(size) for size in cell_sizes])
sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=None,
sequence_feature_columns=seq_columns,
cell_type=cell,
learning_rate=learning_rate,
config=config,
predict_probabilities=True)
train_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=12321)
eval_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=32123)
sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
prediction_dict = sequence_estimator.predict(
input_fn=eval_input_fn, as_iterable=False)
for i, state_size in enumerate([4, 4, 8, 8, 7, 7]):
state_piece = prediction_dict[dynamic_rnn_estimator._get_state_name(i)]
self.assertListEqual(list(state_piece.shape), [batch_size, state_size])
def testLegacyConstructor(self):
"""Exercise legacy constructor function."""
num_units = 16
num_layers = 6
output_keep_prob = 0.9
input_keep_prob = 0.7
batch_size = 11
learning_rate = 0.1
train_sequence_length = 21
train_steps = 121
def get_input_fn(batch_size,
sequence_length,
state_dict,
starting_step=0):
def input_fn():
sequence = constant_op.constant(
[[(starting_step + i + j) % 2
for j in range(sequence_length + 1)]
for i in range(batch_size)],
dtype=dtypes.int32)
labels = array_ops.slice(sequence, [0, 0],
[batch_size, sequence_length])
inputs = array_ops.expand_dims(
math_ops.to_float(
array_ops.slice(sequence, [0, 1],
[batch_size, sequence_length])), 2)
input_dict = state_dict
input_dict['inputs'] = inputs
return input_dict, labels
return input_fn
seq_columns = [
feature_column.real_valued_column('inputs', dimension=1)
]
config = run_config.RunConfig(tf_random_seed=21212)
model_dir = tempfile.mkdtemp()
sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=num_units,
num_rnn_layers=num_layers,
input_keep_probability=input_keep_prob,
output_keep_probability=output_keep_prob,
sequence_feature_columns=seq_columns,
learning_rate=learning_rate,
config=config,
model_dir=model_dir)
train_input_fn = get_input_fn(batch_size,
train_sequence_length,
state_dict={})
sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
def testLegacyConstructor(self):
"""Exercise legacy constructor function."""
num_units = 16
num_layers = 6
output_keep_prob = 0.9
input_keep_prob = 0.7
batch_size = 11
learning_rate = 0.1
train_sequence_length = 21
train_steps = 121
def get_input_fn(batch_size, sequence_length, state_dict, starting_step=0):
def input_fn():
sequence = constant_op.constant(
[[(starting_step + i + j) % 2 for j in range(sequence_length + 1)]
for i in range(batch_size)],
dtype=dtypes.int32)
labels = array_ops.slice(sequence, [0, 0],
[batch_size, sequence_length])
inputs = array_ops.expand_dims(
math_ops.to_float(
array_ops.slice(sequence, [0, 1], [batch_size, sequence_length
])), 2)
input_dict = state_dict
input_dict['inputs'] = inputs
return input_dict, labels
return input_fn
seq_columns = [feature_column.real_valued_column('inputs', dimension=1)]
config = run_config.RunConfig(tf_random_seed=21212)
model_dir = tempfile.mkdtemp()
sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=num_units,
num_rnn_layers=num_layers,
input_keep_probability=input_keep_prob,
output_keep_probability=output_keep_prob,
sequence_feature_columns=seq_columns,
learning_rate=learning_rate,
config=config,
model_dir=model_dir)
train_input_fn = get_input_fn(
batch_size, train_sequence_length, state_dict={})
sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
def testLearnShiftByOne(self):
"""Tests that `_MultiValueRNNEstimator` can learn a 'shift-by-one' example.
Each label sequence consists of the input sequence 'shifted' by one place.
The RNN must learn to 'remember' the previous input.
"""
batch_size = 16
sequence_length = 32
train_steps = 200
eval_steps = 20
cell_size = 4
learning_rate = 0.3
accuracy_threshold = 0.9
def get_shift_input_fn(batch_size, sequence_length, seed=None):
def input_fn():
random_sequence = tf.random_uniform(
[batch_size, sequence_length + 1], 0, 2, dtype=tf.int32, seed=seed)
labels = tf.slice(
random_sequence, [0, 0], [batch_size, sequence_length])
inputs = tf.expand_dims(
tf.to_float(tf.slice(
random_sequence, [0, 1], [batch_size, sequence_length])), 2)
return {'inputs': inputs}, labels
return input_fn
seq_columns = [tf.contrib.layers.real_valued_column(
'inputs', dimension=cell_size)]
config = tf.contrib.learn.RunConfig(tf_random_seed=21212)
sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=cell_size,
sequence_feature_columns=seq_columns,
learning_rate=learning_rate,
config=config)
train_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=12321)
eval_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=32123)
sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
evaluation = sequence_estimator.evaluate(
input_fn=eval_input_fn, steps=eval_steps)
accuracy = evaluation['accuracy']
self.assertGreater(accuracy, accuracy_threshold,
'Accuracy should be higher than {}; got {}'.format(
accuracy_threshold, accuracy))
def testLearnShiftByOne(self):
"""Tests that learning a 'shift-by-one' example.
Each label sequence consists of the input sequence 'shifted' by one place.
The RNN must learn to 'remember' the previous input.
"""
batch_size = 16
sequence_length = 32
train_steps = 200
eval_steps = 20
cell_size = 4
learning_rate = 0.3
accuracy_threshold = 0.9
def get_shift_input_fn(batch_size, sequence_length, seed=None):
def input_fn():
random_sequence = tf.random_uniform(
[batch_size, sequence_length + 1], 0, 2, dtype=tf.int32, seed=seed)
labels = tf.slice(
random_sequence, [0, 0], [batch_size, sequence_length])
inputs = tf.expand_dims(
tf.to_float(tf.slice(
random_sequence, [0, 1], [batch_size, sequence_length])), 2)
return {'inputs': inputs}, labels
return input_fn
seq_columns = [tf.contrib.layers.real_valued_column(
'inputs', dimension=cell_size)]
config = tf.contrib.learn.RunConfig(tf_random_seed=21212)
sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=cell_size,
sequence_feature_columns=seq_columns,
learning_rate=learning_rate,
config=config,
predict_probabilities=True)
train_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=12321)
eval_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=32123)
sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
evaluation = sequence_estimator.evaluate(
input_fn=eval_input_fn, steps=eval_steps)
accuracy = evaluation['accuracy']
self.assertGreater(accuracy, accuracy_threshold,
'Accuracy should be higher than {}; got {}'.format(
accuracy_threshold, accuracy))
# Testing `predict` when `predict_probabilities=True`.
prediction_dict = sequence_estimator.predict(
input_fn=eval_input_fn, as_iterable=False)
self.assertListEqual(
sorted(list(prediction_dict.keys())),
sorted([dynamic_rnn_estimator.RNNKeys.PREDICTIONS_KEY,
dynamic_rnn_estimator.RNNKeys.PROBABILITIES_KEY,
dynamic_rnn_estimator.RNNKeys.FINAL_STATE_KEY]))
predictions = prediction_dict[dynamic_rnn_estimator.RNNKeys.PREDICTIONS_KEY]
probabilities = prediction_dict[
dynamic_rnn_estimator.RNNKeys.PROBABILITIES_KEY]
self.assertListEqual(list(predictions.shape), [batch_size, sequence_length])
self.assertListEqual(
list(probabilities.shape), [batch_size, sequence_length, 2])
def testMultipleRuns(self):
"""Tests resuming training by feeding state."""
cell_sizes = [4, 7]
batch_size = 11
learning_rate = 0.1
train_sequence_length = 21
train_steps = 121
prediction_steps = [3, 2, 5, 11, 6]
def get_input_fn(batch_size, sequence_length, state_dict, starting_step=0):
def input_fn():
sequence = constant_op.constant(
[[(starting_step + i + j) % 2 for j in range(sequence_length + 1)]
for i in range(batch_size)],
dtype=dtypes.int32)
labels = array_ops.slice(sequence, [0, 0],
[batch_size, sequence_length])
inputs = array_ops.expand_dims(
math_ops.to_float(
array_ops.slice(sequence, [0, 1], [batch_size, sequence_length
])), 2)
input_dict = state_dict
input_dict['inputs'] = inputs
return input_dict, labels
return input_fn
seq_columns = [feature_column.real_valued_column('inputs', dimension=1)]
config = run_config.RunConfig(tf_random_seed=21212)
cell = core_rnn_cell_impl.MultiRNNCell(
[core_rnn_cell_impl.BasicLSTMCell(size) for size in cell_sizes])
model_dir = tempfile.mkdtemp()
sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=None,
sequence_feature_columns=seq_columns,
cell_type=cell,
learning_rate=learning_rate,
config=config,
model_dir=model_dir)
train_input_fn = get_input_fn(
batch_size, train_sequence_length, state_dict={})
sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
def incremental_predict(estimator, increments):
"""Run `estimator.predict` for `i` steps for `i` in `increments`."""
step = 0
incremental_state_dict = {}
for increment in increments:
input_fn = get_input_fn(
batch_size,
increment,
state_dict=incremental_state_dict,
starting_step=step)
prediction_dict = estimator.predict(
input_fn=input_fn, as_iterable=False)
step += increment
incremental_state_dict = {
k: v
for (k, v) in prediction_dict.items()
if k.startswith(dynamic_rnn_estimator.RNNKeys.STATE_PREFIX)
}
return prediction_dict
pred_all_at_once = incremental_predict(sequence_estimator,
[sum(prediction_steps)])
pred_step_by_step = incremental_predict(sequence_estimator,
prediction_steps)
# Check that the last `prediction_steps[-1]` steps give the same
# predictions.
np.testing.assert_array_equal(
pred_all_at_once['predictions'][:, -1 * prediction_steps[-1]:],
pred_step_by_step['predictions'],
err_msg='Mismatch on last {} predictions.'.format(prediction_steps[-1]))
# Check that final states are identical.
for k, v in pred_all_at_once.items():
if k.startswith(dynamic_rnn_estimator.RNNKeys.STATE_PREFIX):
np.testing.assert_array_equal(
v, pred_step_by_step[k], err_msg='Mismatch on state {}.'.format(k))
def testMultipleRuns(self):
"""Tests resuming training by feeding state."""
cell_sizes = [4, 7]
batch_size = 11
learning_rate = 0.1
train_sequence_length = 21
train_steps = 121
prediction_steps = [3, 2, 5, 11, 6]
def get_input_fn(
batch_size, sequence_length, state_dict, starting_step=0):
def input_fn():
sequence = tf.constant(
[[(starting_step + i + j) % 2 for j in range(sequence_length + 1)]
for i in range(batch_size)], dtype=tf.int32)
labels = tf.slice(
sequence, [0, 0], [batch_size, sequence_length])
inputs = tf.expand_dims(
tf.to_float(tf.slice(
sequence, [0, 1], [batch_size, sequence_length])), 2)
input_dict = state_dict
input_dict['inputs'] = inputs
return input_dict, labels
return input_fn
seq_columns = [tf.contrib.layers.real_valued_column(
'inputs', dimension=1)]
config = tf.contrib.learn.RunConfig(tf_random_seed=21212)
cell = tf.contrib.rnn.MultiRNNCell(
[tf.contrib.rnn.BasicLSTMCell(size) for size in cell_sizes])
model_dir = tempfile.mkdtemp()
sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier(
num_classes=2,
num_units=None,
sequence_feature_columns=seq_columns,
cell_type=cell,
learning_rate=learning_rate,
config=config,
model_dir=model_dir)
train_input_fn = get_input_fn(
batch_size, train_sequence_length, state_dict={})
sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
def incremental_predict(estimator, increments):
"""Run `estimator.predict` for `i` steps for `i` in `increments`."""
step = 0
incremental_state_dict = {}
for increment in increments:
input_fn = get_input_fn(
batch_size,
increment,
state_dict=incremental_state_dict,
starting_step=step)
prediction_dict = estimator.predict(
input_fn=input_fn, as_iterable=False)
step += increment
incremental_state_dict = {
k: v
for (k, v) in prediction_dict.items()
if k.startswith(dynamic_rnn_estimator.RNNKeys.STATE_PREFIX)
}
return prediction_dict
pred_all_at_once = incremental_predict(
sequence_estimator, [sum(prediction_steps)])
pred_step_by_step = incremental_predict(
sequence_estimator, prediction_steps)
# Check that the last `prediction_steps[-1]` steps give the same
# predictions.
np.testing.assert_array_equal(
pred_all_at_once['predictions'][:, -1 * prediction_steps[-1]:],
pred_step_by_step['predictions'],
err_msg='Mismatch on last {} predictions.'.format(prediction_steps[-1]))
# Check that final states are identical.
for k, v in pred_all_at_once.items():
if k.startswith(dynamic_rnn_estimator.RNNKeys.STATE_PREFIX):
np.testing.assert_array_equal(
v, pred_step_by_step[k], err_msg='Mismatch on state {}.'.format(k))
