def estimator_fn(): return dynamic_rnn_estimator.multi_value_rnn_classifier( num_classes=2, num_units=self.NUM_RNN_CELL_UNITS, sequence_feature_columns=self.sequence_feature_columns, context_feature_columns=self.context_feature_columns, predict_probabilities=True, model_dir=model_dir)
def testLearnShiftByOne(self): """Tests that `_MultiValueRNNEstimator` can learn a 'shift-by-one' example. Each label sequence consists of the input sequence 'shifted' by one place. The RNN must learn to 'remember' the previous input. """ batch_size = 16 sequence_length = 32 train_steps = 200 eval_steps = 20 cell_size = 4 learning_rate = 0.3 accuracy_threshold = 0.9 def get_shift_input_fn(batch_size, sequence_length, seed=None): def input_fn(): random_sequence = tf.random_uniform( [batch_size, sequence_length + 1], 0, 2, dtype=tf.int32, seed=seed) labels = tf.slice(random_sequence, [0, 0], [batch_size, sequence_length]) inputs = tf.expand_dims( tf.to_float( tf.slice(random_sequence, [0, 1], [batch_size, sequence_length])), 2) return {'inputs': inputs}, labels return input_fn seq_columns = [ tf.contrib.layers.real_valued_column('inputs', dimension=cell_size) ] config = tf.contrib.learn.RunConfig(tf_random_seed=21212) sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier( num_classes=2, num_units=cell_size, sequence_feature_columns=seq_columns, learning_rate=learning_rate, config=config) train_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=12321) eval_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=32123) sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) evaluation = sequence_estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) accuracy = evaluation['accuracy'] self.assertGreater( accuracy, accuracy_threshold, 'Accuracy should be higher than {}; got {}'.format( accuracy_threshold, accuracy))
def testMultiRNNState(self): """Test that state flattening/reconstruction works for `MultiRNNCell`.""" batch_size = 11 sequence_length = 16 train_steps = 5 cell_sizes = [4, 8, 7] learning_rate = 0.1 def get_shift_input_fn(batch_size, sequence_length, seed=None): def input_fn(): random_sequence = random_ops.random_uniform( [batch_size, sequence_length + 1], 0, 2, dtype=dtypes.int32, seed=seed) labels = array_ops.slice(random_sequence, [0, 0], [batch_size, sequence_length]) inputs = array_ops.expand_dims( math_ops.to_float( array_ops.slice(random_sequence, [0, 1], [batch_size, sequence_length])), 2) input_dict = { dynamic_rnn_estimator._get_state_name(i): random_ops.random_uniform( [batch_size, cell_size], seed=((i + 1) * seed)) for i, cell_size in enumerate([4, 4, 8, 8, 7, 7]) } input_dict['inputs'] = inputs return input_dict, labels return input_fn seq_columns = [feature_column.real_valued_column('inputs', dimension=1)] config = run_config.RunConfig(tf_random_seed=21212) cell = core_rnn_cell_impl.MultiRNNCell( [core_rnn_cell_impl.BasicLSTMCell(size) for size in cell_sizes]) sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier( num_classes=2, num_units=None, sequence_feature_columns=seq_columns, cell_type=cell, learning_rate=learning_rate, config=config, predict_probabilities=True) train_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=12321) eval_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=32123) sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) prediction_dict = sequence_estimator.predict( input_fn=eval_input_fn, as_iterable=False) for i, state_size in enumerate([4, 4, 8, 8, 7, 7]): state_piece = prediction_dict[dynamic_rnn_estimator._get_state_name(i)] self.assertListEqual(list(state_piece.shape), [batch_size, state_size])
def testLegacyConstructor(self): """Exercise legacy constructor function.""" num_units = 16 num_layers = 6 output_keep_prob = 0.9 input_keep_prob = 0.7 batch_size = 11 learning_rate = 0.1 train_sequence_length = 21 train_steps = 121 def get_input_fn(batch_size, sequence_length, state_dict, starting_step=0): def input_fn(): sequence = constant_op.constant( [[(starting_step + i + j) % 2 for j in range(sequence_length + 1)] for i in range(batch_size)], dtype=dtypes.int32) labels = array_ops.slice(sequence, [0, 0], [batch_size, sequence_length]) inputs = array_ops.expand_dims( math_ops.to_float( array_ops.slice(sequence, [0, 1], [batch_size, sequence_length])), 2) input_dict = state_dict input_dict['inputs'] = inputs return input_dict, labels return input_fn seq_columns = [ feature_column.real_valued_column('inputs', dimension=1) ] config = run_config.RunConfig(tf_random_seed=21212) model_dir = tempfile.mkdtemp() sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier( num_classes=2, num_units=num_units, num_rnn_layers=num_layers, input_keep_probability=input_keep_prob, output_keep_probability=output_keep_prob, sequence_feature_columns=seq_columns, learning_rate=learning_rate, config=config, model_dir=model_dir) train_input_fn = get_input_fn(batch_size, train_sequence_length, state_dict={}) sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
def testLegacyConstructor(self): """Exercise legacy constructor function.""" num_units = 16 num_layers = 6 output_keep_prob = 0.9 input_keep_prob = 0.7 batch_size = 11 learning_rate = 0.1 train_sequence_length = 21 train_steps = 121 def get_input_fn(batch_size, sequence_length, state_dict, starting_step=0): def input_fn(): sequence = constant_op.constant( [[(starting_step + i + j) % 2 for j in range(sequence_length + 1)] for i in range(batch_size)], dtype=dtypes.int32) labels = array_ops.slice(sequence, [0, 0], [batch_size, sequence_length]) inputs = array_ops.expand_dims( math_ops.to_float( array_ops.slice(sequence, [0, 1], [batch_size, sequence_length ])), 2) input_dict = state_dict input_dict['inputs'] = inputs return input_dict, labels return input_fn seq_columns = [feature_column.real_valued_column('inputs', dimension=1)] config = run_config.RunConfig(tf_random_seed=21212) model_dir = tempfile.mkdtemp() sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier( num_classes=2, num_units=num_units, num_rnn_layers=num_layers, input_keep_probability=input_keep_prob, output_keep_probability=output_keep_prob, sequence_feature_columns=seq_columns, learning_rate=learning_rate, config=config, model_dir=model_dir) train_input_fn = get_input_fn( batch_size, train_sequence_length, state_dict={}) sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps)
def testLearnShiftByOne(self): """Tests that `_MultiValueRNNEstimator` can learn a 'shift-by-one' example. Each label sequence consists of the input sequence 'shifted' by one place. The RNN must learn to 'remember' the previous input. """ batch_size = 16 sequence_length = 32 train_steps = 200 eval_steps = 20 cell_size = 4 learning_rate = 0.3 accuracy_threshold = 0.9 def get_shift_input_fn(batch_size, sequence_length, seed=None): def input_fn(): random_sequence = tf.random_uniform( [batch_size, sequence_length + 1], 0, 2, dtype=tf.int32, seed=seed) labels = tf.slice( random_sequence, [0, 0], [batch_size, sequence_length]) inputs = tf.expand_dims( tf.to_float(tf.slice( random_sequence, [0, 1], [batch_size, sequence_length])), 2) return {'inputs': inputs}, labels return input_fn seq_columns = [tf.contrib.layers.real_valued_column( 'inputs', dimension=cell_size)] config = tf.contrib.learn.RunConfig(tf_random_seed=21212) sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier( num_classes=2, num_units=cell_size, sequence_feature_columns=seq_columns, learning_rate=learning_rate, config=config) train_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=12321) eval_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=32123) sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) evaluation = sequence_estimator.evaluate( input_fn=eval_input_fn, steps=eval_steps) accuracy = evaluation['accuracy'] self.assertGreater(accuracy, accuracy_threshold, 'Accuracy should be higher than {}; got {}'.format( accuracy_threshold, accuracy))
def testLearnShiftByOne(self): """Tests that learning a 'shift-by-one' example. Each label sequence consists of the input sequence 'shifted' by one place. The RNN must learn to 'remember' the previous input. """ batch_size = 16 sequence_length = 32 train_steps = 200 eval_steps = 20 cell_size = 4 learning_rate = 0.3 accuracy_threshold = 0.9 def get_shift_input_fn(batch_size, sequence_length, seed=None): def input_fn(): random_sequence = tf.random_uniform( [batch_size, sequence_length + 1], 0, 2, dtype=tf.int32, seed=seed) labels = tf.slice( random_sequence, [0, 0], [batch_size, sequence_length]) inputs = tf.expand_dims( tf.to_float(tf.slice( random_sequence, [0, 1], [batch_size, sequence_length])), 2) return {'inputs': inputs}, labels return input_fn seq_columns = [tf.contrib.layers.real_valued_column( 'inputs', dimension=cell_size)] config = tf.contrib.learn.RunConfig(tf_random_seed=21212) sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier( num_classes=2, num_units=cell_size, sequence_feature_columns=seq_columns, learning_rate=learning_rate, config=config, predict_probabilities=True) train_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=12321) eval_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=32123) sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) evaluation = sequence_estimator.evaluate( input_fn=eval_input_fn, steps=eval_steps) accuracy = evaluation['accuracy'] self.assertGreater(accuracy, accuracy_threshold, 'Accuracy should be higher than {}; got {}'.format( accuracy_threshold, accuracy)) # Testing `predict` when `predict_probabilities=True`. prediction_dict = sequence_estimator.predict( input_fn=eval_input_fn, as_iterable=False) self.assertListEqual( sorted(list(prediction_dict.keys())), sorted([dynamic_rnn_estimator.RNNKeys.PREDICTIONS_KEY, dynamic_rnn_estimator.RNNKeys.PROBABILITIES_KEY, dynamic_rnn_estimator.RNNKeys.FINAL_STATE_KEY])) predictions = prediction_dict[dynamic_rnn_estimator.RNNKeys.PREDICTIONS_KEY] probabilities = prediction_dict[ dynamic_rnn_estimator.RNNKeys.PROBABILITIES_KEY] self.assertListEqual(list(predictions.shape), [batch_size, sequence_length]) self.assertListEqual( list(probabilities.shape), [batch_size, sequence_length, 2])
def testMultipleRuns(self): """Tests resuming training by feeding state.""" cell_sizes = [4, 7] batch_size = 11 learning_rate = 0.1 train_sequence_length = 21 train_steps = 121 prediction_steps = [3, 2, 5, 11, 6] def get_input_fn(batch_size, sequence_length, state_dict, starting_step=0): def input_fn(): sequence = constant_op.constant( [[(starting_step + i + j) % 2 for j in range(sequence_length + 1)] for i in range(batch_size)], dtype=dtypes.int32) labels = array_ops.slice(sequence, [0, 0], [batch_size, sequence_length]) inputs = array_ops.expand_dims( math_ops.to_float( array_ops.slice(sequence, [0, 1], [batch_size, sequence_length ])), 2) input_dict = state_dict input_dict['inputs'] = inputs return input_dict, labels return input_fn seq_columns = [feature_column.real_valued_column('inputs', dimension=1)] config = run_config.RunConfig(tf_random_seed=21212) cell = core_rnn_cell_impl.MultiRNNCell( [core_rnn_cell_impl.BasicLSTMCell(size) for size in cell_sizes]) model_dir = tempfile.mkdtemp() sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier( num_classes=2, num_units=None, sequence_feature_columns=seq_columns, cell_type=cell, learning_rate=learning_rate, config=config, model_dir=model_dir) train_input_fn = get_input_fn( batch_size, train_sequence_length, state_dict={}) sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) def incremental_predict(estimator, increments): """Run `estimator.predict` for `i` steps for `i` in `increments`.""" step = 0 incremental_state_dict = {} for increment in increments: input_fn = get_input_fn( batch_size, increment, state_dict=incremental_state_dict, starting_step=step) prediction_dict = estimator.predict( input_fn=input_fn, as_iterable=False) step += increment incremental_state_dict = { k: v for (k, v) in prediction_dict.items() if k.startswith(dynamic_rnn_estimator.RNNKeys.STATE_PREFIX) } return prediction_dict pred_all_at_once = incremental_predict(sequence_estimator, [sum(prediction_steps)]) pred_step_by_step = incremental_predict(sequence_estimator, prediction_steps) # Check that the last `prediction_steps[-1]` steps give the same # predictions. np.testing.assert_array_equal( pred_all_at_once['predictions'][:, -1 * prediction_steps[-1]:], pred_step_by_step['predictions'], err_msg='Mismatch on last {} predictions.'.format(prediction_steps[-1])) # Check that final states are identical. for k, v in pred_all_at_once.items(): if k.startswith(dynamic_rnn_estimator.RNNKeys.STATE_PREFIX): np.testing.assert_array_equal( v, pred_step_by_step[k], err_msg='Mismatch on state {}.'.format(k))
def testMultipleRuns(self): """Tests resuming training by feeding state.""" cell_sizes = [4, 7] batch_size = 11 learning_rate = 0.1 train_sequence_length = 21 train_steps = 121 prediction_steps = [3, 2, 5, 11, 6] def get_input_fn( batch_size, sequence_length, state_dict, starting_step=0): def input_fn(): sequence = tf.constant( [[(starting_step + i + j) % 2 for j in range(sequence_length + 1)] for i in range(batch_size)], dtype=tf.int32) labels = tf.slice( sequence, [0, 0], [batch_size, sequence_length]) inputs = tf.expand_dims( tf.to_float(tf.slice( sequence, [0, 1], [batch_size, sequence_length])), 2) input_dict = state_dict input_dict['inputs'] = inputs return input_dict, labels return input_fn seq_columns = [tf.contrib.layers.real_valued_column( 'inputs', dimension=1)] config = tf.contrib.learn.RunConfig(tf_random_seed=21212) cell = tf.contrib.rnn.MultiRNNCell( [tf.contrib.rnn.BasicLSTMCell(size) for size in cell_sizes]) model_dir = tempfile.mkdtemp() sequence_estimator = dynamic_rnn_estimator.multi_value_rnn_classifier( num_classes=2, num_units=None, sequence_feature_columns=seq_columns, cell_type=cell, learning_rate=learning_rate, config=config, model_dir=model_dir) train_input_fn = get_input_fn( batch_size, train_sequence_length, state_dict={}) sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) def incremental_predict(estimator, increments): """Run `estimator.predict` for `i` steps for `i` in `increments`.""" step = 0 incremental_state_dict = {} for increment in increments: input_fn = get_input_fn( batch_size, increment, state_dict=incremental_state_dict, starting_step=step) prediction_dict = estimator.predict( input_fn=input_fn, as_iterable=False) step += increment incremental_state_dict = { k: v for (k, v) in prediction_dict.items() if k.startswith(dynamic_rnn_estimator.RNNKeys.STATE_PREFIX) } return prediction_dict pred_all_at_once = incremental_predict( sequence_estimator, [sum(prediction_steps)]) pred_step_by_step = incremental_predict( sequence_estimator, prediction_steps) # Check that the last `prediction_steps[-1]` steps give the same # predictions. np.testing.assert_array_equal( pred_all_at_once['predictions'][:, -1 * prediction_steps[-1]:], pred_step_by_step['predictions'], err_msg='Mismatch on last {} predictions.'.format(prediction_steps[-1])) # Check that final states are identical. for k, v in pred_all_at_once.items(): if k.startswith(dynamic_rnn_estimator.RNNKeys.STATE_PREFIX): np.testing.assert_array_equal( v, pred_step_by_step[k], err_msg='Mismatch on state {}.'.format(k))