def test_unifiedLSTM_with_cond(self): # This test is to demonstrate the graph rewrite of grappler plugin under # the condition that the function returns different number of internal # states. input_shape = 10 rnn_state_size = 8 output_shape = 8 timestep = 4 batch = 100 epoch = 1 with self.cached_session(config=_config, use_gpu=True) as sess: (x_train, y_train), _ = testing_utils.get_test_data( train_samples=batch, test_samples=0, input_shape=(timestep, input_shape), num_classes=output_shape) y_train = keras.utils.to_categorical(y_train, output_shape) layer = UnifiedLSTM(rnn_state_size, return_runtime=True) inputs = array_ops.placeholder( dtypes.float32, shape=(None, timestep, input_shape), name='inputs') predict = array_ops.placeholder( dtypes.float32, shape=(None, output_shape), name='predict') zeros = array_ops.zeros([batch, output_shape]) dummy_runtime = constant_op.constant( 'unknown', dtype=dtypes.string, name='runtime') a = constant_op.constant(0) b = constant_op.constant(1) # Will always run the lstm layer. outputs, runtime = control_flow_ops.cond( gen_math_ops.less(a, b), lambda: layer(inputs), lambda: (zeros, dummy_runtime)) loss = losses.softmax_cross_entropy(predict, outputs) optimizer = gradient_descent.GradientDescentOptimizer(0.001) train_op = optimizer.minimize(loss) sess.run([variables.global_variables_initializer()]) existing_loss = 0 for _ in range(epoch): loss_value, _, runtime_value = sess.run([loss, train_op, runtime], { inputs: x_train, predict: y_train }) if test.is_gpu_available(): self.assertEqual(runtime_value, b'cudnn') else: self.assertEqual(runtime_value, b'cpu') # Make sure the loss is updated for every epoch # (layer weights properly updated). self.assertNotEqual(existing_loss, loss_value) existing_loss = loss_value
def test_could_use_defun_backend(self, activation, recurrent_activation, recurrent_dropout, unroll, use_bias, bias_regularizer): layer = UnifiedLSTM(1, activation=activation, recurrent_activation=recurrent_activation, recurrent_dropout=recurrent_dropout, unroll=unroll, use_bias=use_bias, bias_regularizer=bias_regularizer) self.assertFalse(layer.could_use_cudnn)
def test_dynamic_behavior_LSTM(self): num_samples = 2 timesteps = 3 embedding_dim = 4 units = 2 layer = UnifiedLSTM(units, input_shape=(None, embedding_dim)) model = keras.models.Sequential() model.add(layer) model.compile(gradient_descent.GradientDescentOptimizer(0.001), 'mse') x = np.random.random((num_samples, timesteps, embedding_dim)) y = np.random.random((num_samples, units)) model.train_on_batch(x, y)
def test_unified_lstm_output_on_multiple_kernel(self): input_shape = 10 rnn_state_size = 8 timestep = 4 batch = 100 x_train = np.random.random((batch, timestep, input_shape)) inputs = keras.layers.Input(shape=[timestep, input_shape], dtype=dtypes.float32) with test_util.device(use_gpu=False): layer = UnifiedLSTM(rnn_state_size) output = layer(inputs) cpu_model = keras.models.Model(inputs, output) weights = cpu_model.get_weights() y_1 = cpu_model.predict(x_train) with test_util.device(use_gpu=True): layer = UnifiedLSTM(rnn_state_size) output = layer(inputs) gpu_model = keras.models.Model(inputs, output) gpu_model.set_weights(weights) y_2 = gpu_model.predict(x_train) # Note that CuDNN uses 'sigmoid' as activation, so the unified LSTM uses # 'sigmoid' as default. Construct the canonical LSTM with sigmoid to achieve # the same output. with test_util.device(use_gpu=True): layer = keras.layers.LSTM(rnn_state_size, recurrent_activation='sigmoid') output = layer(inputs) canonical_model = keras.models.Model(inputs, output) # Remove the extra cudnn bias since canonical lstm will not use it. canonical_model.set_weights(weights[:3]) y_3 = canonical_model.predict(x_train) self.assertAllClose(y_1, y_2) self.assertAllClose(y_2, y_3)
def test_static_shape_inference_LSTM(self): # Github issue: 15165 timesteps = 3 embedding_dim = 4 units = 2 model = keras.models.Sequential() inputs = keras.layers.Dense( embedding_dim, input_shape=(timesteps, embedding_dim)) model.add(inputs) layer = UnifiedLSTM(units, return_sequences=True) model.add(layer) outputs = model.layers[-1].output self.assertEqual(outputs.get_shape().as_list(), [None, timesteps, units])
def test_unifiedLSTM(self): input_shape = 10 rnn_state_size = 8 output_shape = 8 timestep = 4 batch = 100 epoch = 1 with self.cached_session(config=_config, use_gpu=True) as sess: (x_train, y_train), _ = testing_utils.get_test_data( train_samples=batch, test_samples=0, input_shape=(timestep, input_shape), num_classes=output_shape) y_train = keras.utils.to_categorical(y_train, output_shape) layer = UnifiedLSTM(rnn_state_size, return_runtime=True) inputs = array_ops.placeholder(dtypes.float32, shape=(None, timestep, input_shape), name='inputs') predict = array_ops.placeholder(dtypes.float32, shape=(None, output_shape), name='predict') outputs, runtime = layer(inputs) loss = losses.softmax_cross_entropy(predict, outputs) optimizer = gradient_descent.GradientDescentOptimizer(0.001) train_op = optimizer.minimize(loss) sess.run([variables.global_variables_initializer()]) existing_loss = 0 for _ in range(epoch): loss_value, _, runtime_value = sess.run( [loss, train_op, runtime], { inputs: x_train, predict: y_train }) if test.is_gpu_available(): self.assertEqual(runtime_value, b'cudnn') else: self.assertEqual(runtime_value, b'cpu') # Make sure the loss is updated for every epoch # (layer weights properly updated). self.assertNotEqual(existing_loss, loss_value) existing_loss = loss_value
def test_return_state(self): num_states = 2 timesteps = 3 embedding_dim = 4 units = 3 num_samples = 2 inputs = keras.Input(batch_shape=(num_samples, timesteps, embedding_dim)) layer = UnifiedLSTM(units, return_state=True, stateful=True) outputs = layer(inputs) state = outputs[1:] assert len(state) == num_states model = keras.models.Model(inputs, state[0]) inputs = np.random.random((num_samples, timesteps, embedding_dim)) state = model.predict(inputs) self.assertAllClose(keras.backend.eval(layer.states[0]), state, atol=1e-4)
def _time_performance_run_unifed_lstm_gpu(self, test_config, x_train, y_train): # Get performance number for Unified_LSTM with grappler swap the impl input_shape = test_config['input_shape'] rnn_state_size = test_config['rnn_state_size'] timestep = test_config['timestep'] layer = UnifiedLSTM(rnn_state_size) inputs = keras.layers.Input(shape=[timestep, input_shape], dtype=dtypes.float32) outputs = layer(inputs) model = keras.models.Model(inputs, outputs) model.compile('sgd', 'mse') sec_per_epoch = self._measure_performance(test_config, model, x_train, y_train) logging.info('Average performance for %s per epoch is: %s', 'Unified LSTM', sec_per_epoch) return sec_per_epoch
def test_specify_state_with_masking(self): num_states = 2 timesteps = 3 embedding_dim = 4 units = 3 num_samples = 2 inputs = keras.Input((timesteps, embedding_dim)) _ = keras.layers.Masking()(inputs) initial_state = [keras.Input((units, )) for _ in range(num_states)] output = UnifiedLSTM(units)(inputs, initial_state=initial_state) model = keras.models.Model([inputs] + initial_state, output) model.compile( loss='categorical_crossentropy', optimizer=gradient_descent.GradientDescentOptimizer(0.01)) inputs = np.random.random((num_samples, timesteps, embedding_dim)) initial_state = [ np.random.random((num_samples, units)) for _ in range(num_states) ] targets = np.random.random((num_samples, units)) model.train_on_batch([inputs] + initial_state, targets)
def DISABLED_test_specify_initial_state_non_keras_tensor(self): num_states = 2 timesteps = 3 embedding_dim = 4 units = 3 num_samples = 2 # Test with non-Keras tensor inputs = keras.Input((timesteps, embedding_dim)) initial_state = [ keras.backend.random_normal_variable((num_samples, units), 0, 1) for _ in range(num_states) ] layer = UnifiedLSTM(units) output = layer(inputs, initial_state=initial_state) model = keras.models.Model(inputs, output) model.compile( loss='categorical_crossentropy', optimizer=gradient_descent.GradientDescentOptimizer(0.01)) inputs = np.random.random((num_samples, timesteps, embedding_dim)) targets = np.random.random((num_samples, units)) model.train_on_batch(inputs, targets)