def test_multi_output_model_with_none_masking(self): with self.test_session(): def func(x): return [x * 0.2, x * 0.3] def output_shape(input_shape): return [input_shape, input_shape] i = keras.layers.Input(shape=(3, 2, 1)) o = keras.layers.Lambda(function=func, output_shape=output_shape)(i) self.assertEqual(keras.backend.int_shape(o[0]), (None, 3, 2, 1)) self.assertEqual(keras.backend.int_shape(o[1]), (None, 3, 2, 1)) o = keras.layers.add(o) model = keras.Model(i, o) i2 = keras.layers.Input(shape=(3, 2, 1)) o2 = model(i2) model2 = keras.Model(i2, o2) x = np.random.random((4, 3, 2, 1)) out = model2.predict(x) assert out.shape == (4, 3, 2, 1) self.assertAllClose(out, x * 0.2 + x * 0.3, atol=1e-4)
def test_cudnnrnn_bidirectional(self): if test.is_gpu_available(cuda_only=True): with self.test_session(use_gpu=True): rnn = keras.layers.CuDNNGRU samples = 2 dim = 2 timesteps = 2 output_dim = 2 mode = 'concat' x = np.random.random((samples, timesteps, dim)) target_dim = 2 * output_dim if mode == 'concat' else output_dim y = np.random.random((samples, target_dim)) # test with Sequential model model = keras.Sequential() model.add( keras.layers.Bidirectional( rnn(output_dim), merge_mode=mode, input_shape=(None, dim))) model.compile( loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001)) model.fit(x, y, epochs=1, batch_size=1) # test config model.get_config() model = keras.models.model_from_json(model.to_json()) model.summary() # test stacked bidirectional layers model = keras.Sequential() model.add( keras.layers.Bidirectional( rnn(output_dim, return_sequences=True), merge_mode=mode, input_shape=(None, dim))) model.add(keras.layers.Bidirectional(rnn(output_dim), merge_mode=mode)) model.compile( loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001)) model.fit(x, y, epochs=1, batch_size=1) # test with functional API inputs = keras.Input((timesteps, dim)) outputs = keras.layers.Bidirectional( rnn(output_dim), merge_mode=mode)( inputs) model = keras.Model(inputs, outputs) model.compile( loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001)) model.fit(x, y, epochs=1, batch_size=1) # Bidirectional and stateful inputs = keras.Input(batch_shape=(1, timesteps, dim)) outputs = keras.layers.Bidirectional( rnn(output_dim, stateful=True), merge_mode=mode)( inputs) model = keras.Model(inputs, outputs) model.compile( loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001)) model.fit(x, y, epochs=1, batch_size=1)
def test_Bidirectional_with_constants_layer_passing_initial_state(self): with self.test_session(): # Test basic case. x = keras.Input((5, 5)) c = keras.Input((3, )) s_for = keras.Input((32, )) s_bac = keras.Input((32, )) cell = _RNNCellWithConstants(32) custom_objects = {'_RNNCellWithConstants': _RNNCellWithConstants} with keras.utils.CustomObjectScope(custom_objects): layer = keras.layers.Bidirectional(keras.layers.RNN(cell)) y = layer(x, initial_state=[s_for, s_bac], constants=c) model = keras.Model([x, s_for, s_bac, c], y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch([ np.zeros((6, 5, 5)), np.zeros((6, 32)), np.zeros((6, 32)), np.zeros((6, 3)) ], np.zeros((6, 64))) # Test basic case serialization. x_np = np.random.random((6, 5, 5)) s_fw_np = np.random.random((6, 32)) s_bk_np = np.random.random((6, 32)) c_np = np.random.random((6, 3)) y_np = model.predict([x_np, s_fw_np, s_bk_np, c_np]) weights = model.get_weights() config = layer.get_config() with keras.utils.CustomObjectScope(custom_objects): layer = keras.layers.Bidirectional.from_config( copy.deepcopy(config)) y = layer(x, initial_state=[s_for, s_bac], constants=c) model = keras.Model([x, s_for, s_bac, c], y) model.set_weights(weights) y_np_2 = model.predict([x_np, s_fw_np, s_bk_np, c_np]) self.assertAllClose(y_np, y_np_2, atol=1e-4) # Verify that state is used y_np_2_different_s = model.predict( [x_np, s_fw_np + 10., s_bk_np + 10., c_np]) assert np.mean(y_np - y_np_2_different_s) != 0 # Test flat list inputs with keras.utils.CustomObjectScope(custom_objects): layer = keras.layers.Bidirectional.from_config( copy.deepcopy(config)) y = layer([x, s_for, s_bac, c]) model = keras.Model([x, s_for, s_bac, c], y) model.set_weights(weights) y_np_3 = model.predict([x_np, s_fw_np, s_bk_np, c_np]) self.assertAllClose(y_np, y_np_3, atol=1e-4)
def test_saving_model_with_long_weights_names(self): if h5py is None: return # Skip test if models cannot be saved. with self.test_session(): x = keras.Input(shape=(2, ), name='nested_model_input') f = x for i in range(4): f = keras.layers.Dense(2, name='nested_model_dense_%d' % (i, ))(f) # This layer name will make the `weights_name` # HDF5 attribute blow out of proportion. f = keras.layers.Dense(2, name='nested_model_output' + ('x' * (2**14)))(f) nested_model = keras.Model(inputs=[x], outputs=[f], name='nested_model') x = keras.Input(shape=(2, ), name='outer_model_input') f = nested_model(x) f = keras.layers.Dense(2, name='outer_model_output')(f) model = keras.Model(inputs=[x], outputs=[f]) model.compile(loss='mse', optimizer='adam', metrics=['acc']) x = np.random.random((1, 2)) y = np.random.random((1, 2)) model.train_on_batch(x, y) out = model.predict(x) fd, fname = tempfile.mkstemp('.h5') keras.models.save_model(model, fname) model = keras.models.load_model(fname) # Check that the HDF5 files contains chunked array # of weight names. with h5py.File(fname, 'r') as h5file: num_weight_arrays = len([ attr for attr in h5file['model_weights']['nested_model'].attrs if attr.startswith('weight_names') ]) # The chunking of layer names array should have happened. self.assertGreater(num_weight_arrays, 0) out2 = model.predict(x) self.assertAllClose(out, out2, atol=1e-05) # Cleanup os.close(fd) os.remove(fname)
def test_using_tf_layers_in_keras_functional_model(self): with self.test_session(): np.random.seed(1337) (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( train_samples=200, test_samples=100, input_shape=(10, ), num_classes=2) y_train = keras.utils.to_categorical(y_train) y_test = keras.utils.to_categorical(y_test) inputs = keras.Input(shape=(10, )) x = tf_core_layers.Dense(32, activation=nn.relu)(inputs) outputs = tf_core_layers.Dense(2, activation=nn.softmax)(x) model = keras.Model(inputs, outputs) model.summary() model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=10, batch_size=16, validation_data=(x_test, y_test), verbose=0) self.assertGreater(history.history['val_acc'][-1], 0.85)
def get_nested_model_3(input_dim, num_classes): # A functional-API model with a subclassed model inside. # NOTE: this requires the inner subclass to implement `compute_output_shape`. inputs = keras.Input(shape=(input_dim,)) x = keras.layers.Dense(32, activation='relu')(inputs) x = keras.layers.BatchNormalization()(x) class Inner(keras.Model): def __init__(self): super(Inner, self).__init__() self.dense1 = keras.layers.Dense(32, activation='relu') self.dense2 = keras.layers.Dense(5, activation='relu') self.bn = keras.layers.BatchNormalization() def call(self, inputs): x = self.dense1(inputs) x = self.dense2(x) return self.bn(x) def compute_output_shape(self, input_shape): return tensor_shape.TensorShape((input_shape[0], 5)) test_model = Inner() x = test_model(x) # pylint: disable=not-callable outputs = keras.layers.Dense(num_classes)(x) return keras.Model(inputs, outputs, name='nested_model_3')
def get_functional_graph_model(input_dim, num_classes): # A simple functional-API model (a.k.a. graph network) inputs = keras.Input(shape=(input_dim,)) x = keras.layers.Dense(32, activation='relu')(inputs) x = keras.layers.BatchNormalization()(x) outputs = keras.layers.Dense(num_classes)(x) return keras.Model(inputs, outputs)
def test_Bidirectional_state_reuse(self): rnn = keras.layers.LSTM samples = 2 dim = 5 timesteps = 3 units = 3 with self.test_session(): inputs = keras.Input((timesteps, dim)) layer = keras.layers.Bidirectional( rnn(units, return_state=True, return_sequences=True)) outputs = layer(inputs) output, state = outputs[0], outputs[1:] # test passing invalid initial_state: passing a tensor with self.assertRaises(ValueError): output = keras.layers.Bidirectional(rnn(units))( output, initial_state=state[0]) # test valid usage: passing a list output = keras.layers.Bidirectional(rnn(units))( output, initial_state=state) model = keras.Model(inputs, output) inputs = np.random.rand(samples, timesteps, dim) outputs = model.predict(inputs)
def test_Bidirectional_dropout(self): rnn = keras.layers.LSTM samples = 2 dim = 5 timesteps = 3 units = 3 merge_mode = 'sum' x = [np.random.rand(samples, timesteps, dim)] with self.test_session(): inputs = keras.Input((timesteps, dim)) wrapped = keras.layers.Bidirectional(rnn(units, dropout=0.2, recurrent_dropout=0.2), merge_mode=merge_mode) outputs = _to_list(wrapped(inputs, training=True)) assert all(not getattr(x, '_uses_learning_phase') for x in outputs) inputs = keras.Input((timesteps, dim)) wrapped = keras.layers.Bidirectional(rnn(units, dropout=0.2, return_state=True), merge_mode=merge_mode) outputs = _to_list(wrapped(inputs)) assert all(x._uses_learning_phase for x in outputs) model = keras.Model(inputs, outputs) assert model.uses_learning_phase y1 = _to_list(model.predict(x)) y2 = _to_list(model.predict(x)) for x1, x2 in zip(y1, y2): self.assertAllClose(x1, x2, atol=1e-5)
def test_state_reuse_with_dropout(self): layer_class = keras.layers.SimpleRNN embedding_dim = 4 units = 3 timesteps = 2 num_samples = 2 with self.test_session(): input1 = keras.Input(batch_shape=(num_samples, timesteps, embedding_dim)) layer = layer_class(units, return_state=True, return_sequences=True, dropout=0.2) state = layer(input1)[1:] input2 = keras.Input(batch_shape=(num_samples, timesteps, embedding_dim)) output = layer_class(units)(input2, initial_state=state) model = keras.Model([input1, input2], output) inputs = [ np.random.random((num_samples, timesteps, embedding_dim)), np.random.random((num_samples, timesteps, embedding_dim)) ] model.predict(inputs)
def test_model_methods_with_eager_tensors_single_io(self): x = keras.layers.Input(shape=(3, ), name='input') y = keras.layers.Dense(4, name='dense')(x) model = keras.Model(x, y) optimizer = RMSPropOptimizer(learning_rate=0.001) loss = 'mse' metrics = ['mae'] model.compile(optimizer, loss, metrics=metrics) inputs = keras.backend.zeros(shape=(10, 3)) targets = keras.backend.zeros(shape=(10, 4)) model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0) model.fit(inputs, targets, epochs=1, batch_size=3, verbose=0, shuffle=False) model.fit(inputs, targets, epochs=1, batch_size=4, verbose=0, validation_data=(inputs, targets)) model.evaluate(inputs, targets, batch_size=2, verbose=0) model.predict(inputs, batch_size=2) model.train_on_batch(inputs, targets) model.test_on_batch(inputs, targets)
def test_activity_regularization_with_model_composition(self): def reg(x): return keras.backend.sum(x) net_a_input = keras.Input((2, )) net_a = net_a_input net_a = keras.layers.Dense(2, kernel_initializer='ones', use_bias=False, activity_regularizer=reg)(net_a) model_a = keras.Model([net_a_input], [net_a]) net_b_input = keras.Input((2, )) net_b = model_a(net_b_input) model_b = keras.Model([net_b_input], [net_b]) model_b.compile(optimizer='sgd', loss=None) x = np.ones((1, 2)) loss = model_b.evaluate(x) self.assertEqual(loss, 4.)
def test_nested_model_with_tensor_input(self): gpus = 2 input_dim = 10 shape = (input_dim,) num_samples = 16 num_classes = 10 if not check_if_compatible_devices(gpus=gpus): return with self.test_session(): input_shape = (num_samples,) + shape x_train = np.random.randint(0, 255, input_shape) y_train = np.random.randint(0, num_classes, (input_shape[0],)) keras.backend.set_learning_phase(True) y_train = keras.utils.to_categorical(y_train, num_classes) x_train = x_train.astype('float32') y_train = y_train.astype('float32') dataset = data.Dataset.from_tensor_slices((x_train, y_train)) dataset = dataset.repeat() dataset = dataset.batch(4) iterator = dataset.make_one_shot_iterator() inputs, targets = iterator.get_next() input_tensor = keras.layers.Input(tensor=inputs) model = keras.models.Sequential() model.add(keras.layers.Dense(3, input_shape=(input_dim,))) model.add(keras.layers.Dense(num_classes)) output = model(input_tensor) outer_model = keras.Model(input_tensor, output) parallel_model = keras.utils.multi_gpu_model(outer_model, gpus=gpus) parallel_model.compile( loss='categorical_crossentropy', optimizer=keras.optimizers.RMSprop(lr=0.0001, decay=1e-6), metrics=['accuracy'], target_tensors=[targets]) parallel_model.fit(epochs=1, steps_per_epoch=3)
def test_saving_model_with_long_layer_names(self): if h5py is None: return # Skip test if models cannot be saved. with self.test_session(): # This layer name will make the `layers_name` HDF5 attribute blow # out of proportion. Note that it fits into the internal HDF5 # attribute memory limit on its own but because h5py converts # the list of layer names into numpy array, which uses the same # amout of memory for every item, it increases the memory # requirements substantially. x = keras.Input(shape=(2, ), name='input_' + ('x' * (2**15))) f = x for i in range(4): f = keras.layers.Dense(2, name='dense_%d' % (i, ))(f) model = keras.Model(inputs=[x], outputs=[f]) model.compile(loss='mse', optimizer='adam', metrics=['acc']) x = np.random.random((1, 2)) y = np.random.random((1, 2)) model.train_on_batch(x, y) out = model.predict(x) fd, fname = tempfile.mkstemp('.h5') keras.models.save_model(model, fname) model = keras.models.load_model(fname) # Check that the HDF5 files contains chunked array # of layer names. with h5py.File(fname, 'r') as h5file: num_names_arrays = len([ attr for attr in h5file['model_weights'].attrs if attr.startswith('layer_names') ]) # The chunking of layer names array should have happened. self.assertGreater(num_names_arrays, 0) out2 = model.predict(x) self.assertAllClose(out, out2, atol=1e-05) # Cleanup os.close(fd) os.remove(fname)
def test_model_saving_to_pre_created_h5py_file(self): if h5py is None: self.skipTest('h5py required to run this test') with self.test_session(): inputs = keras.Input(shape=(3,)) x = keras.layers.Dense(2)(inputs) outputs = keras.layers.Dense(3)(x) model = keras.Model(inputs, outputs) model.compile(loss=keras.losses.MSE, optimizer=keras.optimizers.Adam(), metrics=[keras.metrics.categorical_accuracy]) x = np.random.random((1, 3)) y = np.random.random((1, 3)) model.train_on_batch(x, y) out = model.predict(x) fd, fname = tempfile.mkstemp('.h5') with h5py.File(fname, mode='r+') as h5file: keras.models.save_model(model, h5file) loaded_model = keras.models.load_model(h5file) out2 = loaded_model.predict(x) self.assertAllClose(out, out2, atol=1e-05) # Test non-default options in h5 with h5py.File('_', driver='core', backing_store=False) as h5file: keras.models.save_model(model, h5file) loaded_model = keras.models.load_model(h5file) out2 = loaded_model.predict(x) self.assertAllClose(out, out2, atol=1e-05) # Cleanup os.close(fd) os.remove(fname)
def test_stateful_metrics(self): np.random.seed(1334) class BinaryTruePositives(keras.layers.Layer): """Stateful Metric to count the total true positives over all batches. Assumes predictions and targets of shape `(samples, 1)`. Arguments: threshold: Float, lower limit on prediction value that counts as a positive class prediction. name: String, name for the metric. """ def __init__(self, name='true_positives', **kwargs): super(BinaryTruePositives, self).__init__(name=name, **kwargs) self.true_positives = keras.backend.variable(value=0, dtype='int32') def reset_states(self): keras.backend.set_value(self.true_positives, 0) def __call__(self, y_true, y_pred): """Computes the number of true positives in a batch. Args: y_true: Tensor, batch_wise labels y_pred: Tensor, batch_wise predictions Returns: The total number of true positives seen this epoch at the completion of the batch. """ y_true = keras.backend.cast(y_true, 'int32') y_pred = keras.backend.cast(keras.backend.round(y_pred), 'int32') correct_preds = keras.backend.cast( keras.backend.equal(y_pred, y_true), 'int32') true_pos = keras.backend.cast( keras.backend.sum(correct_preds * y_true), 'int32') current_true_pos = self.true_positives * 1 self.add_update(keras.backend.update_add( self.true_positives, true_pos), inputs=[y_true, y_pred]) return current_true_pos + true_pos metric_fn = BinaryTruePositives() config = keras.metrics.serialize(metric_fn) metric_fn = keras.metrics.deserialize( config, custom_objects={'BinaryTruePositives': BinaryTruePositives}) # Test on simple model inputs = keras.Input(shape=(2, )) outputs = keras.layers.Dense(1, activation='sigmoid')(inputs) model = keras.Model(inputs, outputs) model.compile(optimizer='sgd', loss='binary_crossentropy', metrics=['acc', metric_fn]) # Test fit, evaluate samples = 1000 x = np.random.random((samples, 2)) y = np.random.randint(2, size=(samples, 1)) model.fit(x, y, epochs=1, batch_size=10) outs = model.evaluate(x, y, batch_size=10) preds = model.predict(x) def ref_true_pos(y_true, y_pred): return np.sum(np.logical_and(y_pred > 0.5, y_true == 1)) # Test correctness (e.g. updates should have been run) self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5)
def test_stateful_metrics(self): with self.test_session(): np.random.seed(1334) class BinaryTruePositives(keras.layers.Layer): """Stateful Metric to count the total true positives over all batches. Assumes predictions and targets of shape `(samples, 1)`. Arguments: threshold: Float, lower limit on prediction value that counts as a positive class prediction. name: String, name for the metric. """ def __init__(self, name='true_positives', **kwargs): super(BinaryTruePositives, self).__init__(name=name, **kwargs) self.true_positives = keras.backend.variable(value=0, dtype='int32') self.stateful = True def reset_states(self): keras.backend.set_value(self.true_positives, 0) def __call__(self, y_true, y_pred): """Computes the number of true positives in a batch. Args: y_true: Tensor, batch_wise labels y_pred: Tensor, batch_wise predictions Returns: The total number of true positives seen this epoch at the completion of the batch. """ y_true = math_ops.cast(y_true, 'int32') y_pred = math_ops.cast(math_ops.round(y_pred), 'int32') correct_preds = math_ops.cast(math_ops.equal(y_pred, y_true), 'int32') true_pos = math_ops.cast( math_ops.reduce_sum(correct_preds * y_true), 'int32') current_true_pos = self.true_positives * 1 self.add_update( state_ops.assign_add(self.true_positives, true_pos), inputs=[y_true, y_pred]) return current_true_pos + true_pos metric_fn = BinaryTruePositives() config = keras.metrics.serialize(metric_fn) metric_fn = keras.metrics.deserialize( config, custom_objects={'BinaryTruePositives': BinaryTruePositives}) # Test on simple model inputs = keras.Input(shape=(2,)) outputs = keras.layers.Dense(1, activation='sigmoid')(inputs) model = keras.Model(inputs, outputs) model.compile(optimizer='sgd', loss='binary_crossentropy', metrics=['acc', metric_fn]) # Test fit, evaluate samples = 100 x = np.random.random((samples, 2)) y = np.random.randint(2, size=(samples, 1)) val_samples = 10 val_x = np.random.random((val_samples, 2)) val_y = np.random.randint(2, size=(val_samples, 1)) history = model.fit(x, y, epochs=1, batch_size=10, validation_data=(val_x, val_y)) outs = model.evaluate(x, y, batch_size=10) preds = model.predict(x) def ref_true_pos(y_true, y_pred): return np.sum(np.logical_and(y_pred > 0.5, y_true == 1)) # Test correctness (e.g. updates should have been run) self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5) # Test correctness of the validation metric computation val_preds = model.predict(val_x) val_outs = model.evaluate(val_x, val_y, batch_size=10) self.assertAllClose( val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5) self.assertAllClose( val_outs[2], history.history['val_true_positives'][-1], atol=1e-5) # Test with generators gen = [(np.array([x0]), np.array([y0])) for x0, y0 in zip(x, y)] val_gen = [(np.array([x0]), np.array([y0])) for x0, y0 in zip(val_x, val_y)] history = model.fit_generator(iter(gen), epochs=1, steps_per_epoch=samples, validation_data=iter(val_gen), validation_steps=val_samples) outs = model.evaluate_generator(iter(gen), steps=samples) preds = model.predict_generator(iter(gen), steps=samples) # Test correctness of the metric results self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5) # Test correctness of the validation metric computation val_preds = model.predict_generator(iter(val_gen), steps=val_samples) val_outs = model.evaluate_generator(iter(val_gen), steps=val_samples) self.assertAllClose( val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5) self.assertAllClose( val_outs[2], history.history['val_true_positives'][-1], atol=1e-5)