def test_wide_deep_model_with_sub_model_trained(self): linear_model = linear.LinearModel(units=1) dnn_model = sequential.Sequential([core.Dense(units=1, input_dim=3)]) wide_deep_model = wide_deep.WideDeepModel( linear.LinearModel(units=1), sequential.Sequential([core.Dense(units=1, input_dim=3)])) linear_inp = np.random.uniform(low=-5, high=5, size=(64, 2)) dnn_inp = np.random.uniform(low=-5, high=5, size=(64, 3)) inputs = [linear_inp, dnn_inp] output = .3 * linear_inp[:, 0] + .2 * dnn_inp[:, 1] linear_model.compile(optimizer='sgd', loss='mse', metrics=[], run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) dnn_model.compile(optimizer='adam', loss='mse', metrics=[], run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) linear_model.fit(linear_inp, output, epochs=50) dnn_model.fit(dnn_inp, output, epochs=50) wide_deep_model.compile(optimizer=['sgd', 'adam'], loss='mse', metrics=[], run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) wide_deep_model.fit(inputs, output, epochs=50)
def test_wide_deep_model_with_multi_outputs(self): with context.eager_mode(): inp = input_layer.Input(shape=(1, ), name='linear') l = linear.LinearModel(units=2, use_bias=False)(inp) l1, l2 = array_ops.split(l, num_or_size_splits=2, axis=1) linear_model = training.Model(inp, [l1, l2]) linear_model.set_weights([np.asarray([[0.5, 0.3]])]) h = core.Dense(units=2, use_bias=False)(inp) h1, h2 = array_ops.split(h, num_or_size_splits=2, axis=1) dnn_model = training.Model(inp, [h1, h2]) dnn_model.set_weights([np.asarray([[0.1, -0.5]])]) wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model) inp_np = np.asarray([[1.]]) out1, out2 = wide_deep_model(inp_np) # output should be (0.5 + 0.1), and (0.3 - 0.5) self.assertAllClose([[0.6]], out1) self.assertAllClose([[-0.2]], out2) wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model, activation='relu') out1, out2 = wide_deep_model(inp_np) # output should be relu((0.5 + 0.1)), and relu((0.3 - 0.5)) self.assertAllClose([[0.6]], out1) self.assertAllClose([[0.]], out2)
def test_wide_deep_model_as_layer(self): linear_model = linear.LinearModel(units=1) dnn_model = sequential.Sequential([core.Dense(units=1)]) linear_input = input_layer.Input(shape=(3, ), name='linear') dnn_input = input_layer.Input(shape=(5, ), name='dnn') wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model) wide_deep_output = wide_deep_model((linear_input, dnn_input)) input_b = input_layer.Input(shape=(1, ), name='b') output_b = core.Dense(units=1)(input_b) model = training.Model(inputs=[linear_input, dnn_input, input_b], outputs=[wide_deep_output + output_b]) linear_input_np = np.random.uniform(low=-5, high=5, size=(64, 3)) dnn_input_np = np.random.uniform(low=-5, high=5, size=(64, 5)) input_b_np = np.random.uniform(low=-5, high=5, size=(64, )) output_np = linear_input_np[:, 0] + .2 * dnn_input_np[:, 1] + input_b_np model.compile(optimizer='sgd', loss='mse', metrics=[], run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) model.fit([linear_input_np, dnn_input_np, input_b_np], output_np, epochs=5)
def test_wide_deep_model_with_two_feature_columns(self): vocab_list = ['alpha', 'beta', 'gamma'] vocab_val = [0.4, 0.6, 0.9] data = np.random.choice(vocab_list, size=256) y = np.zeros_like(data, dtype=np.float32) for vocab, val in zip(vocab_list, vocab_val): indices = np.where(data == vocab) y[indices] = val + np.random.uniform( low=-0.01, high=0.01, size=indices[0].shape) cat_column = fc.categorical_column_with_vocabulary_list( key='symbol', vocabulary_list=vocab_list) ind_column = fc.indicator_column(cat_column) emb_column = fc.embedding_column(cat_column, dimension=5) linear_feature_layer = dense_features_v2.DenseFeatures([ind_column]) linear_model = linear.LinearModel(use_bias=False, kernel_initializer='zeros') combined_linear = sequential.Sequential( [linear_feature_layer, linear_model]) dnn_model = sequential.Sequential([core.Dense(units=1)]) dnn_feature_layer = dense_features_v2.DenseFeatures([emb_column]) combined_dnn = sequential.Sequential([dnn_feature_layer, dnn_model]) wide_deep_model = wide_deep.WideDeepModel(combined_linear, combined_dnn) opt = gradient_descent.SGD(learning_rate=0.1) wide_deep_model.compile(opt, 'mse', [], run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) wide_deep_model.fit(x={'symbol': data}, y=y, batch_size=32, epochs=10) self.assertEqual(3, linear_model.inputs[0].shape[1]) self.assertEqual(5, dnn_model.inputs[0].shape[1])
def test_linear_model_with_feature_column(self): with context.eager_mode(): vocab_list = ['alpha', 'beta', 'gamma'] vocab_val = [0.4, 0.6, 0.9] data = np.random.choice(vocab_list, size=256) y = np.zeros_like(data, dtype=np.float32) for vocab, val in zip(vocab_list, vocab_val): indices = np.where(data == vocab) y[indices] = val + np.random.uniform( low=-0.01, high=0.01, size=indices[0].shape) cat_column = fc.categorical_column_with_vocabulary_list( key='symbol', vocabulary_list=vocab_list) ind_column = fc.indicator_column(cat_column) dense_feature_layer = dense_features_v2.DenseFeatures([ind_column]) linear_model = linear.LinearModel(use_bias=False, kernel_initializer='zeros') combined = sequential.Sequential( [dense_feature_layer, linear_model]) opt = gradient_descent.SGD(learning_rate=0.1) combined.compile(opt, 'mse', []) combined.fit(x={'symbol': data}, y=y, batch_size=32, epochs=10) self.assertAllClose( [[0.4], [0.6], [0.9]], combined.layers[1].dense_layers[0].kernel.numpy(), atol=0.01)
def test_wide_deep_model_backprop(self): with self.cached_session(): linear_model = linear.LinearModel(units=1, kernel_initializer='zeros') dnn_model = sequential.Sequential( [core.Dense(units=1, kernel_initializer='zeros')]) wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model) linear_inp = np.array([1.]) dnn_inp = np.array([1.]) inputs = [linear_inp, dnn_inp] output = linear_inp + 2 * dnn_inp linear_opt = gradient_descent.SGD(learning_rate=.1) dnn_opt = gradient_descent.SGD(learning_rate=.3) wide_deep_model.compile( optimizer=[linear_opt, dnn_opt], loss='mse', metrics=[], run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) self.evaluate(variables.global_variables_initializer()) wide_deep_model.fit(inputs, output, epochs=1) self.assertAllClose( [[0.3]], self.evaluate( wide_deep_model.linear_model.dense_layers[0].kernel)) self.assertAllClose( [[0.9]], self.evaluate(wide_deep_model.dnn_model.layers[0].kernel))
def test_linear_model_with_single_input(self): model = linear.LinearModel() inp = np.random.uniform(low=-5, high=5, size=(64, 2)) output = .3 * inp[:, 0] + .2 * inp[:, 1] model.compile('sgd', 'mse', []) model.fit(inp, output, epochs=5) self.assertTrue(model.built)
def test_linear_model_with_multi_input(self): model = linear.LinearModel() input_a = np.random.uniform(low=-5, high=5, size=(64, 1)) input_b = np.random.uniform(low=-5, high=5, size=(64, 1)) output = .3 * input_a + .2 * input_b model.compile('sgd', 'mse', []) model.fit([input_a, input_b], output, epochs=5)
def test_train_premade_widedeep_model_with_feature_layers(self): vocab_list = ['alpha', 'beta', 'gamma'] vocab_val = [0.4, 0.6, 0.9] data = np.random.choice(vocab_list, size=256) y = np.zeros_like(data, dtype=np.float32) for vocab, val in zip(vocab_list, vocab_val): indices = np.where(data == vocab) y[indices] = val + np.random.uniform( low=-0.01, high=0.01, size=indices[0].shape) cat_column = tf.feature_column.categorical_column_with_vocabulary_list( key='symbol', vocabulary_list=vocab_list) ind_column = tf.feature_column.indicator_column(cat_column) # TODO(tanzheny): use emb column for dense part once b/139667019 is fixed. # emb_column = feature_column.embedding_column(cat_column, dimension=5) keras_input = keras.layers.Input(name='symbol', shape=3, dtype=tf.dtypes.string) # build linear part with feature layer. linear_feature_layer = dense_features.DenseFeatures([ind_column]) linear_model = linear.LinearModel(units=1, name='Linear', kernel_initializer='zeros') combined_linear = keras.Sequential( [linear_feature_layer, linear_model]) # build dnn part with feature layer. dnn_feature_layer = dense_features.DenseFeatures([ind_column]) dense_layer = keras.layers.Dense(units=1, name='DNNDense', kernel_initializer='zeros') combined_dnn = keras.Sequential([dnn_feature_layer, dense_layer]) # build and compile wide deep. wide_deep_model = wide_deep.WideDeepModel(combined_linear, combined_dnn) wide_deep_model._set_inputs({'symbol': keras_input}) sgd_opt = gradient_descent.SGD(0.1) adam_opt = adam.Adam(0.1) wide_deep_model.compile([sgd_opt, adam_opt], 'mse', ['mse']) # build estimator. train_input_fn = numpy_io.numpy_input_fn(x={'symbol': data}, y=y, num_epochs=20, shuffle=False) eval_input_fn = numpy_io.numpy_input_fn(x={'symbol': data}, y=y, num_epochs=20, shuffle=False) est = keras_lib.model_to_estimator(keras_model=wide_deep_model, config=self._config, checkpoint_format='saver') before_eval_results = est.evaluate(input_fn=eval_input_fn, steps=1) est.train(input_fn=train_input_fn, steps=20) after_eval_results = est.evaluate(input_fn=eval_input_fn, steps=1) self.assertLess(after_eval_results['loss'], before_eval_results['loss']) self.assertLess(after_eval_results['loss'], 0.1)
def test_config(self): linear_model = linear.LinearModel(units=1) dnn_model = sequential.Sequential([core.Dense(units=1, input_dim=3)]) wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model) config = wide_deep_model.get_config() cloned_wide_deep_model = wide_deep.WideDeepModel.from_config(config) self.assertEqual(linear_model.units, cloned_wide_deep_model.linear_model.units) self.assertEqual(dnn_model.layers[0].units, cloned_wide_deep_model.dnn_model.layers[0].units)
def test_linear_model_with_mismatched_dict_inputs(self): model = linear.LinearModel() input_a = np.random.uniform(low=-5, high=5, size=(64, 1)) input_b = np.random.uniform(low=-5, high=5, size=(64, 1)) output = .3 * input_a + .2 * input_b model.compile('sgd', 'mse', []) model.build({'a': tensor_shape.TensorShape([None, 1]), 'b': tensor_shape.TensorShape([None, 1])}) with self.assertRaisesRegex(ValueError, 'Missing keys'): model.fit({'c': input_a, 'b': input_b}, output, epochs=5)
def test_linear_model(self, distribution, data_fn): with distribution.scope(): model = linear.LinearModel() opt = gradient_descent.SGD(learning_rate=0.1) model.compile(opt, 'mse') if data_fn == get_numpy: inputs, output = get_numpy() hist = model.fit(inputs, output, epochs=5) else: hist = model.fit(get_dataset(), epochs=5) self.assertLess(hist.history['loss'][4], 0.2)
def test_wide_deep_model_with_single_input(self): linear_model = linear.LinearModel(units=1) dnn_model = sequential.Sequential([core.Dense(units=1, input_dim=3)]) wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model) inputs = np.random.uniform(low=-5, high=5, size=(64, 3)) output = .3 * inputs[:, 0] wide_deep_model.compile(optimizer=['sgd', 'adam'], loss='mse', metrics=[], run_eagerly=testing_utils.should_run_eagerly()) wide_deep_model.fit(inputs, output, epochs=5)
def test_linear_model_as_layer(self): input_a = input_layer.Input(shape=(1, ), name='a') output_a = linear.LinearModel()(input_a) input_b = input_layer.Input(shape=(1, ), name='b') output_b = core.Dense(units=1)(input_b) output = output_a + output_b model = training.Model(inputs=[input_a, input_b], outputs=[output]) input_a_np = np.random.uniform(low=-5, high=5, size=(64, 1)) input_b_np = np.random.uniform(low=-5, high=5, size=(64, 1)) output_np = .3 * input_a_np + .2 * input_b_np model.compile('sgd', 'mse', []) model.fit([input_a_np, input_b_np], output_np, epochs=5)
def test_config_with_custom_objects(self): def my_activation(x): return x linear_model = linear.LinearModel(units=1) dnn_model = sequential.Sequential([core.Dense(units=1, input_dim=3)]) wide_deep_model = wide_deep.WideDeepModel( linear_model, dnn_model, activation=my_activation) config = wide_deep_model.get_config() cloned_wide_deep_model = wide_deep.WideDeepModel.from_config( config, custom_objects={'my_activation': my_activation}) self.assertEqual(cloned_wide_deep_model.activation, my_activation)
def test_wide_deep_model(self): linear_model = linear.LinearModel(units=1) dnn_model = sequential.Sequential([core.Dense(units=1, input_dim=3)]) wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model) linear_inp = np.random.uniform(low=-5, high=5, size=(64, 2)) dnn_inp = np.random.uniform(low=-5, high=5, size=(64, 3)) inputs = [linear_inp, dnn_inp] output = .3 * linear_inp[:, 0] + .2 * dnn_inp[:, 1] wide_deep_model.compile(optimizer=['sgd', 'adam'], loss='mse', metrics=[], run_eagerly=testing_utils.should_run_eagerly()) wide_deep_model.fit(inputs, output, epochs=5) self.assertTrue(wide_deep_model.built)
def test_wide_deep_model(self, distribution, data_fn): with distribution.scope(): linear_model = linear.LinearModel(units=1) dnn_model = sequential.Sequential([core.Dense(units=1)]) wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model) linear_opt = gradient_descent.SGD(learning_rate=0.05) dnn_opt = adagrad.Adagrad(learning_rate=0.1) wide_deep_model.compile(optimizer=[linear_opt, dnn_opt], loss='mse') if data_fn == 'numpy': inputs, output = get_numpy() hist = wide_deep_model.fit(inputs, output, epochs=5) else: hist = wide_deep_model.fit(get_dataset(), epochs=5) self.assertLess(hist.history['loss'][4], 0.2)
def test_linear_model_with_sparse_input(self): indices = constant_op.constant([[0, 0], [0, 2], [1, 0], [1, 1]], dtype=dtypes.int64) values = constant_op.constant([.4, .6, .8, .5]) shape = constant_op.constant([2, 3], dtype=dtypes.int64) model = linear.LinearModel() inp = sparse_tensor.SparseTensor(indices, values, shape) output = model(inp) self.evaluate(variables.global_variables_initializer()) if context.executing_eagerly(): weights = model.get_weights() weights[0] = np.ones((3, 1)) model.set_weights(weights) output = model(inp) self.assertAllClose([[1.], [1.3]], self.evaluate(output))
def test_train_premade_linear_model(self): (x_train, y_train ), _, train_inp_fn, eval_inp_fn = get_resource_for_simple_model() linear_model = linear.LinearModel(units=1) opt = gradient_descent.SGD(0.1) linear_model.compile(opt, 'mse', ['mse']) linear_model.fit(x_train, y_train, epochs=10) est = keras_lib.model_to_estimator(keras_model=linear_model, config=self._config, checkpoint_format='saver') before_eval_results = est.evaluate(input_fn=eval_inp_fn, steps=1) est.train(input_fn=train_inp_fn, steps=500) after_eval_results = est.evaluate(input_fn=eval_inp_fn, steps=1) self.assertLess(after_eval_results['loss'], before_eval_results['loss']) self.assertLess(after_eval_results['loss'], 0.1)
def test_linear_model_with_sparse_input_and_custom_training(self): batch_size = 64 indices = [] values = [] target = np.zeros((batch_size, 1)) with context.eager_mode(): for i in range(64): rand_int = np.random.randint(3) if rand_int == 0: indices.append((i, 0)) val = np.random.uniform(low=-5, high=5) values.append(val) target[i] = 0.3 * val elif rand_int == 1: indices.append((i, 1)) val = np.random.uniform(low=-5, high=5) values.append(val) target[i] = 0.2 * val else: indices.append((i, 0)) indices.append((i, 1)) val_1 = np.random.uniform(low=-5, high=5) val_2 = np.random.uniform(low=-5, high=5) values.append(val_1) values.append(val_2) target[i] = 0.3 * val_1 + 0.2 * val_2 indices = np.asarray(indices) values = np.asarray(values) shape = constant_op.constant([batch_size, 2], dtype=dtypes.int64) inp = sparse_tensor.SparseTensor(indices, values, shape) model = linear.LinearModel(use_bias=False) opt = gradient_descent.SGD() for _ in range(20): with backprop.GradientTape() as t: output = model(inp) loss = backend.mean( losses.mean_squared_error(target, output)) grads = t.gradient(loss, model.trainable_variables) grads_and_vars = zip(grads, model.trainable_variables) opt.apply_gradients(grads_and_vars)
def test_train_premade_linear_model_with_dense_features(self): vocab_list = ['alpha', 'beta', 'gamma'] vocab_val = [0.4, 0.6, 0.9] data = np.random.choice(vocab_list, size=256) y = np.zeros_like(data, dtype=np.float32) for vocab, val in zip(vocab_list, vocab_val): indices = np.where(data == vocab) y[indices] = val + np.random.uniform( low=-0.01, high=0.01, size=indices[0].shape) cat_column = tf.feature_column.categorical_column_with_vocabulary_list( key='symbol', vocabulary_list=vocab_list) ind_column = tf.feature_column.indicator_column(cat_column) keras_input = keras.layers.Input(name='symbol', shape=3, dtype=tf.dtypes.string) feature_layer = dense_features.DenseFeatures([ind_column]) h = feature_layer({'symbol': keras_input}) linear_model = linear.LinearModel(units=1) h = linear_model(h) model = keras.Model(inputs=keras_input, outputs=h) opt = gradient_descent.SGD(0.1) model.compile(opt, 'mse', ['mse']) train_input_fn = numpy_io.numpy_input_fn(x={'symbol': data}, y=y, num_epochs=20, shuffle=False) eval_input_fn = numpy_io.numpy_input_fn(x={'symbol': data}, y=y, num_epochs=20, shuffle=False) est = keras_lib.model_to_estimator(keras_model=model, config=self._config, checkpoint_format='saver') before_eval_results = est.evaluate(input_fn=eval_input_fn, steps=1) est.train(input_fn=train_input_fn, steps=30) after_eval_results = est.evaluate(input_fn=eval_input_fn, steps=1) self.assertLess(after_eval_results['loss'], before_eval_results['loss']) self.assertLess(after_eval_results['loss'], 0.05)
def test_wide_deep_model_with_single_feature_column(self): vocab_list = ['alpha', 'beta', 'gamma'] vocab_val = [0.4, 0.6, 0.9] data = np.random.choice(vocab_list, size=256) y = np.zeros_like(data, dtype=np.float32) for vocab, val in zip(vocab_list, vocab_val): indices = np.where(data == vocab) y[indices] = val + np.random.uniform( low=-0.01, high=0.01, size=indices[0].shape) cat_column = feature_column_v2.categorical_column_with_vocabulary_list( key='symbol', vocabulary_list=vocab_list) ind_column = feature_column_v2.indicator_column(cat_column) dense_feature_layer = dense_features_v2.DenseFeatures([ind_column]) linear_model = linear.LinearModel(use_bias=False, kernel_initializer='zeros') dnn_model = keras.Sequential([keras.layers.Dense(units=1)]) wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model) combined = keras.Sequential([dense_feature_layer, wide_deep_model]) opt = gradient_descent.SGD(learning_rate=0.1) combined.compile(opt, 'mse', [], run_eagerly=testing_utils.should_run_eagerly()) combined.fit(x={'symbol': data}, y=y, batch_size=32, epochs=10)
def test_config(self): linear_model = linear.LinearModel(units=3, use_bias=True) config = linear_model.get_config() cloned_linear_model = linear.LinearModel.from_config(config) self.assertEqual(linear_model.units, cloned_linear_model.units)
def test_linear_model_with_int_input(self): inp = input_layer.Input(shape=(1, ), dtype=dtypes.int32) with self.assertRaisesRegexp(TypeError, 'Unable to build'): linear.LinearModel()(inp)