def test_phases_end_to_end(self):
train_dataset, test_dataset = testing_utils.get_test_data(
train_samples=128,
test_samples=64,
input_shape=(10,),
num_classes=10,
random_seed=42)
# TODO: Consider performing `tf.data.Dataset` transformations
# within get_test_data function.
train_dataset = train_dataset.batch(32)
test_dataset = test_dataset.batch(32)
model1 = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(10),
])
model1.compile(
optimizer=tf.keras.optimizers.Adam(0.01), loss='mse', metrics=['mae'])
model2 = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(10),
])
model2.compile(
optimizer=tf.keras.optimizers.Adam(0.01), loss='mse', metrics=['mae'])
ensemble = MeanEnsemble(submodels=[model1, model2])
ensemble.compile(
optimizer=tf.keras.optimizers.Adam(0.01), loss='mse', metrics=['mae'])
controller = SequentialController(phases=[
TrainKerasModelsPhase([
model1,
model2,
], dataset=train_dataset),
TrainKerasModelsPhase([ensemble], dataset=train_dataset),
])
train_dataset, test_dataset = testing_utils.get_test_data(
train_samples=128,
test_samples=64,
input_shape=(10,),
num_classes=2,
random_seed=42)
model_search = ModelSearch(controller)
model_search.run()
self.assertIsInstance(
model_search.get_best_models(num_models=1)[0], MeanEnsemble)
def test_adanet_controller_end_to_end(self):
train_dataset, test_dataset = testing_utils.get_test_data(
train_samples=1280,
test_samples=640,
input_shape=(10, ),
num_classes=10,
random_seed=42)
train_dataset = train_dataset.batch(32)
test_dataset = test_dataset.batch(32)
candidate_phase = AdaNetCandidatePhase(
train_dataset,
candidates_per_iteration=2,
optimizer='adam',
loss='sparse_categorical_crossentropy',
output_units=10)
# TODO: Setting candidates_per_iteration greater than the one
# for the candidate phase will lead to unexpected behavior.
ensemble_phase = AdaNetEnsemblePhase(
train_dataset,
candidates_per_iteration=2,
optimizer='adam',
loss='sparse_categorical_crossentropy')
adanet_controller = AdaNetController(candidate_phase,
ensemble_phase,
iterations=5)
model_search = ModelSearch(adanet_controller)
model_search.run()
self.assertIsInstance(
model_search.get_best_models(num_models=1)[0], MeanEnsemble)
def test_tuner_end_to_end(self):
train_dataset, test_dataset = testing_utils.get_test_data(
train_samples=128,
test_samples=64,
input_shape=(10, ),
num_classes=10,
random_seed=42)
# TODO: Consider performing `tf.data.Dataset` transformations
# within get_test_data function.
train_dataset = train_dataset.batch(32)
test_dataset = test_dataset.batch(32)
def build_model(hp):
model = tf.keras.Sequential()
model.add(
tf.keras.layers.Dense(units=hp.Int('units',
min_value=32,
max_value=512,
step=32),
activation='relu'))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
model.compile(optimizer=tf.keras.optimizers.Adam(
hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model
# Define phases.
tuner = tuners.RandomSearch(build_model,
objective='val_accuracy',
max_trials=3,
executions_per_trial=1,
directory=self.test_subdirectory,
project_name='helloworld')
tuner_phase = KerasTunerPhase(tuner,
train_dataset,
validation_data=test_dataset)
def build_ensemble():
ensemble = MeanEnsemble(submodels=tuner_phase.get_best_models(
num_models=2))
ensemble.compile(optimizer=tf.keras.optimizers.Adam(0.01),
loss='mse',
metrics=['mae'])
return [ensemble]
ensemble_phase = TrainKerasModelsPhase(build_ensemble,
dataset=train_dataset)
controller = SequentialController(phases=[tuner_phase, ensemble_phase])
# Execute phases.
model_search = ModelSearch(controller)
model_search.run()
self.assertIsInstance(
model_search.get_best_models(num_models=1)[0], MeanEnsemble)
def test_lifecycle(self, ensemble, want_results, output_units=None):
train_dataset, test_dataset = testing_utils.get_test_data(
train_samples=128,
test_samples=64,
input_shape=(10, ),
num_classes=2,
random_seed=42)
# TODO: Consider performing `tf.data.Dataset` transformations
# within get_test_data function.
train_dataset = train_dataset.batch(32).repeat(10)
test_dataset = test_dataset.batch(32).repeat(10)
model1 = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(2),
])
model1.compile(optimizer=tf.keras.optimizers.Adam(0.01), loss='mse')
model1.fit(train_dataset)
model1.trainable = False # Since models inside ensemble should be trained.
model1_pre_train_weights = model1.get_weights()
model2 = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(2),
])
model2.compile(optimizer=tf.keras.optimizers.Adam(0.01), loss='mse')
model2.fit(train_dataset)
model2.trainable = False # Since models inside ensemble should be trained.
model2_pre_train_weights = model2.get_weights()
if output_units:
ensemble = ensemble(submodels=[model1, model2],
output_units=output_units)
else:
ensemble = ensemble(submodels=[model1, model2])
ensemble.compile(optimizer=tf.keras.optimizers.Adam(0.01),
loss='mse',
metrics=['mae'])
ensemble.fit(train_dataset)
# Make sure submodel weights were not altered during ensemble training.
model1_post_train_weights = model1.get_weights()
model2_post_train_weights = model2.get_weights()
self.assertAllClose(model1_pre_train_weights,
model1_post_train_weights)
self.assertAllClose(model2_pre_train_weights,
model2_post_train_weights)
eval_results = ensemble.evaluate(test_dataset)
self.assertAllClose(eval_results, want_results)
def test_phases_end_to_end(self):
train_dataset, test_dataset = testing_utils.get_test_data(
train_samples=128,
test_samples=64,
input_shape=(10,),
num_classes=10,
random_seed=42)
# TODO: Consider performing `tf.data.Dataset` transformations
# within get_test_data function.
train_dataset = train_dataset.batch(32)
test_dataset = test_dataset.batch(32)
model1 = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(10),
])
model1.compile(
optimizer=tf.keras.optimizers.Adam(0.01), loss='mse', metrics=['mae'])
model2 = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(10),
])
model2.compile(
optimizer=tf.keras.optimizers.Adam(0.01), loss='mse', metrics=['mae'])
# TODO: This test could potentially have the best model be
# a non-ensemble Keras model. Therefore, need to address this issue and
# remove the freeze_submodels flag.
ensemble = MeanEnsemble(submodels=[model1, model2], freeze_submodels=False)
ensemble.compile(
optimizer=tf.keras.optimizers.Adam(0.01), loss='mse', metrics=['mae'])
controller = SequentialController(phases=[
InputPhase(train_dataset, test_dataset),
KerasTrainerPhase([model1, model2]),
KerasTrainerPhase([ensemble]),
])
model_search = ModelSearch(controller)
model_search.run()
self.assertIsInstance(
model_search.get_best_models(num_models=1)[0], MeanEnsemble)
