def tuneHP(self, hyperModel, X_train, X_test, y_train, y_test, tuner_epochs=50, tuner_batch_size=10000, tuner_mode=0):
     if tuner_mode == 0:
         tuner = kt.Hyperband(hyperModel,
                              objective=kt.Objective("auc", direction="max"),  # ['loss', 'auc', 'accuracy', 'val_loss', 'val_auc', 'val_accuracy']
                              max_epochs=200,
                              hyperband_iterations=3,
                              factor=3,
                              seed=seed_value,
                              directory='tuning',
                              project_name='model_hyperband_1',
                              overwrite=True)
     elif tuner_mode == 1:
         tuner = kt.BayesianOptimization(hyperModel,
                                         objective='val_loss',
                                         max_trials=100,
                                         seed=seed_value,
                                         directory='tuning',
                                         project_name='model_bayesian_1',
                                         overwrite=True)
     elif tuner_mode == 2:
         tuner = kt.RandomSearch(hyperModel,
                                 objective='val_loss',
                                 max_trials=1000,
                                 seed=seed_value,
                                 directory='tuning',
                                 project_name='model_random_1',
                                 overwrite=True)
     else:
         raise ValueError('Invalid tuner mode')
     tuner.search(X_train, y_train, epochs=tuner_epochs, batch_size=tuner_batch_size, validation_data=(X_test, y_test), verbose=0)
     # tuner.search(X_train, y_train, epochs=tuner_epochs, validation_data=(X_test, y_test), verbose=1)
     best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
     print(tuner.search_space_summary())
     return best_hps, tuner
Esempio n. 2
0
    def fit(self, x=None, y=None, validation_data=None, **kwargs):
        # Initialize HyperGraph model
        x = layer_utils.format_inputs(x, 'train_x')
        y = layer_utils.format_inputs(y, 'train_y')

        # TODO: Set the shapes only if they are not provided by the user when
        #  initiating the HyperHead or Block.
        for x_input, input_node in zip(x, self.inputs):
            input_node.shape = x_input.shape[1:]
        for y_input, output_node in zip(y, self.outputs):
            if len(y_input.shape) == 1:
                y_input = np.reshape(y_input, y_input.shape + (1, ))
            output_node.shape = y_input.shape[1:]
            output_node.in_hypermodels[0].output_shape = output_node.shape

        # Prepare the dataset
        if validation_data is None:
            (x, y), (x_val, y_val) = layer_utils.split_train_to_valid(x, y)
            validation_data = x_val, y_val

        self.tuner = kerastuner.RandomSearch(hypermodel=self,
                                             objective='val_loss',
                                             max_trials=self.max_trials,
                                             directory=self.directory)

        # TODO: allow early stop if epochs is not specified.
        self.tuner.search(x=x, y=y, validation_data=validation_data, **kwargs)
Esempio n. 3
0
def tuner_fn(working_dir: Text, train_data_pattern: Text,
             eval_data_pattern: Text,
             schema: schema_pb2.Schema) -> component.TunerFnResult:
    """Build the tuner using the Keras Tuner API.

  Args:
    working_dir: working dir for KerasTuner.
    train_data_pattern: file pattern of training tfrecord data.
    eval_data_pattern: file pattern of eval tfrecord data.
    schema: Schema of the input data.

  Returns:
    A namedtuple contains the following:
      - tuner: A KerasTuner that will be used for tuning.
      - train_dataset: A tf.data.Dataset of training data.
      - eval_dataset: A tf.data.Dataset of eval data.
  """
    hparams = kerastuner.HyperParameters()
    hparams.Choice('learning_rate', [1e-1, 1e-3])
    hparams.Int('num_layers', 1, 5)

    # TODO(jyzhao): support params, e.g., max_trials in user input config.
    tuner = kerastuner.RandomSearch(_build_keras_model,
                                    max_trials=5,
                                    hyperparameters=hparams,
                                    allow_new_entries=False,
                                    objective='val_accuracy',
                                    directory=working_dir,
                                    project_name='iris')

    return component.TunerFnResult(
        tuner=tuner,
        train_dataset=_make_input_dataset(train_data_pattern, schema, 10),
        eval_dataset=_make_input_dataset(eval_data_pattern, schema, 10))
Esempio n. 4
0
def tuner_builder_random():

    global _project_name

    tuner = kt.RandomSearch(hypermodel = builder , objective = "val_loss", max_trials=10, directory = f'./tuner_data/{_project_name}', )

    return tuner
Esempio n. 5
0
def build_search():
    # Load model
    hypermodel = HyperMNIST(10)
    tuner = kt.RandomSearch(hypermodel,
                            objective='val_acc',
                            max_trials=5,
                            executions_per_trial=3,
                            directory='./tuner',
                            project_name='mnist')

    tuner.search_space_summary()

    return tuner
Esempio n. 6
0
def tuner_fn(fn_args: FnArgs) -> TunerFnResult:
    """Build the tuner using the KerasTuner API.

  Args:
    fn_args: Holds args as name/value pairs.
      - working_dir: working dir for tuning.
      - train_files: List of file paths containing training tf.Example data.
      - eval_files: List of file paths containing eval tf.Example data.
      - train_steps: number of train steps.
      - eval_steps: number of eval steps.
      - schema_path: optional schema of the input data.
      - transform_graph_path: optional transform graph produced by TFT.

  Returns:
    A namedtuple contains the following:
      - tuner: A BaseTuner that will be used for tuning.
      - fit_kwargs: Args to pass to tuner's run_trial function for fitting the
                    model , e.g., the training and validation dataset. Required
                    args depend on the above tuner's implementation.
  """
    # RandomSearch is a subclass of kerastuner.Tuner which inherits from
    # BaseTuner.
    tuner = kerastuner.RandomSearch(_build_keras_model,
                                    max_trials=6,
                                    hyperparameters=_get_hyperparameters(),
                                    allow_new_entries=False,
                                    objective=kerastuner.Objective(
                                        'val_sparse_categorical_accuracy',
                                        'max'),
                                    directory=fn_args.working_dir,
                                    project_name='penguin_tuning')

    transform_graph = tft.TFTransformOutput(fn_args.transform_graph_path)

    train_dataset = _input_fn(fn_args.train_files,
                              fn_args.data_accessor,
                              transform_graph,
                              batch_size=_TRAIN_BATCH_SIZE)

    eval_dataset = _input_fn(fn_args.eval_files,
                             fn_args.data_accessor,
                             transform_graph,
                             batch_size=_EVAL_BATCH_SIZE)

    return TunerFnResult(tuner=tuner,
                         fit_kwargs={
                             'x': train_dataset,
                             'validation_data': eval_dataset,
                             'steps_per_epoch': fn_args.train_steps,
                             'validation_steps': fn_args.eval_steps
                         })
def test_tunable_false_hypermodel(tmp_dir):
    def build_model(hp):
        input_shape = (256, 256, 3)
        inputs = tf.keras.Input(shape=input_shape)

        with hp.name_scope('xception'):
            # Tune the pooling of Xception by supplying the search space
            # beforehand.
            hp.Choice('pooling', ['avg', 'max'])
            xception = kerastuner.applications.HyperXception(
                include_top=False,
                input_shape=input_shape,
                tunable=False).build(hp)
        x = xception(inputs)

        x = tf.keras.layers.Dense(
            hp.Int('hidden_units', 50, 100, step=10),
            activation='relu')(x)
        outputs = tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')(x)

        model = tf.keras.Model(inputs, outputs)

        optimizer = tf.keras.optimizers.get(hp.Choice('optimizer', ['adam', 'sgd']))
        optimizer.learning_rate = hp.Float(
            'learning_rate', 1e-4, 1e-2, sampling='log')

        model.compile(optimizer, loss='sparse_categorical_crossentropy')
        return model

    tuner = kerastuner.RandomSearch(
        objective='val_loss',
        hypermodel=build_model,
        max_trials=4,
        directory=tmp_dir)

    x = np.random.random(size=(2, 256, 256, 3))
    y = np.random.randint(0, NUM_CLASSES, size=(2,))

    tuner.search(x, y, validation_data=(x, y), batch_size=2)

    hps = tuner.oracle.get_space()
    assert 'xception/pooling' in hps
    assert 'hidden_units' in hps
    assert 'optimizer' in hps
    assert 'learning_rate' in hps

    # Make sure no HPs from building xception were added.
    assert len(hps.space) == 4
Esempio n. 8
0
def tuner_fn(fn_args: FnArgs) -> TunerFnResult:
  """Build the tuner using the KerasTuner API.

  Args:
    fn_args: Holds args as name/value pairs.
      - working_dir: working dir for tuning.
      - train_files: List of file paths containing training tf.Example data.
      - eval_files: List of file paths containing eval tf.Example data.
      - train_steps: number of train steps.
      - eval_steps: number of eval steps.
      - schema_path: optional schema of the input data.
      - transform_graph_path: optional transform graph produced by TFT.

  Returns:
    A namedtuple contains the following:
      - tuner: A BaseTuner that will be used for tuning.
      - fit_kwargs: Args to pass to tuner's run_trial function for fitting the
                    model , e.g., the training and validation dataset. Required
                    args depend on the above tuner's implementation.
  """
  hp = kerastuner.HyperParameters()
  # Defines search space.
  hp.Choice('learning_rate', [1e-1, 1e-3])
  hp.Int('num_layers', 1, 5)

  # RandomSearch is a subclass of Keras model Tuner.
  tuner = kerastuner.RandomSearch(
      _build_keras_model,
      max_trials=5,
      hyperparameters=hp,
      allow_new_entries=False,
      objective='val_sparse_categorical_accuracy',
      directory=fn_args.working_dir,
      project_name='test')

  schema = schema_pb2.Schema()
  io_utils.parse_pbtxt_file(fn_args.schema_path, schema)
  train_dataset = _input_fn(fn_args.train_files, schema)
  eval_dataset = _input_fn(fn_args.eval_files, schema)

  return TunerFnResult(
      tuner=tuner,
      fit_kwargs={
          'x': train_dataset,
          'validation_data': eval_dataset,
          'steps_per_epoch': fn_args.train_steps,
          'validation_steps': fn_args.eval_steps
      })
def test_tunable_false_hypermodel(tmp_dir):
    def build_model(hp):
        input_shape = (256, 256, 3)
        inputs = tf.keras.Input(shape=input_shape)

        with hp.name_scope("xception"):
            # Tune the pooling of Xception by supplying the search space
            # beforehand.
            hp.Choice("pooling", ["avg", "max"])
            xception = kerastuner.applications.HyperXception(
                include_top=False, input_shape=input_shape,
                tunable=False).build(hp)
        x = xception(inputs)

        x = tf.keras.layers.Dense(hp.Int("hidden_units", 50, 100, step=10),
                                  activation="relu")(x)
        outputs = tf.keras.layers.Dense(NUM_CLASSES, activation="softmax")(x)

        model = tf.keras.Model(inputs, outputs)

        optimizer = tf.keras.optimizers.get(
            hp.Choice("optimizer", ["adam", "sgd"]))
        optimizer.learning_rate = hp.Float("learning_rate",
                                           1e-4,
                                           1e-2,
                                           sampling="log")

        model.compile(optimizer, loss="sparse_categorical_crossentropy")
        return model

    tuner = kerastuner.RandomSearch(objective="val_loss",
                                    hypermodel=build_model,
                                    max_trials=4,
                                    directory=tmp_dir)

    x = np.random.random(size=(2, 256, 256, 3))
    y = np.random.randint(0, NUM_CLASSES, size=(2, ))

    tuner.search(x, y, validation_data=(x, y), batch_size=2)

    hps = tuner.oracle.get_space()
    assert "xception/pooling" in hps
    assert "hidden_units" in hps
    assert "optimizer" in hps
    assert "learning_rate" in hps

    # Make sure no HPs from building xception were added.
    assert len(hps.space) == 4
Esempio n. 10
0
def test_tuning_correctness(tmp_dir):
    tuner = kerastuner.RandomSearch(
        seed=1337,
        hypermodel=MockHyperModel(),
        max_trials=2,
        objective='loss',
        executions_per_trial=2,
        directory=tmp_dir,
    )
    tuner.search()
    assert len(tuner.trials) == 2

    m0_e0_epochs = [
        float(np.average(x)) for x in MockHyperModel.mode_0_execution_0
    ]
    m0_e1_epochs = [
        float(np.average(x)) for x in MockHyperModel.mode_0_execution_1
    ]
    m0_epochs = [(a + b) / 2 for a, b in zip(m0_e0_epochs, m0_e1_epochs)
                 ] + [m0_e1_epochs[-1]]
    m1_epochs = [float(np.average(x)) for x in MockHyperModel.mode_1_execution]

    # Score tracking correctness
    first_trial, second_trial = sorted(tuner.trials, key=lambda x: x.score)
    assert first_trial.score == min(m0_epochs)
    assert second_trial.score == min(m1_epochs)
    state = tuner.get_state()
    assert state['best_trial']['score'] == min(m0_epochs)
    assert tuner._get_best_trials(1)[0] == first_trial

    # Metrics tracking correctness
    assert len(first_trial.executions) == 2
    e0 = first_trial.executions[0]
    e1 = first_trial.executions[1]
    e0_per_epoch_metrics = e0.per_epoch_metrics.metrics_history['loss']
    e1_per_epoch_metrics = e1.per_epoch_metrics.metrics_history['loss']
    e0_per_batch_metrics = e0.per_batch_metrics.metrics_history['loss']
    e1_per_batch_metrics = e1.per_batch_metrics.metrics_history['loss']
    assert e0_per_epoch_metrics == m0_e0_epochs
    assert e0_per_batch_metrics == MockHyperModel.mode_0_execution_0[-1]
    assert e1_per_epoch_metrics == m0_e1_epochs
    assert e1_per_batch_metrics == MockHyperModel.mode_0_execution_1[-1]
    assert first_trial.averaged_metrics.metrics_history['loss'] == m0_epochs
def test_get_best_hyperparameters(tmp_dir):
    hp1 = kerastuner.HyperParameters()
    hp1.Fixed('a', 1)
    trial1 = kerastuner.engine.trial.Trial(hyperparameters=hp1)
    trial1.status = 'COMPLETED'
    trial1.score = 10

    hp2 = kerastuner.HyperParameters()
    hp2.Fixed('a', 2)
    trial2 = kerastuner.engine.trial.Trial(hyperparameters=hp2)
    trial2.status = 'COMPLETED'
    trial2.score = 9

    tuner = kerastuner.RandomSearch(objective='val_accuracy',
                                    hypermodel=build_model,
                                    max_trials=2,
                                    directory=tmp_dir)

    tuner.oracle.trials = {trial1.trial_id: trial1, trial2.trial_id: trial2}

    hps = tuner.get_best_hyperparameters()[0]
    assert hps['a'] == 1
Esempio n. 12
0
    def setUp(self):
        import Exercise06_03
        self.exercises = Exercise06_03

        self.file_url = 'https://raw.githubusercontent.com/PacktWorkshops/The-TensorFlow-Workshop/master/Chapter06/dataset/connect-4.csv'

        self.data = pd.read_csv(self.file_url)
        self.target = self.data.pop('class')

        self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
            self.data, self.target, test_size=0.2, random_state=42)

        np.random.seed(8)
        tf.random.set_seed(8)

        tuner = kt.RandomSearch(model_builder,
                                objective='val_accuracy',
                                max_trials=10)
        tuner.search(self.X_train,
                     self.y_train,
                     validation_data=(self.X_test, self.y_test))
        best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
        self.best_l2 = best_hps.get('l2')
        self.model = tuner.hypermodel.build(best_hps)
Esempio n. 13
0
def tuner_fn(fn_args: FnArgs) -> TunerFnResult:
    """Build the tuner using CloudTuner (KerasTuner instance).
    Args:
      fn_args: Holds args used to train and tune the model as name/value pairs. See
        https://www.tensorflow.org/tfx/api_docs/python/tfx/components/trainer/fn_args_utils/FnArgs.
    Returns:
      A namedtuple contains the following:
        - tuner: A BaseTuner that will be used for tuning.
        - fit_kwargs: Args to pass to tuner's run_trial function for fitting the
                      model , e.g., the training and validation dataset. Required
                      args depend on the above tuner's implementation.
    """
    custom_config_dict = _get_custom_config_dict(fn_args)

    max_trials = custom_config_dict.get('max_trials', MAX_TRIALS)

    transform_graph = tft.TFTransformOutput(fn_args.transform_graph_path)

    # Construct a build_keras_model_fn that just takes hyperparams from get_hyperparameters as input.
    build_keras_model_fn = functools.partial(
        _build_keras_model, tf_transform_output=transform_graph)

    # CloudTuner is a subclass of kerastuner.Tuner which inherits from BaseTuner.
    #is_local_run = "custom_config" not in fn_args.custom_config
    is_local_run = custom_config_dict.get("is_local_run", True)
    absl.logging.info('is_local_run : %s' % is_local_run)
    if is_local_run:
        tuner = kerastuner.RandomSearch(build_keras_model_fn,
                                        max_trials=max_trials,
                                        hyperparameters=_get_hyperparameters(),
                                        allow_new_entries=False,
                                        objective=kerastuner.Objective(
                                            'val_binary_accuracy', 'max'),
                                        directory=fn_args.working_dir,
                                        project_name='titanic_tuning')
    else:
        tuner = CloudTuner(
            build_keras_model_fn,
            project_id=fn_args.custom_config['ai_platform_training_args']
            ['project'],
            region=fn_args.custom_config['ai_platform_training_args']
            ['region'],
            max_trials=max_trials,
            hyperparameters=_get_hyperparameters(),
            objective=kerastuner.Objective('val_binary_accuracy', 'max'),
            # objective=kerastuner.Objective('auc', 'min'),
            directory=fn_args.working_dir)

    train_dataset = _input_fn(fn_args.train_files,
                              fn_args.data_accessor,
                              transform_graph,
                              batch_size=TRAIN_BATCH_SIZE)

    eval_dataset = _input_fn(fn_args.eval_files,
                             fn_args.data_accessor,
                             transform_graph,
                             batch_size=EVAL_BATCH_SIZE)

    return TunerFnResult(tuner=tuner,
                         fit_kwargs={
                             'x': train_dataset,
                             'validation_data': eval_dataset,
                             'steps_per_epoch': fn_args.train_steps,
                             'validation_steps': fn_args.eval_steps
                         })
Esempio n. 14
0
        if loss_choice == "focal":
            model.compile(loss=focal_loss(),
                          optimizer="adam",
                          metrics=['accuracy'])
        elif loss_choice == "categorical_crossentropy":
            model.compile(loss="categorical_crossentropy",
                          optimizer="adam",
                          metrics=['accuracy'])
        return model


hypermodel = MyHyperModel(num_classes=15)

tuner = kt.RandomSearch(hypermodel,
                        objective='val_accuracy',
                        max_trials=100,
                        executions_per_trial=2,
                        directory='SCSSK',
                        project_name='Hyperparameter tuning')

tuner.search(inputs3d,
             targets,
             batch_size=1000,
             epochs=50,
             shuffle=True,
             verbose=0,
             validation_split=0.05)

models = tuner.get_best_models(num_models=20)
for i, model in enumerate(models):
    model.save(f"best_3dsubm{i+1}.h5")
Esempio n. 15
0
                units=hp.Int(f'{i}_units', min_value=5, max_value=50, step=5),
                activation='relu',
                kernel_regularizer=tf.keras.regularizers.l2(0.0001)))

    model.add(tf.keras.layers.Dense(units=(np.shape(target)[1])))
    model.compile(optimizer=hp.Choice('optimiser',
                                      ['adam', 'adagrad', 'adadelta']),
                  loss='mean_squared_error')

    return model


# Performing random search with objective of minimising loss function
tuner = kt.RandomSearch(build_model,
                        objective='val_loss',
                        max_trials=5,
                        executions_per_trial=3,
                        directory='optimization_folder')
tuner.search_space_summary()
tuner.search(x_train, y_train, validation_data=(x_test, y_test), epochs=10)

# finding the best hyperparameters for the neural network
best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]

# Building the model with the optimised hyperparameters
best_model = tuner.hypermodel.build(best_hps)
""" Call back is performed when the difference between the loss function of 
    traing data and test data is large. This also prevents overfitting
    """
monitor = tf.keras.callbacks.EarlyStopping(monitor='val_loss',
                                           min_delta=1e-3,
Esempio n. 16
0
    print("[INFO] Instantiating a hyperband tuner object...")
    tuner = kt.Hyperband(build_model,
                         objective="val_accuracy",
                         max_epochs=config.EPOCHS,
                         factor=3,
                         seed=42,
                         directory=config.OUTPUT_PATH,
                         project_name=args["tuner"])

# Check for random search tuner
elif args["tuner"] == "random":
    # Instantiate the random search tuner object
    print("[INFO] Instantiating a random search tuner object...")
    tuner = kt.RandomSearch(build_model,
                            objective="val_accuracy",
                            max_trials=10,
                            seed=42,
                            directory=config.OUTPUT_PATH,
                            project_name=args["tuner"])

# Otherwise, use the bayesian optimization tuner
else:
    # Instantiate the bayesian optimization tuner object
    print("[INFO] Instantiating a bayesian optimization tuner object...")
    tuner = kt.BayesianOptimization(build_model,
                                    objective="val_accuracy",
                                    max_trials=10,
                                    seed=42,
                                    directory=config.OUTPUT_PATH,
                                    project_name=args["tuner"])

# Perform the hyperparameter search
Esempio n. 17
0
        model.add(Dropout(drop_rate))

    model.add(Dense(5))
    model.add(Activation('softmax'))

    loss='sparse_categorical_crossentropy'
    
    model.compile(optimizer=tf.keras.optimizers.Adam(hp.Float('learning_rate', 1e-4, 1e-2, sampling='log')),loss='sparse_categorical_crossentropy',metrics=['accuracy'])
    print(model.summary())
    return model
  
LOG_DIR = f"{int (time.time ())}"
tuner=kt.RandomSearch
tuner = kt.RandomSearch(
    build_model,
    objective='val_acc',
    max_trials=2,
    executions_per_trial=1,
    directory=LOG_DIR)

tuner.search(x=X,
             y=y,
             verbose=2, # just slapping this here bc jupyter notebook. The console out was getting messy.
             epochs=1,
             batch_size=64,
             validation_data=(X_test,y_test))

print (tuner.get_best_hyperparameters () [0].values)
print (tuner.results_summary ())
print (tuner.get_best_models () [0].summary ())
model=tuner.get_best_models()[0]
tf.keras.models.save_model(model,'mymodel.h5')
Esempio n. 18
0
def main(_):
    flags_obj = flags.FLAGS

    setup_keras_tuner_config()

    if flags_obj.distribution_strategy == 'tpu':
        resolver = tf.distribute.cluster_resolver.TPUClusterResolver()
        tf.config.experimental_connect_to_cluster(resolver)
        tf.tpu.experimental.initialize_tpu_system(resolver)
        strategy = tf.distribute.experimental.TPUStrategy(resolver)
        strategy_scope = strategy.scope()
        print("All devices: ", tf.config.list_logical_devices('TPU'))
    elif flags_obj.distribution_strategy == 'gpu':
        strategy = tf.distribute.MirroredStrategy()
        strategy_scope = strategy.scope()
        devices = ["device:GPU:%d" % i for i in range(flags_obj.num_gpus)]

    print('NUMBER OF DEVICES: ', strategy.num_replicas_in_sync)

    ## identify data paths and sources
    root_dir = flags_obj.data_dir  # this is gs://<bucket>/folder where tfrecord are stored
    file_pattern = "{}/image_classification_builder-train*.tfrecord*".format(
        root_dir)
    val_file_pattern = "{}/image_classification_builder-validation*.tfrecord*".format(
        root_dir)

    file_list = tf.io.gfile.glob(file_pattern)
    all_files = tf.data.Dataset.list_files(tf.io.gfile.glob(file_pattern))

    val_file_list = tf.io.gfile.glob(val_file_pattern)
    val_all_files = tf.data.Dataset.list_files(
        tf.io.gfile.glob(val_file_pattern))

    train_all_ds = tf.data.TFRecordDataset(
        all_files, num_parallel_reads=tf.data.experimental.AUTOTUNE)
    val_all_ds = tf.data.TFRecordDataset(
        val_all_files, num_parallel_reads=tf.data.experimental.AUTOTUNE)

    # perform data engineering
    dataset = train_all_ds.map(decode_and_resize)
    val_dataset = val_all_ds.map(decode_and_resize)

    #
    BATCH_SIZE = flags_obj.train_batch_size
    VALIDATION_BATCH_SIZE = flags_obj.validation_batch_size
    dataset = dataset.map(normalize,
                          num_parallel_calls=tf.data.experimental.AUTOTUNE)
    val_dataset = val_dataset.map(
        normalize, num_parallel_calls=tf.data.experimental.AUTOTUNE)

    val_ds = val_dataset.batch(VALIDATION_BATCH_SIZE)

    AUTOTUNE = tf.data.experimental.AUTOTUNE
    train_ds = prepare_for_training(dataset)

    FINE_TUNING_CHOICE = True
    NUM_CLASSES = 5
    IMAGE_SIZE = (224, 224)

    train_sample_size = 0
    for raw_record in train_all_ds:
        train_sample_size += 1
    print('TRAIN_SAMPLE_SIZE = ', train_sample_size)
    validation_sample_size = 0
    for raw_record in val_all_ds:
        validation_sample_size += 1
    print('VALIDATION_SAMPLE_SIZE = ', validation_sample_size)

    STEPS_PER_EPOCHS = train_sample_size // BATCH_SIZE
    VALIDATION_STEPS = validation_sample_size // VALIDATION_BATCH_SIZE
    """Runs the hyperparameter search."""
    if (flags_obj.tuner_type.lower() == 'BayesianOptimization'.lower()):
        tuner = kt.BayesianOptimization(hypermodel=model_builder,
                                        objective='val_accuracy',
                                        tune_new_entries=True,
                                        allow_new_entries=True,
                                        max_trials=5,
                                        directory=flags_obj.model_dir,
                                        project_name='hp_tune_bo',
                                        overwrite=True)
    elif (flags_obj.tuner_type.lower() == 'RandomSearch'.lower()):
        tuner = kt.RandomSearch(hypermodel=model_builder,
                                objective='val_accuracy',
                                tune_new_entries=True,
                                allow_new_entries=True,
                                max_trials=5,
                                directory=flags_obj.model_dir,
                                project_name='hp_tune_rs',
                                overwrite=True)
    else:
        tuner = kt.Hyperband(hypermodel=model_builder,
                             objective='val_accuracy',
                             max_epochs=3,
                             factor=2,
                             distribution_strategy=strategy,
                             directory=flags_obj.model_dir,
                             project_name='hp_tune_hb',
                             overwrite=True)

    tuner.search(train_ds,
                 steps_per_epoch=STEPS_PER_EPOCHS,
                 validation_data=val_ds,
                 validation_steps=VALIDATION_STEPS,
                 epochs=3,
                 callbacks=[
                     tf.keras.callbacks.EarlyStopping('val_accuracy'),
                     ClearTrainingOutput()
                 ])

    # Get the optimal hyperparameters
    best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]

    print(f"""
        The hyperparameter search is done. 
        The best number of nodes in the dense layer is {best_hps.get('units')}.
        The best activation function in mid dense layer is {best_hps.get('dense_activation')}.
        """)

    # Build the model with the optimal hyperparameters and train it on the data
    model = tuner.hypermodel.build(best_hps)
    checkpoint_prefix = os.path.join(flags_obj.model_dir,
                                     "best_hp_train_ckpt_{epoch}")
    callbacks = [
        tf.keras.callbacks.TensorBoard(
            log_dir=os.path.join(flags_obj.model_dir, 'tensorboard_logs')),
        tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,
                                           save_weights_only=True)
    ]

    model.fit(train_ds,
              epochs=3,
              steps_per_epoch=STEPS_PER_EPOCHS,
              validation_data=val_ds,
              validation_steps=VALIDATION_STEPS,
              callbacks=callbacks)

    logging.info('INSIDE MAIN FUNCTION user input model_dir %s',
                 flags_obj.model_dir)
    # Save model trained with chosen HP in user specified bucket location
    model_save_dir = os.path.join(flags_obj.model_dir, 'best_save_model')
    model.save(model_save_dir)
Esempio n. 19
0
                   batch_size=100,
                   verbose=2,
                   validation_split=0.2)
    print('acc :', best_model.evaluate(x_test, y_test, verbose=0))


tuner_bayesian = kt.BayesianOptimization(
    model_builder,
    objective='val_acc',
    max_trials=5,
    directory='keras_tuner/bayesian',  # 따로 만들어주지 않아도 알아서 만들어줌
    project_name='mnist')

tuner_random_search = kt.RandomSearch(
    model_builder,
    objective='val_acc',
    max_trials=5,
    directory='keras_tuner/random_search',  # 따로 만들어주지 않아도 알아서 만들어줌
    project_name='mnist')

# 세가지중 hyperband 가 가장 잘찾는다 이거로 쓰면됨
tuner_hyperband = kt.Hyperband(
    model_builder,
    objective='val_loss',
    max_trials=5,
    directory='keras_tuner/hyperband',  # 따로 만들어주지 않아도 알아서 만들어줌
    project_name='mnist')

# optimize_hyper_parameter(tuner_bayesian)
# optimize_hyper_parameter(tuner_random_search)
optimize_hyper_parameter(tuner_hyperband)
Esempio n. 20
0
        lrn_rate = 1e-2
        loss_choice = hp.Choice('loss_function', ["focal","categorical_crossentropy"])        
        
        if loss_choice == "focal":
            model.compile(loss=focal_loss(),
                          optimizer=Adam(lrn_rate),
                          metrics=['accuracy'])
        else:
            model.compile(loss="categorical_crossentropy",
                          optimizer=Adam(lrn_rate),
                          metrics=['accuracy'])
        
        return model

hypermodel = MyHyperModel(num_classes=15)

tuner = kt.RandomSearch(
    hypermodel,
    objective='val_accuracy',
    max_trials=100,
    executions_per_trial=3,
    directory='SCSSK',
    project_name='Dense Tuner')

tuner.search(inputs2d, targets,batch_size=1000, epochs=500,shuffle=True, verbose=0, validation_split = 0.05)

models=tuner.get_best_models(num_models=20)
for i,model in enumerate(models):
    model.save(f"best_2dsubm{i+1}.h5")

tuner.results_summary()
Esempio n. 21
0
def hyper_parameter_search(search_type='BO',
                           objective='mse',
                           seed=101,
                           max_trails=10,
                           directory=os.path.normpath('C:/'),
                           project_name='',
                           max_epochs=10,
                           factor=3,
                           epochs=10,
                           train_data=(),
                           val_data=()):
    '''
    Given the search type this method uses that optimization 
    method from keras tuner and finds the best parameters.
    and returns the model with the best parameteres. 
    '''
    search_type = search_type.upper()

    if search_type == 'BO' or search_type == 'BAYESIANOPTIMIZATION':
        tuner = kt.BayesianOptimization(model_build,
                                        objective=objective,
                                        seed=seed,
                                        max_trials=max_trails,
                                        directory=directory,
                                        project_name=project_name)

    elif search_type == 'RS' or search_type == 'RANDOMSEARCH':
        tuner = kt.RandomSearch(model_build,
                                objective=objective,
                                seed=seed,
                                max_trials=max_trails,
                                directory=directory,
                                project_name=project_name)

    elif search_type == 'HB' or search_type == 'HYPERBAND':
        tuner = kt.Hyperband(model_build,
                             max_epochs=max_epochs,
                             objective=objective,
                             factor=factor,
                             directory=directory,
                             project_name=project_name)
    else:
        raise ValueError(
            'The requested keras tuner search type doesnot exist\n')

    tuner.search(train_data[0],
                 train_data[1],
                 epochs=epochs,
                 validation_data=(val_data[0], val_data[1]),
                 callbacks=[ClearTrainingOutput()],
                 verbose=1)

    best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]

    print(f"""
        The hyperparameter search is complete. The optimal units
        {best_hps.get('units')} and the optimal learning rate is 
        {best_hps.get('learning_rate')} and the optimal dropout
        {best_hps.get('dropout')} and the optimal activation
        {best_hps.get('dense_activation')}.""")
    model = tuner.hypermodel.build(best_hps)
    return model
Esempio n. 22
0
def main(_):
    flags_obj = flags.FLAGS

    strategy = tf.distribute.MirroredStrategy()

    data_dir = get_builtin_data()
    train_gtr, validation_gtr = make_generators(data_dir, flags_obj)
    idx_labels = map_labels(train_gtr)
    """Runs the hyperparameter search."""
    if (flags_obj.tuner_type.lower() == 'BayesianOptimization'.lower()):
        tuner = kt.tuners.BayesianOptimization(hypermodel=model_builder,
                                               objective='val_accuracy',
                                               tune_new_entries=True,
                                               allow_new_entries=True,
                                               max_trials=5,
                                               directory=flags_obj.model_dir,
                                               project_name='hp_tune_bo',
                                               overwrite=True)
    elif (flags_obj.tuner_type.lower() == 'RandomSearch'.lower()):
        tuner = kt.RandomSearch(hypermodel=model_builder,
                                objective='val_accuracy',
                                tune_new_entries=True,
                                allow_new_entries=True,
                                max_trials=5,
                                directory=flags_obj.model_dir,
                                project_name='hp_tune_rs',
                                overwrite=True)
    else:
        tuner = kt.Hyperband(hypermodel=model_builder,
                             objective='val_accuracy',
                             max_epochs=3,
                             factor=2,
                             distribution_strategy=strategy,
                             directory=flags_obj.model_dir,
                             project_name='hp_tune_hb',
                             overwrite=True)

    tuner.search(train_gtr,
                 steps_per_epoch=train_gtr.samples // train_gtr.batch_size,
                 validation_data=validation_gtr,
                 validation_steps=validation_gtr.samples //
                 validation_gtr.batch_size,
                 epochs=3,
                 callbacks=[tf.keras.callbacks.EarlyStopping('val_accuracy')])

    # Get the optimal hyperparameters
    best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]

    print(f"""
        The hyperparameter search is done. 
        The best number of nodes in the dense layer is {best_hps.get('units')}.
        The optimal learning rate for the optimizer is {best_hps.get('learning_rate')}.
        """)

    # Build the model with the optimal hyperparameters and train it on the data
    model = tuner.hypermodel.build(best_hps)
    checkpoint_prefix = os.path.join(flags_obj.model_dir,
                                     "best_hp_train_ckpt_{epoch}")
    callbacks = [
        tf.keras.callbacks.TensorBoard(
            log_dir=os.path.join(flags_obj.model_dir, 'tensorboard_logs')),
        tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,
                                           save_weights_only=True)
    ]

    steps_per_epoch = train_gtr.samples // train_gtr.batch_size
    validation_steps = validation_gtr.samples // validation_gtr.batch_size
    model.fit(train_gtr,
              epochs=flags_obj.train_epoch_best,
              steps_per_epoch=steps_per_epoch,
              validation_data=validation_gtr,
              validation_steps=validation_steps,
              callbacks=callbacks)

    logging.info('INSIDE MAIN FUNCTION user input model_dir %s',
                 flags_obj.model_dir)
    # Save model trained with chosen HP in user specified bucket location
    model_save_dir = os.path.join(flags_obj.model_dir, 'best_save_model')
    model.save(model_save_dir)