def tune(): # train_generator = RobertaDataGenerator(config.train_path) # train_dataset = tf.data.Dataset.from_generator(train_generator.generate, # output_types=({'ids': tf.int32, 'att': tf.int32, 'tti': tf.int32}, # {'sts': tf.int32, 'ets': tf.int32})) # train_dataset = train_dataset.padded_batch(32, # padded_shapes=({'ids': [None], 'att': [None], 'tti': [None]}, # {'sts': [None], 'ets': [None]}), # padding_values=({'ids': 1, 'att': 0, 'tti': 0}, # {'sts': 0, 'ets': 0})) # train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE) # # val_generator = RobertaDataGenerator(config.validation_path) # val_dataset = tf.data.Dataset.from_generator(val_generator.generate, # output_types=({'ids': tf.int32, 'att': tf.int32, 'tti': tf.int32}, # {'sts': tf.int32, 'ets': tf.int32})) # val_dataset = val_dataset.padded_batch(32, # padded_shapes=({'ids': [None], 'att': [None], 'tti': [None]}, # {'sts': [None], 'ets': [None]}), # padding_values=({'ids': 1, 'att': 0, 'tti': 0}, # {'sts': 0, 'ets': 0})) # val_dataset = val_dataset.prefetch(tf.data.experimental.AUTOTUNE) train_dataset = RobertaData(Config.train_path, 'train').get_data() val_dataset = RobertaData(Config.validation_path, 'val').get_data() tuner = Hyperband(get_tunable_roberta, objective='val_loss', max_epochs=10, factor=3, hyperband_iterations=3, seed=Config.seed, directory='tuner_logs', project_name='feat_roberta') tuner.search_space_summary() callbacks = [ tf.keras.callbacks.ReduceLROnPlateau(patience=2, verbose=1), tf.keras.callbacks.EarlyStopping(patience=3, verbose=1) ] tuner.search(train_dataset[0], train_dataset[1], epochs=10, verbose=1, callbacks=callbacks, batch_size=32, validation_data=val_dataset) tuner.results_summary() best_hps: List[HyperParameters] = tuner.get_best_hyperparameters( num_trials=5) for hp in best_hps: print(f'{hp.values}\n') model = tuner.hypermodel.build(best_hps[0]) tf.keras.utils.plot_model(model, to_file='best_hp_tuned_model.png', show_shapes=True, show_layer_names=True, expand_nested=True) model.summary()