def find_best_NN(x_train_main, y_train_main): tuner = Hyperband(build_model, objective="loss", max_epochs=10, hyperband_iterations=10) tuner.search(x_train, y_train, batch_size=1, epochs=10, validation_split=0.3) tuner.results_summary() print("\n\n\n") print("\n\n\nHERE IS THE BEST MODEL\n\n\n") best_params = tuner.get_best_hyperparameters()[0] best_model = tuner.hypermodel.build(best_params) best_model.summary() return best_model
def find_best(x_train, y_train): # создаю тюнер, который сможет подобрать оптимальную архитектуру модели tuner = Hyperband(build_model, objective="loss", max_epochs=10, hyperband_iterations=3) print("\n\n\n") # начинается автоматический подбор гиперпараметров print('[INFO] start searching') tuner.search(x_train, y_train, batch_size=128, epochs=10, validation_split=0.2) # выбираем лучшую модель print("\n\n\nRESULTS SUMMARY") tuner.results_summary() print("\n\n\n") # получаем лучшую модель print("\n\n\nHERE IS THE BEST MODEL\n\n\n") best_params = tuner.get_best_hyperparameters()[0] best_model = tuner.hypermodel.build(best_params) best_model.summary() return best_model
def nn_tune_train(data: pd.DataFrame, model_names: list)->dict: num_classes = len(data.label.unique()) with tqdm(total=num_classes) as bar: for X_train, X_test, y_train, y_test, label in ut.data_split_classwise(data): bar.set_description(f'Tuning on model {label}') for name in model_names: tuner = Hyperband( _build_model, objective='val_loss', max_epochs=50, factor=3, directory='hyperband', project_name=f'slp{label}' ) tuner.search(X_train, y_train, epochs=50, validation_data=(X_test, y_test)) best_hps = tuner.get_best_hyperparameters(num_trials=1)[0] print(f"{name} optimal params: {best_hps}") bar.update(1)
def tuneHyperband(X, y, max_trials=3): """ Perform Hyperband Tuning to search for the best model and Hyperparameters Arguments: X: Input dataset y: Label or output dataset max_trials: Trials required to perform tuning """ hypermodel = HyperResNet(input_shape=(128, 128, 3), num_classes=10) tuner = Hyperband( hypermodel, max_epochs=max_trials, objective='val_accuracy', seed=42, ) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=49) # searches the tuner space defined by hyperparameters (hp) and returns the # best model tuner.search(X_train, y_train, epochs=5, validation_data=(X_test, y_test)) hyp = tuner.get_best_hyperparameters(num_trials=1)[0] #hyp = tuner.oracle.get_best_trials(num_trials=1)[0].hyperparameters.values #best_hps = np.stack(hyp).astype(None) history = tuner_hist(X, y, tuner, hyp) """ Return: models[0] : best model obtained after tuning best_hps : best Hyperprameters obtained after tuning, stored as array history : history of the data executed from the given model """ return tuner.get_best_models(1)[0], hyp, history
def main(): metadata = load_metadata() # train_metadata2, test_metadata2 = load_metadata2() ''' Fraction of positive samples wanted in both test and validation set, the same as it is in the test set of the Kaggle Chest X-Ray Images (Pneumonia) dataset''' train_metadata, val_metadata, test_metadata = split_dataset(metadata, augmentation + 1) # train_metadata = train_metadata.head(8) # val_metadata = val_metadata.head(256) if augmentation > 0: pos_train_metadata = train_metadata[train_metadata['Pneumonia'] == 1] neg_train_metadata = train_metadata[train_metadata['Pneumonia'] == 0] pos_train_metadata_augmented = augment(metadata=pos_train_metadata, rate=augmentation, batch_size=16, seed=seed) train_metadata = pd.concat([pos_train_metadata, pos_train_metadata_augmented, neg_train_metadata], ignore_index=True) # train_metadata = pd.concat(([train_metadata, train_metadata2]), ignore_index=True) train_metadata = shuffle_DataFrame(train_metadata, seed) """val_metadata2, test_metadata2 = split_dataset2(test_metadata2) val_metadata = pd.concat(([val_metadata, val_metadata2]), ignore_index=True) test_metadata = pd.concat(([test_metadata, test_metadata2]), ignore_index=True)""" print('Train:') print(Counter(train_metadata['Pneumonia'])) print('Validation:') print(Counter(val_metadata['Pneumonia'])) print('Test:') print(Counter(test_metadata['Pneumonia'])) train_ds = make_pipeline(file_paths=train_metadata['File Path'].to_numpy(), y=train_metadata['Pneumonia'].to_numpy(), shuffle=True, batch_size=batch_size, seed=seed) val_ds = make_pipeline(file_paths=val_metadata['File Path'].to_numpy(), y=val_metadata['Pneumonia'].to_numpy(), shuffle=False, batch_size=val_batch_size) samples_ds = make_pipeline(file_paths=train_metadata['File Path'].to_numpy(), y=train_metadata['Pneumonia'].to_numpy(), shuffle=True, batch_size=16, seed=seed) show_samples(samples_ds) del samples_ds # Not needed anymore """str_time = datetime.datetime.now().strftime("%m%d-%H%M%S") log_dir = f'{logs_root}/{str_time}' logs_cb = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, profile_batch=0) history = model.fit(x=train_ds, validation_data=val_ds, epochs=300, shuffle=False, callbacks=[logs_cb])""" # TODO what if I use val_loss instead? tuner = Hyperband(make_model_DenseNet121, objective=kt.Objective("val_auc", direction="max"), # Careful to keep the direction updated max_epochs=epochs_base, hyperband_iterations=2, directory='computations', project_name='base-1dataset-densenet121-auc-auc') early_stopping_cb = tf.keras.callbacks.EarlyStopping(monitor='val_auc', patience=patience) tuner.search(x=train_ds, validation_data=val_ds, epochs=epochs_base, shuffle=False, callbacks=[early_stopping_cb]) model_maker = ModelMaker(tuner) tuner_ft = Hyperband(model_maker.make_model_DenseNet121, objective=kt.Objective("val_auc", direction="max"), # Careful to keep the direction updated max_epochs=epochs_ft, hyperband_iterations=2, directory='computations', project_name='fine-1dataset-densenet121-auc-auc') early_stopping_cb = tf.keras.callbacks.EarlyStopping(monitor='val_auc', patience=patience) tuner_ft.search(x=train_ds, validation_data=val_ds, epochs=epochs_ft, shuffle=False, callbacks=[early_stopping_cb]) best_ft_model = tuner_ft.get_best_models()[0] test_ds = make_pipeline(file_paths=test_metadata['File Path'].to_numpy(), y=test_metadata['Pneumonia'].to_numpy(), shuffle=False, batch_size=val_batch_size) test_results = best_ft_model.evaluate(x=test_ds, return_dict=True, verbose=1) print('\nHyper-parameters for the fine-tuned model:') print(tuner_ft.get_best_hyperparameters()[0].values) print('\nTest results on the fine-tuned model:') print(test_results) hps = tuner_ft.get_best_hyperparameters()[0].values dev_metadata = pd.concat([train_metadata, val_metadata], ignore_index=True) dev_metadata = shuffle_DataFrame(dev_metadata, seed=seed) dev_metadata.reset_index(inplace=True, drop=True) fold_size = int(np.ceil(len(dev_metadata) / n_folds)) train_datasets, val_datasets = [], [] for fold in range(n_folds): fold_val_metadata = dev_metadata.iloc[fold * fold_size:fold * fold_size + fold_size, :] val_datasets.append(make_pipeline(file_paths=fold_val_metadata['File Path'].to_numpy(), y=fold_val_metadata['Pneumonia'].to_numpy(), batch_size=val_batch_size, shuffle=False)) fold_train_metadata1 = dev_metadata.iloc[:fold * fold_size, :] fold_train_metadata2 = dev_metadata.iloc[fold * fold_size + fold_size:, :] fold_train_metadata = pd.concat([fold_train_metadata1, fold_train_metadata2], ignore_index=False) train_datasets.append(make_pipeline(file_paths=fold_train_metadata['File Path'].to_numpy(), y=fold_train_metadata['Pneumonia'].to_numpy(), batch_size=batch_size, shuffle=True, seed=seed)) histories, means = k_fold_resumable_fit(model_maker=model_maker.make_model_DenseNet121, hp=hps, comp_dir='computations', project='xval', train_datasets=train_datasets, val_datasets=val_datasets, log_dir='computations/xval/logs', epochs=epochs_ft) print(histories) print(means) pass
if mode == 'tune': tuner = Hyperband( simpnet, objective='val_sparse_categorical_accuracy', max_epochs=15, hyperband_iterations=1, ) tuner.search( train, epochs=20, validation_data=test, callbacks=SimpNet.get_callbacks(), ) best_hps = tuner.get_best_hyperparameters(num_trials=1) # Build the model with the optimal hyperparameters and train it on the data for 50 epochs model = tuner.hypermodel.build(best_hps) history = model.fit( train, epochs=50, validation_data=test, callbacks=SimpNet.get_callbacks(), verbose=2, ) elif mode == 'test': hp = HyperParameters() # best 1 0.34% misclassification
hypermodel = MyHyperModel(num_classes=1) tuner = Hyperband(hypermodel, objective='accuracy', max_epochs=10, seed=10, project_name='divorce test') tuner.search(X_train.values, y_train.values.flatten(), epochs=10, validation_data=(X_test.values, y_test.values.flatten())) params = tuner.get_best_hyperparameters()[0] model = tuner.hypermodel.build(params) model.fit(X.values, y.values.flatten(), epochs=20) hyperband_accuracy_df = pd.DataFrame(model.history.history) hyperband_accuracy_df[['loss', 'accuracy']].plot() plt.title('Loss & Accuracy Per EPOCH') plt.xlabel('EPOCH') plt.ylabel('Accruacy') plt.show() random_tuner = RandomSearch(hypermodel, objective='accuracy',
class AutoQKeras: """Performs autoquantization in Keras model. Arguments: model: Model to be quantized. metrics: List of metrics to be used. custom_objects: Custom objects used by Keras during quantization. goal: Metric to compute secondary goal of search (bits or energy) output_dir: name of output directory to store results. mode: random, hyperband or bayesian used by kerastuner. transfer_weights: if true, transfer weights from unquantized model. frozen_layers: if true, these layers will not be quantized but weights transferred from original model. activation_bits: parameter to be used by 'model_quantize'. limit: limit the number of bits in quantizers specified as a dictionary. tune_filters: one of "block", "layer", "none" for tuning entire network, each layer separately, or no tuning. tune_filters_exceptions: name of layers that will not be tuned. layer_indexes: indexes of layers we will quantize. learning_rate_optimizer: if true, user will provide lr scheduler callback. quantization_config: file name of dictionary containing configuration of quantizers for kernel, bias and activation. tuner_kwargs: parameters for kerastuner depending on whether mode is random, hyperband or baeysian. Please refer to the documentation of kerstuner Tuners. """ def __init__(self, model, metrics=None, custom_objects=None, goal=None, output_dir="result", mode="random", transfer_weights=False, frozen_layers=None, activation_bits=4, limit=None, tune_filters="none", tune_filters_exceptions=None, learning_rate_optimizer=False, layer_indexes=None, quantization_config=None, overwrite=True, **tuner_kwargs): if not metrics: metrics = [] if not custom_objects: custom_objects = {} # goal: { "type": ["bits", "energy"], "params": {...} } or ForgivingFactor # type # For type == "bits": # delta_p: increment (in %) of the accuracy if trial is smaller. # delta_n: decrement (in %) of the accuracy if trial is bigger. # rate: rate of decrease/increase in model size in terms of bits. # input_bits; size of input tensors. # output_bits; size of output tensors. # stress: parameter to reduce reference size to force tuner to # choose smaller models. # config: configuration on what to compute for each layer # minimum configuration is { "default": ["parameters", "activations"] } # use simplest one - number of bits if not goal: goal = { "type": "bits", "params": { "delta_p": 8.0, "delta_n": 8.0, "rate": 2.0, "stress": 1.0, "input_bits": 8, "output_bits": 8, "ref_bits": 8, "config": { "default": ["parameters", "activations"] } } } self.overwrite = overwrite # if we have not created it already, create new one. if not isinstance(goal, ForgivingFactor): target = forgiving_factor[goal["type"]](**goal["params"]) else: target = goal # if no metrics were specified, we want to make sure we monitor at least # accuracy. if not metrics: metrics = ["acc"] self.hypermodel = AutoQKHyperModel( model, metrics, custom_objects, target, transfer_weights=transfer_weights, frozen_layers=frozen_layers, activation_bits=activation_bits, limit=limit, tune_filters=tune_filters, tune_filters_exceptions=tune_filters_exceptions, layer_indexes=layer_indexes, learning_rate_optimizer=learning_rate_optimizer, quantization_config=quantization_config) # right now we create unique results directory idx = 0 name = output_dir if self.overwrite: while os.path.exists(name): idx += 1 name = output_dir + "_" + str(idx) output_dir = name self.output_dir = output_dir # let's ignore mode for now assert mode in ["random", "bayesian", "hyperband"] if mode == "random": self.tuner = RandomSearch(self.hypermodel, objective=kt.Objective( "val_score", "max"), project_name=output_dir, **tuner_kwargs) elif mode == "bayesian": self.tuner = BayesianOptimization(self.hypermodel, objective=kt.Objective( "val_score", "max"), project_name=output_dir, **tuner_kwargs) elif mode == "hyperband": self.tuner = Hyperband(self.hypermodel, objective=kt.Objective("val_score", "max"), project_name=output_dir, **tuner_kwargs) else: pass self.tuner.search_space_summary() def _has_earlystopping(self, callbacks): """Check if EarlyStopping has been defined or not.""" if callbacks is None: return False for callback in callbacks: if isinstance(callback, tf.keras.callbacks.EarlyStopping): return True return False def history(self, number_of_trials=-1): """Returns the history of the model search.""" trials = self.tuner.oracle.get_best_trials(number_of_trials) state = [trial.get_state() for trial in trials] result = {} result["score"] = [ state[i]["score"] for i in range(len(state)) if trials[i].score is not None ] for i in range(len(state)): if trials[i].score is not None: keys = state[i]["metrics"]["metrics"].keys() for key in keys: if key != "score" and not key.startswith( "val_") and key != "loss" and key != "trial": cur_accuracy = state[i]["metrics"]["metrics"][key][ "observations"][0]["value"][0] if "val_" + key in state[i]["metrics"]["metrics"].keys( ): cur_val_accuracy = state[i]["metrics"]["metrics"][ "val_" + key]["observations"][0]["value"][0] else: cur_val_accuracy = None # only update result if both key and val_key exist if cur_val_accuracy: if key not in result.keys(): result[key] = [cur_accuracy] result["val_" + key] = [cur_val_accuracy] else: result[key].append(cur_accuracy) result["val_" + key].append(cur_val_accuracy) result["trial_size"] = [ state[i]["metrics"]["metrics"]["trial"]["observations"][0]["value"] [0] for i in range(len(state)) if trials[i].score is not None ] return result def fit(self, *fit_args, **fit_kwargs): """Invokes tuner fit algorithm.""" callbacks = fit_kwargs.get("callbacks", None) if callbacks is None: callbacks = [] epochs = fit_kwargs.get("epochs", None) if epochs is None: epochs = 10 if not self._has_earlystopping(callbacks): callbacks = callbacks + [ tf.keras.callbacks.EarlyStopping("val_loss", patience=min(20, epochs // 5)) ] fit_kwargs["callbacks"] = callbacks self.tuner.search(*fit_args, **fit_kwargs) @staticmethod def get_best_lr(qmodel): """Extracts best lr of model.""" return qmodel.optimizer.lr.numpy() def get_best_model(self): params = self.tuner.get_best_hyperparameters()[0] q_model = self.tuner.hypermodel.build(params) self.learning_rate = q_model.optimizer.lr.numpy() return q_model def get_learning_rate(self): return self.learning_rate
objective='val_categorical_accuracy', max_epochs=40, project_name='tuning_categorical' ) tuner.search_space_summary() tuner.search(trainX, trainY,epochs=25,validation_data=(testX, testY),verbose=0) tuner.results_summary() best_model = tuner.get_best_models(num_models=1)[0] # Evaluate the best model. loss, accuracy = best_model.evaluate(testX, testY) tuner.get_best_hyperparameters()[0].values """Pretty good results : 0.898 vs 0.7551 categorical accuracy ## Testing different applications (ResNet, Inception) """ from keras.applications import VGG16 from keras import Model from keras.layers import AveragePooling2D, Input, Flatten, Dense, Dropout from keras.optimizers import Adam from sklearn.model_selection import train_test_split def imagenet_model(lr=1e-3, epochs=25, dropout_rate=0.5,dense_nodes=64,base_model=VGG16): baseModel = base_model(weights="imagenet", include_top=False, input_tensor=Input(shape=(224, 224, 3)))
seed=42, hyperband_iterations=3) #print total search space tuner_hb.search_space_summary() #search through the total search space tuner_hb.search(train_generator, epochs=500, verbose=1, validation_data=(img_test, mask_test)) #save best model and hyperparameters best_model = tuner_hb.get_best_models(1)[0] best_hyperparameters = tuner_hb.get_best_hyperparameters(1)[0] print(tuner_hb.get_best_hyperparameters(1)) # model_json = best_model.to_json() with open("hp_bce_all_basicUNet_model.json", "w") as json_file: json_file.write(model_json) # serialize weights to HDF5 best_model.save_weights("hp_bce_all_tune_basicUNet_tuner_model.h5") with open('best_LGG_basicUNet_Param.txt', 'w') as f: print(best_hyperparameters, file=f) print("Saved model to disk") print(best_hyperparameters) tuner_hb.results_summary() #print best 10 models
model.add(Dense(160, activation='relu')) model.add(Dense(6, activation='softmax')) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) return model tuner = Hyperband(build_model, objective='val_acc', max_epochs=20, directory='vgg_transfer_param', hyperparameters=hp, project_name='vgg_transfer_project') tuner.search(datagen_train, epochs=10, validation_data=datagen_valid) best_hps = tuner.get_best_hyperparameters(1)[0] print(best_hps.values) # model = build_model() # # 编译模型 # # model.compile(optimizer='Adam', # # loss='categorical_crossentropy', # # metrics=[tf.keras.metrics.categorical_accuracy]) # # history_callback = LossHistory() # checkpoint = ModelCheckpoint("trained_model.h5", monitor='val_categorical_accuracy', verbose=1, save_best_only=True, mode='max', period=2) # history = model.fit_generator( # generator=datagen_train, # validation_data=datagen_valid, # epochs=30, # validation_freq=1, # callbacks=[history_callback, checkpoint]
def tune(): # train_generator = RobertaDataGenerator(config.train_path) # train_dataset = tf.data.Dataset.from_generator(train_generator.generate, # output_types=({'ids': tf.int32, 'att': tf.int32, 'tti': tf.int32}, # {'sts': tf.int32, 'ets': tf.int32})) # train_dataset = train_dataset.padded_batch(32, # padded_shapes=({'ids': [None], 'att': [None], 'tti': [None]}, # {'sts': [None], 'ets': [None]}), # padding_values=({'ids': 1, 'att': 0, 'tti': 0}, # {'sts': 0, 'ets': 0})) # train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE) # # val_generator = RobertaDataGenerator(config.validation_path) # val_dataset = tf.data.Dataset.from_generator(val_generator.generate, # output_types=({'ids': tf.int32, 'att': tf.int32, 'tti': tf.int32}, # {'sts': tf.int32, 'ets': tf.int32})) # val_dataset = val_dataset.padded_batch(32, # padded_shapes=({'ids': [None], 'att': [None], 'tti': [None]}, # {'sts': [None], 'ets': [None]}), # padding_values=({'ids': 1, 'att': 0, 'tti': 0}, # {'sts': 0, 'ets': 0})) # val_dataset = val_dataset.prefetch(tf.data.experimental.AUTOTUNE) train_dataset = RobertaData(Config.train_path, 'train').get_data() val_dataset = RobertaData(Config.validation_path, 'val').get_data() tuner = Hyperband(get_tunable_roberta, objective='val_loss', max_epochs=10, factor=3, hyperband_iterations=3, seed=Config.seed, directory='tuner_logs', project_name='feat_roberta') tuner.search_space_summary() callbacks = [ tf.keras.callbacks.ReduceLROnPlateau(patience=2, verbose=1), tf.keras.callbacks.EarlyStopping(patience=3, verbose=1) ] tuner.search(train_dataset[0], train_dataset[1], epochs=10, verbose=1, callbacks=callbacks, batch_size=32, validation_data=val_dataset) tuner.results_summary() best_hps: List[HyperParameters] = tuner.get_best_hyperparameters( num_trials=5) for hp in best_hps: print(f'{hp.values}\n') model = tuner.hypermodel.build(best_hps[0]) tf.keras.utils.plot_model(model, to_file='best_hp_tuned_model.png', show_shapes=True, show_layer_names=True, expand_nested=True) model.summary()