def find_best_NN(x_train_main, y_train_main):
tuner = Hyperband(build_model, objective="loss", max_epochs=10, hyperband_iterations=10)
tuner.search(x_train, y_train, batch_size=1, epochs=10, validation_split=0.3)
tuner.results_summary()
print("\n\n\n")
print("\n\n\nHERE IS THE BEST MODEL\n\n\n")
best_params = tuner.get_best_hyperparameters()[0]
best_model = tuner.hypermodel.build(best_params)
best_model.summary()
return best_model
def find_best(x_train, y_train):
# создаю тюнер, который сможет подобрать оптимальную архитектуру модели
tuner = Hyperband(build_model, objective="loss", max_epochs=10, hyperband_iterations=3)
print("\n\n\n")
# начинается автоматический подбор гиперпараметров
print('[INFO] start searching')
tuner.search(x_train, y_train, batch_size=128, epochs=10, validation_split=0.2)
# выбираем лучшую модель
print("\n\n\nRESULTS SUMMARY")
tuner.results_summary()
print("\n\n\n")
# получаем лучшую модель
print("\n\n\nHERE IS THE BEST MODEL\n\n\n")
best_params = tuner.get_best_hyperparameters()[0]
best_model = tuner.hypermodel.build(best_params)
best_model.summary()
return best_model
def nn_tune_train(data: pd.DataFrame, model_names: list)->dict:
num_classes = len(data.label.unique())
with tqdm(total=num_classes) as bar:
for X_train, X_test, y_train, y_test, label in ut.data_split_classwise(data):
bar.set_description(f'Tuning on model {label}')
for name in model_names:
tuner = Hyperband(
_build_model,
objective='val_loss',
max_epochs=50,
factor=3,
directory='hyperband',
project_name=f'slp{label}'
)
tuner.search(X_train, y_train, epochs=50, validation_data=(X_test, y_test))
best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
print(f"{name} optimal params: {best_hps}")
bar.update(1)
def tuneHyperband(X,
y,
max_trials=3):
"""
Perform Hyperband Tuning to search for the best model and Hyperparameters
Arguments:
X: Input dataset
y: Label or output dataset
max_trials: Trials required to perform tuning
"""
hypermodel = HyperResNet(input_shape=(128, 128, 3), num_classes=10)
tuner = Hyperband(
hypermodel,
max_epochs=max_trials,
objective='val_accuracy',
seed=42,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=49)
# searches the tuner space defined by hyperparameters (hp) and returns the
# best model
tuner.search(X_train, y_train,
epochs=5,
validation_data=(X_test, y_test))
hyp = tuner.get_best_hyperparameters(num_trials=1)[0]
#hyp = tuner.oracle.get_best_trials(num_trials=1)[0].hyperparameters.values
#best_hps = np.stack(hyp).astype(None)
history = tuner_hist(X, y, tuner, hyp)
"""
Return:
models[0] : best model obtained after tuning
best_hps : best Hyperprameters obtained after tuning, stored as array
history : history of the data executed from the given model
"""
return tuner.get_best_models(1)[0], hyp, history
def main():
metadata = load_metadata()
# train_metadata2, test_metadata2 = load_metadata2()
''' Fraction of positive samples wanted in both test and validation set, the same as it is in the test set of
the Kaggle Chest X-Ray Images (Pneumonia) dataset'''
train_metadata, val_metadata, test_metadata = split_dataset(metadata, augmentation + 1)
# train_metadata = train_metadata.head(8)
# val_metadata = val_metadata.head(256)
if augmentation > 0:
pos_train_metadata = train_metadata[train_metadata['Pneumonia'] == 1]
neg_train_metadata = train_metadata[train_metadata['Pneumonia'] == 0]
pos_train_metadata_augmented = augment(metadata=pos_train_metadata, rate=augmentation, batch_size=16, seed=seed)
train_metadata = pd.concat([pos_train_metadata, pos_train_metadata_augmented, neg_train_metadata],
ignore_index=True)
# train_metadata = pd.concat(([train_metadata, train_metadata2]), ignore_index=True)
train_metadata = shuffle_DataFrame(train_metadata, seed)
"""val_metadata2, test_metadata2 = split_dataset2(test_metadata2)
val_metadata = pd.concat(([val_metadata, val_metadata2]), ignore_index=True)
test_metadata = pd.concat(([test_metadata, test_metadata2]), ignore_index=True)"""
print('Train:')
print(Counter(train_metadata['Pneumonia']))
print('Validation:')
print(Counter(val_metadata['Pneumonia']))
print('Test:')
print(Counter(test_metadata['Pneumonia']))
train_ds = make_pipeline(file_paths=train_metadata['File Path'].to_numpy(),
y=train_metadata['Pneumonia'].to_numpy(),
shuffle=True,
batch_size=batch_size,
seed=seed)
val_ds = make_pipeline(file_paths=val_metadata['File Path'].to_numpy(),
y=val_metadata['Pneumonia'].to_numpy(),
shuffle=False,
batch_size=val_batch_size)
samples_ds = make_pipeline(file_paths=train_metadata['File Path'].to_numpy(),
y=train_metadata['Pneumonia'].to_numpy(),
shuffle=True,
batch_size=16,
seed=seed)
show_samples(samples_ds)
del samples_ds # Not needed anymore
"""str_time = datetime.datetime.now().strftime("%m%d-%H%M%S")
log_dir = f'{logs_root}/{str_time}'
logs_cb = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, profile_batch=0)
history = model.fit(x=train_ds, validation_data=val_ds, epochs=300, shuffle=False, callbacks=[logs_cb])"""
# TODO what if I use val_loss instead?
tuner = Hyperband(make_model_DenseNet121,
objective=kt.Objective("val_auc", direction="max"), # Careful to keep the direction updated
max_epochs=epochs_base,
hyperband_iterations=2,
directory='computations',
project_name='base-1dataset-densenet121-auc-auc')
early_stopping_cb = tf.keras.callbacks.EarlyStopping(monitor='val_auc', patience=patience)
tuner.search(x=train_ds,
validation_data=val_ds,
epochs=epochs_base,
shuffle=False,
callbacks=[early_stopping_cb])
model_maker = ModelMaker(tuner)
tuner_ft = Hyperband(model_maker.make_model_DenseNet121,
objective=kt.Objective("val_auc", direction="max"), # Careful to keep the direction updated
max_epochs=epochs_ft,
hyperband_iterations=2,
directory='computations',
project_name='fine-1dataset-densenet121-auc-auc')
early_stopping_cb = tf.keras.callbacks.EarlyStopping(monitor='val_auc', patience=patience)
tuner_ft.search(x=train_ds,
validation_data=val_ds,
epochs=epochs_ft,
shuffle=False,
callbacks=[early_stopping_cb])
best_ft_model = tuner_ft.get_best_models()[0]
test_ds = make_pipeline(file_paths=test_metadata['File Path'].to_numpy(),
y=test_metadata['Pneumonia'].to_numpy(),
shuffle=False,
batch_size=val_batch_size)
test_results = best_ft_model.evaluate(x=test_ds, return_dict=True, verbose=1)
print('\nHyper-parameters for the fine-tuned model:')
print(tuner_ft.get_best_hyperparameters()[0].values)
print('\nTest results on the fine-tuned model:')
print(test_results)
hps = tuner_ft.get_best_hyperparameters()[0].values
dev_metadata = pd.concat([train_metadata, val_metadata], ignore_index=True)
dev_metadata = shuffle_DataFrame(dev_metadata, seed=seed)
dev_metadata.reset_index(inplace=True, drop=True)
fold_size = int(np.ceil(len(dev_metadata) / n_folds))
train_datasets, val_datasets = [], []
for fold in range(n_folds):
fold_val_metadata = dev_metadata.iloc[fold * fold_size:fold * fold_size + fold_size, :]
val_datasets.append(make_pipeline(file_paths=fold_val_metadata['File Path'].to_numpy(),
y=fold_val_metadata['Pneumonia'].to_numpy(),
batch_size=val_batch_size,
shuffle=False))
fold_train_metadata1 = dev_metadata.iloc[:fold * fold_size, :]
fold_train_metadata2 = dev_metadata.iloc[fold * fold_size + fold_size:, :]
fold_train_metadata = pd.concat([fold_train_metadata1, fold_train_metadata2], ignore_index=False)
train_datasets.append(make_pipeline(file_paths=fold_train_metadata['File Path'].to_numpy(),
y=fold_train_metadata['Pneumonia'].to_numpy(),
batch_size=batch_size,
shuffle=True,
seed=seed))
histories, means = k_fold_resumable_fit(model_maker=model_maker.make_model_DenseNet121,
hp=hps,
comp_dir='computations',
project='xval',
train_datasets=train_datasets,
val_datasets=val_datasets,
log_dir='computations/xval/logs',
epochs=epochs_ft)
print(histories)
print(means)
pass
if mode == 'tune':
tuner = Hyperband(
simpnet,
objective='val_sparse_categorical_accuracy',
max_epochs=15,
hyperband_iterations=1,
)
tuner.search(
train,
epochs=20,
validation_data=test,
callbacks=SimpNet.get_callbacks(),
)
best_hps = tuner.get_best_hyperparameters(num_trials=1)
# Build the model with the optimal hyperparameters and train it on the data for 50 epochs
model = tuner.hypermodel.build(best_hps)
history = model.fit(
train,
epochs=50,
validation_data=test,
callbacks=SimpNet.get_callbacks(),
verbose=2,
)
elif mode == 'test':
hp = HyperParameters()
# best 1 0.34% misclassification
hypermodel = MyHyperModel(num_classes=1)
tuner = Hyperband(hypermodel,
objective='accuracy',
max_epochs=10,
seed=10,
project_name='divorce test')
tuner.search(X_train.values,
y_train.values.flatten(),
epochs=10,
validation_data=(X_test.values, y_test.values.flatten()))
params = tuner.get_best_hyperparameters()[0]
model = tuner.hypermodel.build(params)
model.fit(X.values, y.values.flatten(), epochs=20)
hyperband_accuracy_df = pd.DataFrame(model.history.history)
hyperband_accuracy_df[['loss', 'accuracy']].plot()
plt.title('Loss & Accuracy Per EPOCH')
plt.xlabel('EPOCH')
plt.ylabel('Accruacy')
plt.show()
random_tuner = RandomSearch(hypermodel,
objective='accuracy',
class AutoQKeras:
"""Performs autoquantization in Keras model.
Arguments:
model: Model to be quantized.
metrics: List of metrics to be used.
custom_objects: Custom objects used by Keras during quantization.
goal: Metric to compute secondary goal of search (bits or energy)
output_dir: name of output directory to store results.
mode: random, hyperband or bayesian used by kerastuner.
transfer_weights: if true, transfer weights from unquantized model.
frozen_layers: if true, these layers will not be quantized but
weights transferred from original model.
activation_bits: parameter to be used by 'model_quantize'.
limit: limit the number of bits in quantizers specified as a dictionary.
tune_filters: one of "block", "layer", "none" for tuning entire
network, each layer separately, or no tuning.
tune_filters_exceptions: name of layers that will not be tuned.
layer_indexes: indexes of layers we will quantize.
learning_rate_optimizer: if true, user will provide lr scheduler
callback.
quantization_config: file name of dictionary containing configuration of
quantizers for kernel, bias and activation.
tuner_kwargs: parameters for kerastuner depending on whether
mode is random, hyperband or baeysian. Please refer to the
documentation of kerstuner Tuners.
"""
def __init__(self,
model,
metrics=None,
custom_objects=None,
goal=None,
output_dir="result",
mode="random",
transfer_weights=False,
frozen_layers=None,
activation_bits=4,
limit=None,
tune_filters="none",
tune_filters_exceptions=None,
learning_rate_optimizer=False,
layer_indexes=None,
quantization_config=None,
overwrite=True,
**tuner_kwargs):
if not metrics:
metrics = []
if not custom_objects:
custom_objects = {}
# goal: { "type": ["bits", "energy"], "params": {...} } or ForgivingFactor
# type
# For type == "bits":
# delta_p: increment (in %) of the accuracy if trial is smaller.
# delta_n: decrement (in %) of the accuracy if trial is bigger.
# rate: rate of decrease/increase in model size in terms of bits.
# input_bits; size of input tensors.
# output_bits; size of output tensors.
# stress: parameter to reduce reference size to force tuner to
# choose smaller models.
# config: configuration on what to compute for each layer
# minimum configuration is { "default": ["parameters", "activations"] }
# use simplest one - number of bits
if not goal:
goal = {
"type": "bits",
"params": {
"delta_p": 8.0,
"delta_n": 8.0,
"rate": 2.0,
"stress": 1.0,
"input_bits": 8,
"output_bits": 8,
"ref_bits": 8,
"config": {
"default": ["parameters", "activations"]
}
}
}
self.overwrite = overwrite
# if we have not created it already, create new one.
if not isinstance(goal, ForgivingFactor):
target = forgiving_factor[goal["type"]](**goal["params"])
else:
target = goal
# if no metrics were specified, we want to make sure we monitor at least
# accuracy.
if not metrics:
metrics = ["acc"]
self.hypermodel = AutoQKHyperModel(
model,
metrics,
custom_objects,
target,
transfer_weights=transfer_weights,
frozen_layers=frozen_layers,
activation_bits=activation_bits,
limit=limit,
tune_filters=tune_filters,
tune_filters_exceptions=tune_filters_exceptions,
layer_indexes=layer_indexes,
learning_rate_optimizer=learning_rate_optimizer,
quantization_config=quantization_config)
# right now we create unique results directory
idx = 0
name = output_dir
if self.overwrite:
while os.path.exists(name):
idx += 1
name = output_dir + "_" + str(idx)
output_dir = name
self.output_dir = output_dir
# let's ignore mode for now
assert mode in ["random", "bayesian", "hyperband"]
if mode == "random":
self.tuner = RandomSearch(self.hypermodel,
objective=kt.Objective(
"val_score", "max"),
project_name=output_dir,
**tuner_kwargs)
elif mode == "bayesian":
self.tuner = BayesianOptimization(self.hypermodel,
objective=kt.Objective(
"val_score", "max"),
project_name=output_dir,
**tuner_kwargs)
elif mode == "hyperband":
self.tuner = Hyperband(self.hypermodel,
objective=kt.Objective("val_score", "max"),
project_name=output_dir,
**tuner_kwargs)
else:
pass
self.tuner.search_space_summary()
def _has_earlystopping(self, callbacks):
"""Check if EarlyStopping has been defined or not."""
if callbacks is None:
return False
for callback in callbacks:
if isinstance(callback, tf.keras.callbacks.EarlyStopping):
return True
return False
def history(self, number_of_trials=-1):
"""Returns the history of the model search."""
trials = self.tuner.oracle.get_best_trials(number_of_trials)
state = [trial.get_state() for trial in trials]
result = {}
result["score"] = [
state[i]["score"] for i in range(len(state))
if trials[i].score is not None
]
for i in range(len(state)):
if trials[i].score is not None:
keys = state[i]["metrics"]["metrics"].keys()
for key in keys:
if key != "score" and not key.startswith(
"val_") and key != "loss" and key != "trial":
cur_accuracy = state[i]["metrics"]["metrics"][key][
"observations"][0]["value"][0]
if "val_" + key in state[i]["metrics"]["metrics"].keys(
):
cur_val_accuracy = state[i]["metrics"]["metrics"][
"val_" + key]["observations"][0]["value"][0]
else:
cur_val_accuracy = None
# only update result if both key and val_key exist
if cur_val_accuracy:
if key not in result.keys():
result[key] = [cur_accuracy]
result["val_" + key] = [cur_val_accuracy]
else:
result[key].append(cur_accuracy)
result["val_" + key].append(cur_val_accuracy)
result["trial_size"] = [
state[i]["metrics"]["metrics"]["trial"]["observations"][0]["value"]
[0] for i in range(len(state)) if trials[i].score is not None
]
return result
def fit(self, *fit_args, **fit_kwargs):
"""Invokes tuner fit algorithm."""
callbacks = fit_kwargs.get("callbacks", None)
if callbacks is None:
callbacks = []
epochs = fit_kwargs.get("epochs", None)
if epochs is None:
epochs = 10
if not self._has_earlystopping(callbacks):
callbacks = callbacks + [
tf.keras.callbacks.EarlyStopping("val_loss",
patience=min(20, epochs // 5))
]
fit_kwargs["callbacks"] = callbacks
self.tuner.search(*fit_args, **fit_kwargs)
@staticmethod
def get_best_lr(qmodel):
"""Extracts best lr of model."""
return qmodel.optimizer.lr.numpy()
def get_best_model(self):
params = self.tuner.get_best_hyperparameters()[0]
q_model = self.tuner.hypermodel.build(params)
self.learning_rate = q_model.optimizer.lr.numpy()
return q_model
def get_learning_rate(self):
return self.learning_rate
objective='val_categorical_accuracy',
max_epochs=40,
project_name='tuning_categorical'
)
tuner.search_space_summary()
tuner.search(trainX, trainY,epochs=25,validation_data=(testX, testY),verbose=0)
tuner.results_summary()
best_model = tuner.get_best_models(num_models=1)[0]
# Evaluate the best model.
loss, accuracy = best_model.evaluate(testX, testY)
tuner.get_best_hyperparameters()[0].values
"""Pretty good results : 0.898 vs 0.7551 categorical accuracy
## Testing different applications (ResNet, Inception)
"""
from keras.applications import VGG16
from keras import Model
from keras.layers import AveragePooling2D, Input, Flatten, Dense, Dropout
from keras.optimizers import Adam
from sklearn.model_selection import train_test_split
def imagenet_model(lr=1e-3, epochs=25, dropout_rate=0.5,dense_nodes=64,base_model=VGG16):
baseModel = base_model(weights="imagenet", include_top=False,
input_tensor=Input(shape=(224, 224, 3)))
seed=42,
hyperband_iterations=3)
#print total search space
tuner_hb.search_space_summary()
#search through the total search space
tuner_hb.search(train_generator,
epochs=500,
verbose=1,
validation_data=(img_test, mask_test))
#save best model and hyperparameters
best_model = tuner_hb.get_best_models(1)[0]
best_hyperparameters = tuner_hb.get_best_hyperparameters(1)[0]
print(tuner_hb.get_best_hyperparameters(1))
#
model_json = best_model.to_json()
with open("hp_bce_all_basicUNet_model.json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
best_model.save_weights("hp_bce_all_tune_basicUNet_tuner_model.h5")
with open('best_LGG_basicUNet_Param.txt', 'w') as f:
print(best_hyperparameters, file=f)
print("Saved model to disk")
print(best_hyperparameters)
tuner_hb.results_summary() #print best 10 models
model.add(Dense(160, activation='relu'))
model.add(Dense(6, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['acc'])
return model
tuner = Hyperband(build_model,
objective='val_acc',
max_epochs=20,
directory='vgg_transfer_param',
hyperparameters=hp,
project_name='vgg_transfer_project')
tuner.search(datagen_train, epochs=10, validation_data=datagen_valid)
best_hps = tuner.get_best_hyperparameters(1)[0]
print(best_hps.values)
# model = build_model()
# # 编译模型
# # model.compile(optimizer='Adam',
# # loss='categorical_crossentropy',
# # metrics=[tf.keras.metrics.categorical_accuracy])
#
# history_callback = LossHistory()
# checkpoint = ModelCheckpoint("trained_model.h5", monitor='val_categorical_accuracy', verbose=1, save_best_only=True, mode='max', period=2)
# history = model.fit_generator(
# generator=datagen_train,
# validation_data=datagen_valid,
# epochs=30,
# validation_freq=1,
# callbacks=[history_callback, checkpoint]
def tune():
# train_generator = RobertaDataGenerator(config.train_path)
# train_dataset = tf.data.Dataset.from_generator(train_generator.generate,
# output_types=({'ids': tf.int32, 'att': tf.int32, 'tti': tf.int32},
# {'sts': tf.int32, 'ets': tf.int32}))
# train_dataset = train_dataset.padded_batch(32,
# padded_shapes=({'ids': [None], 'att': [None], 'tti': [None]},
# {'sts': [None], 'ets': [None]}),
# padding_values=({'ids': 1, 'att': 0, 'tti': 0},
# {'sts': 0, 'ets': 0}))
# train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)
#
# val_generator = RobertaDataGenerator(config.validation_path)
# val_dataset = tf.data.Dataset.from_generator(val_generator.generate,
# output_types=({'ids': tf.int32, 'att': tf.int32, 'tti': tf.int32},
# {'sts': tf.int32, 'ets': tf.int32}))
# val_dataset = val_dataset.padded_batch(32,
# padded_shapes=({'ids': [None], 'att': [None], 'tti': [None]},
# {'sts': [None], 'ets': [None]}),
# padding_values=({'ids': 1, 'att': 0, 'tti': 0},
# {'sts': 0, 'ets': 0}))
# val_dataset = val_dataset.prefetch(tf.data.experimental.AUTOTUNE)
train_dataset = RobertaData(Config.train_path, 'train').get_data()
val_dataset = RobertaData(Config.validation_path, 'val').get_data()
tuner = Hyperband(get_tunable_roberta,
objective='val_loss',
max_epochs=10,
factor=3,
hyperband_iterations=3,
seed=Config.seed,
directory='tuner_logs',
project_name='feat_roberta')
tuner.search_space_summary()
callbacks = [
tf.keras.callbacks.ReduceLROnPlateau(patience=2, verbose=1),
tf.keras.callbacks.EarlyStopping(patience=3, verbose=1)
]
tuner.search(train_dataset[0],
train_dataset[1],
epochs=10,
verbose=1,
callbacks=callbacks,
batch_size=32,
validation_data=val_dataset)
tuner.results_summary()
best_hps: List[HyperParameters] = tuner.get_best_hyperparameters(
num_trials=5)
for hp in best_hps:
print(f'{hp.values}\n')
model = tuner.hypermodel.build(best_hps[0])
tf.keras.utils.plot_model(model,
to_file='best_hp_tuned_model.png',
show_shapes=True,
show_layer_names=True,
expand_nested=True)
model.summary()
