def search_bestCNN(self, X, Y, testX, testY, epochs=50, max_trails=20, batch_size=64, project_name='A1'): tuner = RandomSearch(self._build_CNN, objective='val_accuracy', max_trials=max_trails, executions_per_trial=1, directory='tunerlog', project_name=project_name) tuner.search(x=X, y=Y, epochs=epochs, batch_size=batch_size, validation_data=(testX, testY), callbacks=[ tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5) ], verbose=2) tuner.search_space_summary() print(tuner.results_summary()) print('best_hyperparameters') print(tuner.get_best_hyperparameters()[0].values) return tuner.get_best_models()
def train_resnet50(): # generating data from existing images train_datagenerator, test_datagenerator = data_generator() # getting the model and callback from model.py lr_reduction = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, min_lr=1e-5) tuner = RandomSearch(project_name=os.path.join(LOGS, 'trial_2/resnet_50'), max_trials=3, executions_per_trial=5, hypermodel=vgg_16, objective='val_accuracy') tuner.search(train_datagenerator, epochs=10, callbacks=[lr_reduction], validation_data=test_datagenerator) best_hps = tuner.get_best_hyperparameters(num_trials=1)[0] model = tuner.hypermodel.build(best_hps) model.fit_generator(train_datagenerator, epochs=EPOCHS, validation_data=test_datagenerator) return model
def build_model(hp): x_train = np.random.random((100, 28, 28)) y_train = np.random.randint(10, size=(100, 1)) x_test = np.random.random((20, 28, 28)) y_test = np.random.randint(10, size=(20, 1)) model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(hp.Choice('dropout_rate', values=[0.2, 0.4])), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile( optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) return model tuner = RandomSearch(build_model, objective='accuracy', max_trials=1, executions_per_trial=1, seed=1) tuner.search(x_train, y_train, epochs=1) self.assertEqual(0.4, tuner.get_best_hyperparameters(1)[0].get('dropout_rate'))
def tuneCNN( X_train, X_test, height, width, num_classes, patience=1, executions_per_trial=1, seed=42, max_trials=3, objective='val_accuracy', directory='my_dir', epochs=10, verbose=0, test_size=0.2): # creates hypermodel object based on the num_classes and the input shape hypermodel = CNNHyperModel(input_shape=( height, width, 3), num_classes=num_classes) # # tuners, establish the object to look through the tuner search space tuner = RandomSearch( hypermodel, objective=objective, seed=seed, max_trials=max_trials, executions_per_trial=executions_per_trial, directory=directory, ) # searches the tuner space defined by hyperparameters (hp) and returns the # best model tuner.search(X_train, validation_data=X_test, callbacks=[tf.keras.callbacks.EarlyStopping(patience=patience)], epochs=epochs, verbose=verbose) # best hyperparamters hyp = tuner.get_best_hyperparameters(num_trials=1)[0] #hyp = tuner.oracle.get_best_trials(num_trials=1)[0].hyperparameters.values #best_hps = np.stack(hyp).astype(None) history = tuner_hist( X_train, X_test, tuner, hyp, img=1, epochs=epochs, verbose=verbose, test_size=test_size) """ Return: models[0] : best model obtained after tuning best_hps : best Hyperprameters obtained after tuning, stored as array history : history of the data executed from the given model """ return tuner.get_best_models(1)[0], hyp, history
def get_best_nn(data, num_output=1, **tuner_kw): """ Find the "best" model based on `MyHyperModel` class. Parameters ---------- data: numpy.array or similar The train and validation data to be used by the hyper parameter tuner. num_output: int, optional The number of outputs for our NN. 1 default for regression. tuner_kw: dictionary A dictionary of parameters to be `RandomSearch` tuner. Returns ------- The trained model instance with the "optimised" parameters. """ # Load encoded data enc_data = data_encode(data, encoder='CatBoostEncoder') x_train_enc, y_train = enc_data.get('train_data') x_val_enc, y_val = enc_data.get('test_data') # Create an instance of the `MyHyperModel` class hyper_model = MyHyperModel(num_output=num_output, nun_features=int(x_train_enc.shape[1])) # Default tuner params default_tuner_params = { 'objective': 'val_loss', 'max_trials': 10, 'directory': 'keras_tuner_output', # Directory for logs, checkpoints, etc 'project_name': 'sgsc' } # Default is utils/keras_tuner_output # Update tuner params tuner_params = {**default_tuner_params, **tuner_kw} # Initialise tuner and run it tuner = RandomSearch(hyper_model, **tuner_params) # Check about seed!! We need to define it? or does it use numpy's by default? tuner.search( x_train_enc, y_train, epochs=5, # Default number of epochs validation_data=(x_val_enc, y_val), verbose=0) # Get best model best_hp = tuner.get_best_hyperparameters()[0] best_model = tuner.hypermodel.build(best_hp) return best_model, best_hp
def find_best_NN(x_train, y_train): tuner = RandomSearch(build_model, objective="loss", max_trials=10, executions_per_trial=1) print("\n\n\n") print('[INFO] start searching') tuner.search(x_train, y_train, batch_size=100, epochs=10, validation_split=0.2) print("\n\n\nRESULTS SUMMARY") tuner.results_summary() print("\n\n\n") print("\n\n\nHERE IS THE BEST MODEL\n\n\n") best_params = tuner.get_best_hyperparameters()[0] best_model = tuner.hypermodel.build(best_params) best_model.summary() return best_model
def find_best_NN(x_train, y_train): # создаю тюнер, который сможет подобрать оптимальную архитектуру модели tuner = RandomSearch(build_model, objective="val_mae", max_trials=40, executions_per_trial=1,) print("\n\n\n") # начинается автоматический подбор гиперпараметров print('[INFO] start searching') tuner.search(x_train, y_train, batch_size=500, epochs=150, validation_split=0.3) # выбираем лучшую модель print("\n\n\nRESULTS SUMMARY") tuner.results_summary() print("\n\n\n") # получаем лучшую модель print("\n\n\nHERE IS THE BEST MODEL\n\n\n") best_params = tuner.get_best_hyperparameters()[0] best_model = tuner.hypermodel.build(best_params) best_model.summary() return best_model
def tune(): tuner = RandomSearch(tuner_model, objective="val_accuracy", max_trials=100, executions_per_trial=1, directory=LOG_DIR, project_name='final_year_project') tuner.search(x=x_train, y=y_train, epochs=3, batch_size=64, validation_data=(x_test, y_test)) with open("tuner.pkl", "wb") as f: pickle.dump(tuner, f) tuner = pickle.load(open("tuner.pkl", "rb")) print(tuner.get_best_hyperparameters()[0].values) print(tuner.results_summary()) print(tuner.get_best_models()[0].summary())
return model tuner = RandomSearch( build_model, objective='val_accuracy', max_trials=1, # how many model variations to test? executions_per_trial=1, # how many trials per variation? (same model could perform differently) directory=LOG_DIR) tuner.search(x=x_train, y=y_train, verbose=2, # just slapping this here bc jupyter notebook. The console out was getting messy. epochs=1, batch_size=64, #callbacks=[tensorboard], # if you have callbacks like tensorboard, they go here. validation_data=(x_test, y_test)) tuner.results_summary() tuner.get_best_hyperparameters()[0].values tuner.get_best_models()[0].summary() with open(f"tuner_{int(time.time())}.pkl", "wb") as f: pickle.dump(tuner, f) #TO LOAD DATA #tuner = pickle.load(open("tuner_1576628824.pkl","rb"))
def main(): logging.getLogger().setLevel(logging.INFO) parser = argparse.ArgumentParser(description='Keras Tuner HP search') parser.add_argument('--epochs', type=int, default=1) parser.add_argument( '--steps-per-epoch', type=int, default=-1) # if set to -1, don't override the normal calcs for this parser.add_argument('--tuner-proj', required=True) parser.add_argument('--bucket-name', required=True) parser.add_argument('--tuner-dir', required=True) parser.add_argument('--tuner-num', required=True) parser.add_argument('--respath', required=True) parser.add_argument('--executions-per-trial', type=int, default=2) parser.add_argument('--max-trials', type=int, default=20) parser.add_argument('--num-best-hps', type=int, default=2) parser.add_argument('--data-dir', default='gs://aju-dev-demos-codelabs/bikes_weather/') args = parser.parse_args() logging.info('Tensorflow version %s', tf.__version__) TRAIN_DATA_PATTERN = args.data_dir + "train*" EVAL_DATA_PATTERN = args.data_dir + "test*" train_batch_size = TRAIN_BATCH_SIZE eval_batch_size = 1000 if args.steps_per_epoch == -1: # calc based on dataset size steps_per_epoch = NUM_EXAMPLES // train_batch_size else: steps_per_epoch = args.steps_per_epoch logging.info('using %s steps per epoch', steps_per_epoch) logging.info('using train batch size %s', train_batch_size) train_dataset = bwmodel.read_dataset(TRAIN_DATA_PATTERN, train_batch_size) eval_dataset = bwmodel.read_dataset( EVAL_DATA_PATTERN, eval_batch_size, tf.estimator.ModeKeys.EVAL, eval_batch_size * 100 * STRATEGY.num_replicas_in_sync) logging.info('executions per trial: %s', args.executions_per_trial) # TODO: parameterize retries = 0 num_retries = 5 sleep_time = 5 while retries < num_retries: try: tuner = RandomSearch( # tuner = Hyperband( create_model, objective='val_mae', # max_epochs=10, # hyperband_iterations=2, max_trials=args.max_trials, distribution_strategy=STRATEGY, executions_per_trial=args.executions_per_trial, directory=args.tuner_dir, project_name=args.tuner_proj) break except Exception as e: logging.warning(e) logging.info('sleeping %s seconds...', sleep_time) time.sleep(sleep_time) retries += 1 sleep_time *= 2 logging.info("search space summary:") logging.info(tuner.search_space_summary()) logging.info("hp tuning model....") tuner.search( train_dataset, validation_data=eval_dataset, validation_steps=eval_batch_size, epochs=args.epochs, steps_per_epoch=steps_per_epoch, ) best_hps = tuner.get_best_hyperparameters(args.num_best_hps) best_hps_list = [best_hps[i].values for i in range(args.num_best_hps)] logging.info('best_hps_list: %s', best_hps_list) best_hp_values = json.dumps(best_hps_list) logging.info('best hyperparameters: %s', best_hp_values) storage_client = storage.Client() logging.info('writing best results to %s', args.respath) bucket = storage_client.get_bucket(args.bucket_name) logging.info('using bucket %s: %s, path %s', args.bucket_name, bucket, args.respath) blob = bucket.blob(args.respath) blob.upload_from_string(best_hp_values)
def search( epochs: int, batch_size: int, n_trials: int, execution_per_trial: int, project: Text, do_cleanup: bool, ): set_seed(SEED) dir_to_clean = os.path.join(SEARCH_DIR, project) if do_cleanup and os.path.exists(dir_to_clean): shutil.rmtree(dir_to_clean) # first 80% for train. remaining 20% for val & test dataset for final eval. ds_tr, ds_val, ds_test = tfds.load( name="mnist", split=["train[:80%]", "train[-20%:]", "test"], data_dir="mnist", shuffle_files=False, ) ds_tr = prepare_dataset(ds_tr, batch_size, shuffle=True, drop_remainder=True) ds_val = prepare_dataset(ds_val, batch_size, shuffle=False, drop_remainder=False) ds_test = prepare_dataset(ds_test, batch_size, shuffle=False, drop_remainder=False) tuner = RandomSearch( build_model, objective="val_accuracy", max_trials=n_trials, executions_per_trial=execution_per_trial, directory=SEARCH_DIR, project_name=project, ) # ? add callbacks tuner.search( ds_tr, epochs=epochs, validation_data=ds_val, ) best_model: tf.keras.Model = tuner.get_best_models(num_models=1)[0] best_model.build((None, DEFAULTS["num_features"])) results = best_model.evaluate(ds_test, return_dict=True) tuner.results_summary(num_trials=1) best_hyperparams = tuner.get_best_hyperparameters(num_trials=1) print(f"Test results: {results}") output = {"results": results, "best_hyperparams": best_hyperparams} with open("search_results.pickle", "wb") as f: pickle.dump(output, f)
def train_data(symbol, timeframe): df = mt.history("EURUSD", "M1", 2) print("traning", df) df.isnull().sum().sum() # there are no nans df.fillna(method="ffill", inplace=True) df = df.loc[~df.index.duplicated(keep='first')] # indicators df = Indicators(df) df = df.dropna() df = df.fillna(method="ffill") df = df.dropna() df.sort_index(inplace=True) df['target'] = list(map(classify, df['return'], df['return_next'])) print(df) df.dropna(inplace=True) df['target'].value_counts() df.dropna(inplace=True) df = df.astype('float32') df = preprocess_df(df) train_x, train_y = df validation_x, validation_y = df train_y = np.asarray(train_y) validation_y = np.asarray(validation_y) print(('%% of Class0 : %f Sell' % (np.count_nonzero(train_y == 0) / float(len(train_y))))) print(('%% of Class1 : %f Buy' % (np.count_nonzero(train_y == 1) / float(len(train_y))))) def build_model(hp): model = Sequential() model.add( LSTM(hp.Int('units', min_value=10, max_value=50, step=1), input_shape=(train_x.shape[1:]), return_sequences=True)) model.add(Dropout(0.1)) model.add(BatchNormalization()) model.add( LSTM(units=hp.Int('units', min_value=10, max_value=50, step=1), return_sequences=True)) model.add(Dropout(0.2)) model.add(BatchNormalization()) model.add( LSTM(units=hp.Int('units', min_value=10, max_value=50, step=1))) model.add(Dropout(0.2)) model.add(BatchNormalization()) model.add( Dense(hp.Int('units', min_value=10, max_value=50, step=1), activation='relu')) model.add(Dropout(0.2)) model.add(Dense(2, activation='softmax')) # Compile model model.compile(optimizer=Adam(hp.Choice('learning_rate', values=[1e-2])), loss='sparse_categorical_crossentropy', metrics=['accuracy']) return model tuner = RandomSearch(build_model, objective='val_accuracy', max_trials=10, executions_per_trial=1, directory='TUN', project_name='IQOTC') # tuner.search_space_summary() stop_early = EarlyStopping(monitor='val_loss', patience=15) tuner.search(train_x, train_y, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.2, verbose=1, callbacks=[stop_early]), best_hps = tuner.get_best_hyperparameters(num_trials=1)[0] print(f""" The optimal number of units layer is {best_hps.get('units')} and the optimal learning rate for the optimizer is {best_hps.get('learning_rate')}. """) filepath = "ThesisBrain" # Build the model with the optimal hyperparameters and train it on the data for 50 epochs model = tuner.hypermodel.build(best_hps) history = model.fit(train_x, train_y, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.2, verbose=1) val_acc_per_epoch = history.history['val_accuracy'] best_epoch = val_acc_per_epoch.index(max(val_acc_per_epoch)) + 1 print(('Best epoch: %d' % (best_epoch, ))) hypermodel = tuner.hypermodel.build(best_hps) scores = model.evaluate(validation_x, validation_y, verbose=0) print(("%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))) del model del history # Retrain the model hypermodel.fit(validation_x, validation_y, batch_size=BATCH_SIZE, epochs=best_epoch, verbose=1) hypermodel.save("models/{}.h5".format(filepath)) scores = hypermodel.evaluate(validation_x, validation_y, verbose=0) print(("%s: %.2f%%" % (hypermodel.metrics_names[1], scores[1] * 100))) scores = scores[1] * 100 return scores
def tuneReg(data, target, max_layers=10, min_layers=2, min_dense=32, max_dense=512, executions_per_trial=3, max_trials=3, epochs=10, activation='relu', step=32, verbose=0, test_size=0.2): # function build model using hyperparameter def build_model(hp): model = keras.Sequential() for i in range(hp.Int('num_layers', min_layers, max_layers)): model.add( Dense(units=hp.Int('units_' + str(i), min_value=min_dense, max_value=max_dense, step=step), activation=activation)) model.add(Dense(1)) model.compile(optimizer=keras.optimizers.Adam( hp.Choice('learning_rate', [1e-2, 1e-3, 1e-4])), loss='mean_squared_error') return model # random search for the model tuner = RandomSearch(build_model, objective='loss', max_trials=max_trials, executions_per_trial=executions_per_trial) # tuner.search_space_summary() # del data[target] X_train, X_test, y_train, y_test = train_test_split(data, target, test_size=0.2, random_state=49) # searches the tuner space defined by hyperparameters (hp) and returns the # best model tuner.search(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), callbacks=[tf.keras.callbacks.TensorBoard('my_dir')]) models = tuner.get_best_models(num_models=1) hyp = tuner.get_best_hyperparameters(num_trials=1)[0] #hyp = tuner.oracle.get_best_trials(num_trials=1)[0].hyperparameters.values #best_hps = np.stack(hyp).astype(None) history = tuner_hist(data, target, tuner, hyp, epochs=epochs, verbose=verbose, test_size=test_size) """ Return: models[0] : best model obtained after tuning best_hps : best Hyperprameters obtained after tuning, stored as map history : history of the data executed from the given model """ return models[0], hyp, history
def tuneClass(X, y, num_classes, max_layers=10, min_layers=2, min_dense=32, max_dense=512, executions_per_trial=3, max_trials=3, activation='relu', loss='categorical_crossentropy', metrics='accuracy', epochs=10, step=32, verbose=0, test_size=0.2): # function build model using hyperparameter le = preprocessing.LabelEncoder() y = tf.keras.utils.to_categorical(le.fit_transform(y), num_classes=num_classes) def build_model(hp): model = keras.Sequential() for i in range(hp.Int('num_layers', min_layers, max_layers)): model.add( Dense(units=hp.Int('units_' + str(i), min_value=min_dense, max_value=max_dense, step=step), activation=activation)) model.add(Dense(num_classes, activation='softmax')) model.compile(optimizer=keras.optimizers.Adam( hp.Choice('learning_rate', [1e-2, 1e-3, 1e-4])), loss=loss, metrics=[metrics]) return model # tuners, establish the object to look through the tuner search space tuner = RandomSearch(build_model, objective='loss', max_trials=max_trials, executions_per_trial=executions_per_trial, directory='models', project_name='class_tuned') # tuner.search_space_summary() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=49) # searches the tuner space defined by hyperparameters (hp) and returns the # best model tuner.search(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test)) models = tuner.get_best_models(num_models=1) hyp = tuner.get_best_hyperparameters(num_trials=1)[0] #hyp = tuner.oracle.get_best_trials(num_trials=1)[0].hyperparameters.values #best_hps = np.stack(hyp).astype(None) history = tuner_hist(X, y, tuner, hyp, epochs=epochs, verbose=verbose, test_size=test_size) """ Return: models[0] : best model obtained after tuning best_hps : best Hyperprameters obtained after tuning, stored as array history : history of the data executed from the given model """ return models[0], hyp, history
tuner = RandomSearch(trial, objective='val_acc', max_trials=1, executions_per_trial=1, directory=LOG_DIR) tuner.search(x=X_train, y=y_train, epochs=100, batch_size=50, shuffle='true', validation_data=(X_val, y_val)) model = RNN(input_shape=X_train.shape[1:], output_shape=n_classes, hyperparams=tuner.get_best_hyperparameters()[0].values).run() hist = model.fit(X_train, y_train, epochs=100, batch_size=50, shuffle='true', validation_data=(X_val, y_val)) model.summary() pickle.dump( hist.history, open('histories/' + sys.argv[1] + '_' + sys.argv[2] + '.pickle', 'wb')) model.save('models/model_' + sys.argv[1] + '_' + sys.argv[2] + '.h5')
def tuneReg(data, target, max_layers=10, min_layers=2, min_dense=32, max_dense=512, executions_per_trial=1, max_trials=5, epochs=10, activation='relu', directory='my_dir', step=32, verbose=0, test_size=0.2): # function build model using hyperparameter def build_model(hp): model = keras.Sequential() model.add( Dense(units=hp.Int('units_0', min_value=min_dense, max_value=max_dense, step=step), input_dim=data.shape[1], activation=activation)) for i in range(hp.Int('num_layers', min_layers, max_layers)): model.add( Dense(units=hp.Int('units_' + str(i + 1), min_value=min_dense, max_value=max_dense, step=step), activation=activation)) model.add( Dropout(rate=hp.Float('dropout_' + str(i), min_value=0.0, max_value=0.5, default=0.20, step=0.05))) model.add(Dense(1, activation='linear')) lrate = hp.Float('learning_rate', min_value=1e-5, max_value=1e-1, sampling='LOG', default=1e-3) model.compile(optimizer=hp.Choice( 'optimizer', values=[ keras.optimizers.Adam(learning_rate=lrate), keras.optimizers.SGD(learning_rate=lrate), keras.optimizers.RMSprop(learning_rate=lrate), keras.optimizers.Adamax(learning_rate=lrate) ]), loss=hp.Choice('loss', values=[ 'mean_squared_logarithmic_error', 'mean_squared_error', 'huber_loss', 'mean_absolute_error', 'cosine_similarity', 'log_cosh' ], default='mean_squared_error'), metrics=['accuracy']) return model # random search for the model tuner = RandomSearch(build_model, objective='val_accuracy', max_trials=max_trials, executions_per_trial=executions_per_trial, directory=directory) # tuner.search_space_summary() # del data[target] X_train, X_test, y_train, y_test = train_test_split(data, target, test_size=0.2, random_state=49) # searches the tuner space defined by hyperparameters (hp) and returns the # best model tuner.search(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), callbacks=[tf.keras.callbacks.TensorBoard('my_dir')]) models = tuner.get_best_models(num_models=1)[0] hyp = tuner.get_best_hyperparameters(num_trials=1)[0] history = tuner_hist(data, target, tuner, hyp, epochs=epochs, verbose=verbose, test_size=test_size) """ Return: models[0] : best model obtained after tuning best_hps : best Hyperprameters obtained after tuning, stored as map history : history of the data executed from the given model """ return models, hyp, history, X_test, y_test
tensorboard_callback = TensorBoard(log_dir, histogram_freq=1, profile_batch=0) start = datetime.now() tuner.search(train_imgs, train_labels, epochs=epochs, batch_size=batch_size, callbacks=[tensorboard_callback], validation_data=(test_imgs, test_labels)) print(f'Time taken to complete {epochs} epochs: {datetime.now() - start}') tuner.results_summary() with open(f"tuner_{int(datetime.now())}.pkl", "wb") as f: pickle.dump(tuner, f) """ tuner = pickle.load(open("tuner_1576628824.pkl","rb")) tuner.get_best_hyperparameters()[0].values tuner.get_best_models()[0].summary() """ ''' # Convolutional Neural Network cnn = Sequential( [ Conv2D(32, (3,3), activation='relu', input_shape=(32,32,3), padding='same'), MaxPooling2D((2,2)), Conv2D(64, (3, 3), activation='relu', padding='same'), MaxPooling2D((2, 2)), Conv2D(64, (3, 3), activation='relu', padding='same'), MaxPooling2D((2, 2)), Conv2D(32, (3, 3), activation='relu', padding='same'), Flatten(),
class AutoEncoder(): def __init__(self, df_source_info, df_fluxes, df_wavelengths): X = self._prepare_data(df_source_info, df_fluxes, df_wavelengths) objids = self.df_quasars['objid'].values print(f'objids = {objids}') X_train, X_test = train_test_split(X, 0.2) self.objids_train, self.objids_test = train_test_split(objids, 0.2) self.scaler = StandardScaler() X_train = self.scaler.fit_transform(X_train) X_test = self.scaler.transform(X_test) self.X_train = np.expand_dims(X_train, axis=2) self.X_test = np.expand_dims(X_test, axis=2) print(f'self.X_train = {self.X_train}') self.optimizer = Nadam(lr=0.001) def _prepare_data(self, df_source_info, df_fluxes, df_wavelengths): if "b'" in str(df_source_info['class'][0]): df_source_info = remove_bytes_from_class(df_source_info) self.df_quasars = df_source_info.loc[df_source_info['class'] == 'QSO'] quasar_objids = self.df_quasars['objid'].to_numpy() quasar_fluxes = df_fluxes.loc[df_fluxes['objid'].isin(quasar_objids)] X = np.delete(quasar_fluxes.values, 0, axis=1) X = X[:, 0::8] print(f'X.shape = {X.shape}') X = X[:, np.mod(np.arange(X[0].size),25)!=0] print(f'X.shape {X.shape}') wavelengths = df_wavelengths.to_numpy() wavelengths = wavelengths[::8] self.wavelengths = wavelengths[0:448] # plot_spectrum(X[0], wavelengths) return X def build_model(self, hp): hyperparameters = { 'layer_1_filters': hp.Choice('layer_1_filters', values=[16, 32, 64, 128, 256], default=64), 'layer_1_kernel_size': hp.Choice('layer_1_kernel_size', values=[3, 5, 7, 9, 11]), 'layer_2_filters': hp.Choice('layer_2_filters', values=[8, 16, 32, 64, 128], default=32), 'layer_2_kernel_size': hp.Choice('layer_2_kernel_size', values=[3, 5, 7, 9]), 'layer_3_filters': hp.Choice('layer_3_filters', values=[4, 8, 16, 32], default=32), 'layer_3_kernel_size': hp.Choice('layer_3_kernel_size', values=[3, 5, 7]), 'layer_4_filters': hp.Choice('layer_4_filters', values=[4, 8, 12, 16], default=16), 'layer_4_kernel_size': hp.Choice('layer_4_kernel_size', values=[3, 5]), 'layer_5_filters': hp.Choice('layer_5_filters', values=[2, 3, 4, 8], default=8), 'layer_5_kernel_size': hp.Choice('layer_5_kernel_size', values=[3]), 'optimizer': hp.Choice('optimizer', values=['adam', 'nadam', 'rmsprop']), 'last_activation': hp.Choice('last_activation', ['tanh']) } # ================================================================================== # # ==================================== ENCODER ===================================== # # ================================================================================== # input_layer = Input(shape=(self.X_train.shape[1], 1)) # encoder x = Conv1D(filters=hyperparameters['layer_1_filters'], kernel_size=hyperparameters['layer_1_kernel_size'], activation='relu', padding='same')(input_layer) x = MaxPooling1D(2)(x) x = Conv1D(filters=hyperparameters['layer_2_filters'], kernel_size=hyperparameters['layer_2_kernel_size'], activation='relu', padding='same')(x) x = MaxPooling1D(2)(x) x = Conv1D(filters=hyperparameters['layer_3_filters'], kernel_size=hyperparameters['layer_3_kernel_size'], activation='relu', padding='same')(x) x = MaxPooling1D(2)(x) x = Conv1D(filters=hyperparameters['layer_4_filters'], kernel_size=hyperparameters['layer_4_kernel_size'], activation='relu', padding='same')(x) x = MaxPooling1D(2)(x) x = Conv1D(filters=hyperparameters['layer_5_filters'], kernel_size=hyperparameters['layer_5_kernel_size'], activation='relu', padding='same')(x) encoded = MaxPooling1D(2, padding="same")(x) # ================================================================================== # # ==================================== DECODER ===================================== # # ================================================================================== # x = Conv1D(filters=hyperparameters['layer_5_filters'], kernel_size=hyperparameters['layer_5_kernel_size'], activation='relu', padding='same')(encoded) x = UpSampling1D(2)(x) x = Conv1D(filters=hyperparameters['layer_4_filters'], kernel_size=hyperparameters['layer_4_kernel_size'], activation='relu', padding='same')(x) x = UpSampling1D(2)(x) x = Conv1D(filters=hyperparameters['layer_3_filters'], kernel_size=hyperparameters['layer_3_kernel_size'], activation='relu', padding='same')(x) x = UpSampling1D(2)(x) x = Conv1D(filters=hyperparameters['layer_2_filters'], kernel_size=hyperparameters['layer_2_kernel_size'], activation='relu', padding='same')(x) x = UpSampling1D(2)(x) x = Conv1D(filters=hyperparameters['layer_1_filters'], kernel_size=hyperparameters['layer_1_kernel_size'], activation='relu', padding='same')(x) x = UpSampling1D(2)(x) decoded = Conv1D(1, 1, activation=hyperparameters['last_activation'], padding='same')(x) self.autoencoder = Model(input_layer, decoded) self.autoencoder.summary() self.autoencoder.compile(loss='mse', optimizer=hyperparameters['optimizer']) return self.autoencoder def train_model(self, epochs, batch_size=32): self.tuner = RandomSearch(self.build_model, objective='val_loss', max_trials=50, executions_per_trial=1, directory='logs/keras-tuner/', project_name='autoencoder') self.tuner.search_space_summary() self.tuner.search(x=self.X_train, y=self.X_train, epochs=24, batch_size=32, validation_data=(self.X_test, self.X_test), callbacks=[EarlyStopping('val_loss', patience=3)]) self.tuner.results_summary() def evaluate_model(self): best_model = self.tuner.get_best_models(1)[0] best_model.save('best_autoencoder_model') best_hyperparameters = self.tuner.get_best_hyperparameters(1)[0] print(f'best_model = {best_model}') print(f'best_hyperparameters = {self.tuner.results_summary()[0]}') nth_qso = 24 X_test = np.squeeze(self.X_test, axis=2) preds = best_model.predict(self.X_test) preds = self.scaler.inverse_transform(np.squeeze(preds, axis=2)) original = self.scaler.inverse_transform(X_test) qso_ra = self.df_quasars.loc[self.df_quasars['objid'] == self.objids_test[nth_qso]]['ra'].values[0] qso_dec = self.df_quasars.loc[self.df_quasars['objid'] == self.objids_test[nth_qso]]['dec'].values[0] qso_plate = self.df_quasars.loc[self.df_quasars['objid'] == self.objids_test[nth_qso]]['plate'].values[0] qso_z = self.df_quasars.loc[self.df_quasars['objid'] == self.objids_test[nth_qso]]['z'].values[0] plotify = Plotify(theme='ugly') _, axs = plotify.get_figax(nrows=2, figsize=(8, 8)) axs[0].plot(self.wavelengths, original[nth_qso], color=plotify.c_orange) axs[1].plot(self.wavelengths, preds[nth_qso], color=plotify.c_orange) axs[0].set_title(f'ra = {qso_ra}, dec = {qso_dec}, z = {qso_z}, plate = {qso_plate}', fontsize=14) axs[1].set_title(f'Autoencoder recreation') axs[0].set_ylabel(r'$F_{\lambda[10^{-17} erg \: cm^{-2}s^{-1} Å^{-1}]}$', fontsize=14) axs[1].set_ylabel(r'$F_{\lambda[10^{-17} erg \: cm^{-2}s^{-1} Å^{-1}]}$', fontsize=14) axs[1].set_xlabel('Wavelength (Å)') plt.subplots_adjust(hspace=0.4) # plt.savefig('plots/autoencoder_gaussian', facecolor=plotify.c_background, dpi=180) plt.show() return preds
plt.xlabel('EPOCH') plt.ylabel('Accruacy') plt.show() random_tuner = RandomSearch(hypermodel, objective='accuracy', max_trials=10, seed=10, project_name='divorce test') random_tuner.search(X_train.values, y_train.values.flatten(), epochs=10, validation_data=(X_test.values, y_test.values.flatten())) random_params = random_tuner.get_best_hyperparameters()[0] random_model = random_tuner.hypermodel.build(params) random_model.fit(X.values, y.values.flatten(), epochs=15) random_accuracy_df = pd.DataFrame(random_model.history.history) random_accuracy_df[['loss', 'accuracy']].plot() plt.title('Loss & Accuracy Per EPOCH For Random Model') plt.xlabel('EPOCH') plt.ylabel('Accruacy') plt.show() bayesian_tuner = BayesianOptimization(hypermodel, objective='accuracy',
activation=hp.Choice('activation', values=['relu', 'sigmoid', 'tanh']), name='recurrent_layer')) model.add(Dense(units=1, activation='linear', name='output_layer')) model.compile(optimizer=SGD(learning_rate=hp.Choice( 'learning_rate', values=list(np.logspace(-10, -0.2, base=2, num=15))), momentum=hp.Choice('momentum', values=list( np.logspace(-10, -0.1, base=2, num=15)))), loss='mse', metrics=['mse']) return model tuner = RandomSearch(build_model, objective='loss', max_trials=30, executions_per_trial=1, seed=2020) tuner.search_space_summary() tuner.search(X_train, y_train, epochs=30, validation_split=0.2) tuner.get_best_hyperparameters()
SEED = 10 direc = 'logs' from kerastuner.tuners import Hyperband turner = RandomSearch(build_model, objective="val_accuracy", max_trials=3, seed=SEED, directory=direc, executions_per_trial=3) print(turner.search_space_summary()) turner.search(x_train, y_train, epochs=50, batch_size=24, validation_data=(x_test, y_test)) print(turner.results_summary()) model = turner.get_best_models()[0] print(model.summary()) model.save("HP_model.h5") print(turner.get_best_hyperparameters()[0].values) hps = turner.oracle.get_best_trials(num_trials=1)[0].hyperparameters model = build_model(hps) print(model.summary())
tuner = RandomSearch( build_model, objective="val_accuracy", max_trials=15, # how many times change the model randomly executions_per_trial=1 # how many times to train the model selected ) tuner.search(x=X_train, y=y_train, epochs=20, batch_size=64, validation_data=(X_test, y_test)) best_hps = tuner.get_best_hyperparameters( num_trials=1)[0] # save hyperparameters when the val_accuracy is highest. print( "The hyperparameter search is complete. The optimal number of units in the first densely-connected layer is %d, " "the number of layers is %d and the optimal learning rate for the optimizer is %f." % (best_hps.get('input_units'), best_hps.get('learning_rate'), best_hps.get('n_layers'))) # model = tuner.hypermodel.build( best_hps) # save the model the best model among best_hps model.save('save.h5') model.fit(X_train, y_train, batch_size=64, epochs=5,
tuner = RandomSearch( build_model, # 위에 def build_model 만들겁니다. objective="val_accuracy", max_trials=15, # 몇번이나 랜덤하게 모델을 학습시킬지 executions_per_trial=1 # 같은 모델을 몇번이나 학습시킬지 ) tuner.search(x=X_train, y=y_train, epochs=20, batch_size=64, validation_data=(X_test, y_test)) best_hps = tuner.get_best_hyperparameters( num_trials=1)[0] # val_accuracy 가 가장 높았을때의 hyperparameters 를 저장한다. print( "The hyperparameter search is complete. The optimal number of units in the first densely-connected layer is %d, " "the number of layers is %d and the optimal learning rate for the optimizer is %f." % (best_hps.get('input_units'), best_hps.get('learning_rate'), best_hps.get('n_layers'))) # model = tuner.hypermodel.build( best_hps) # 위에서 bext_hps 에 저장한 높은 정확성이 있는 모델을 model에 저장한다. model.save('save.h5') model.fit(X_train, y_train, batch_size=64, epochs=5,
def createModel(self): self.model_instance += 1 clear_session() features, label = self.getDataset() X_train, y_train = self.createLag(features, label) X_train = X_train[:, self.lags] learning_rate = float(self.hyperparameters["Learning_Rate"].get()) momentum = float(self.hyperparameters["Momentum"].get()) optimizers = { "Adam": Adam(learning_rate=learning_rate), "SGD": SGD(learning_rate=learning_rate, momentum=momentum), "RMSprop": RMSprop(learning_rate=learning_rate, momentum=momentum) } shape = (X_train.shape[1], X_train.shape[2]) model_choice = self.model_var.get() if not self.do_optimization: model = Sequential() model.add(Input(shape=shape)) if model_choice == 0: model.add(Flatten()) layers = self.no_optimization_choice_var.get() for i in range(layers): neuron_number = self.neuron_numbers_var[i].get() activation_function = self.activation_var[i].get() if model_choice == 0: model.add(Dense(neuron_number, activation=activation_function, kernel_initializer=GlorotUniform(seed=0))) elif model_choice == 1: model.add(Conv1D(filters=neuron_number, kernel_size=2, activation=activation_function, kernel_initializer=GlorotUniform(seed=0))) model.add(MaxPooling1D(pool_size=2)) elif model_choice == 2: if i == layers-1: model.add(LSTM(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) else: model.add(LSTM(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) elif model_choice == 3: if i == layers-1: model.add(Bidirectional(LSTM(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))) model.add(Dropout(0.2)) else: model.add(Bidirectional(LSTM(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))) model.add(Dropout(0.2)) elif model_choice == 4: if i == layers-1: model.add(SimpleRNN(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) else: model.add(SimpleRNN(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) elif model_choice == 5: if i == layers-1: model.add(GRU(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) else: model.add(GRU(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) if model_choice == 1: model.add(Flatten()) model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0))) model.add(Dense(1, activation=self.output_activation.get(), kernel_initializer=GlorotUniform(seed=0))) model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get()) history = model.fit(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get(), verbose=1, shuffle=False) loss = history.history["loss"][-1] self.train_loss.set(loss) elif self.do_optimization: layer = self.optimization_choice_var.get() if model_choice == 0: def build_model(hp): model = Sequential() model.add(Input(shape=shape)) model.add(Flatten()) for i in range(layer): n_min = self.neuron_min_number_var[i].get() n_max = self.neuron_max_number_var[i].get() step = int((n_max - n_min)/4) model.add(Dense(units=hp.Int('MLP_'+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu')) model.add(Dense(1)) model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get()) return model name = str(self.model_instance) + ". MLP" elif model_choice == 1: def build_model(hp): model = Sequential() model.add(Input(shape=shape)) for i in range(layer): n_min = self.neuron_min_number_var[i].get() n_max = self.neuron_max_number_var[i].get() step = int((n_max-n_min)/4) model.add(Conv1D(filters=hp.Int("CNN_"+str(i), min_value=n_min, max_value=n_max, step=step), kernel_size=2, activation="relu", kernel_initializer=GlorotUniform(seed=0))) model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0))) model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0))) model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get()) return model name = str(self.model_instance) + ". CNN" elif model_choice == 2: def build_model(hp): model = Sequential() model.add(Input(shape=shape)) for i in range(layer): n_min = self.neuron_min_number_var[i].get() n_max = self.neuron_max_number_var[i].get() step = int((n_max - n_min)/4) model.add(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0))) if i == layer-1: model.add(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0))) model.add(Dense(1)) model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get()) return model name = str(self.model_instance) + ". LSTM" elif model_choice == 3: def build_model(hp): model = Sequential() model.add(Input(shape=shape)) for i in range(layer): n_min = self.neuron_min_number_var[i].get() n_max = self.neuron_max_number_var[i].get() step = int((n_max - n_min)/4) model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0)))) if i == layer-1: model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0)))) model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0))) model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get()) return model name = str(self.model_instance) + ". Bi-LSTM" tuner = RandomSearch(build_model, objective='loss', max_trials=25, executions_per_trial=2, directory=self.runtime, project_name=name) tuner.search(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get()) hps = tuner.get_best_hyperparameters(num_trials = 1)[0] model = tuner.hypermodel.build(hps) history = model.fit(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get(), verbose=1) loss = history.history["loss"][-1] self.train_loss.set(loss) for i in range(layer): if model_choice == 0: self.best_model_neurons[i].set(model.get_layer(index=i+1).get_config()["units"]) elif model_choice == 1: self.best_model_neurons[i].set(model.get_layer(index=(2*i)).get_config()["filters"]) elif model_choice == 2: self.best_model_neurons[i].set(model.get_layer(index=i).get_config()["units"]) elif model_choice == 3: self.best_model_neurons[i].set(model.get_layer(index=i).get_config()["layer"]["config"]["units"]) model.summary() self.model = model
overwrite=True) tuner.search_space_summary() tuner.search(train_data, train_labels, epochs=20, validation_data=(test_data, test_labels)) tuner.results_summary() # get bets model best_model = tuner.get_best_models(num_models=1)[0] # get the optimal hyperparameters best_hps = tuner.get_best_hyperparameters(num_trials=1)[0] print(f""" The hyperparameter search is complete. - optimal number of units in the 0 layer is {best_hps.get('units_0')} - optimal number of units in the 1 densely-connected layer is {best_hps.get('units_1')} - optimal number of units in the 2 densely-connected layer is {best_hps.get('units_2')} - optimal number of units in the 3 densely-connected layer is {best_hps.get('units_3')} - optimal number of units in the 4 densely-connected layer is {best_hps.get('units_4')} - optimal number of units in the 5 densely-connected layer is {best_hps.get('units_5')} - optimal activation is is {best_hps.get('dense_activation')} - optimal dropout is {best_hps.get('dropout')} - the optimal learning rate for the optimizer is {best_hps.get('learning_rate')}. """) # Build the model with the optimal hyperparameters and train it on the data