x_train = np.asarray(x_train) y_train = np.asarray(y_train) x_predic = x_test.iloc[[1]] x_test = np.asarray(x_test) y_test = np.asarray(y_test) #hparam tunning HP_NUM_UNITS_ONE = hp.HParam('num_units_one', hp.Discrete([5, 10, 20])) HP_NUM_UNITS_TWO = hp.HParam('num_units_two', hp.Discrete([10, 20, 40])) HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd'])) METRIC_ACCURACY = 'accuracy' with tf.summary.create_file_writer(output_dir).as_default(): hp.hparams_config( hparams=[HP_NUM_UNITS_ONE, HP_NUM_UNITS_TWO, HP_OPTIMIZER], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')], ) def train_test_model(hparams): model = tf.keras.models.Sequential() model.add( layers.Dense(hparams[HP_NUM_UNITS_ONE], activation='relu', input_shape=(13, ))) model.add(layers.Dense(hparams[HP_NUM_UNITS_TWO], activation='relu')) model.add(layers.Dense(1, activation='sigmoid')) model.compile( optimizer=hparams[HP_OPTIMIZER], loss='binary_crossentropy', metrics=['accuracy'],
def random_hparam_search(cfg, data, callbacks, log_dir): ''' Conduct a random hyperparameter search over the ranges given for the hyperparameters in config.yml and log results in TensorBoard. Model is trained x times for y random combinations of hyperparameters. :param cfg: Project config :param data: Dict containing the partitioned datasets :param callbacks: List of callbacks for Keras model (excluding TensorBoard) :param log_dir: Base directory in which to store logs :return: (Last model trained, resultant test set metrics, test data generator) ''' # Define HParam objects for each hyperparameter we wish to tune. hp_ranges = cfg['HP_SEARCH']['RANGES'] HPARAMS = [] HPARAMS.append( hp.HParam('KERNEL_SIZE', hp.Discrete(hp_ranges['KERNEL_SIZE']))) HPARAMS.append( hp.HParam('MAXPOOL_SIZE', hp.Discrete(hp_ranges['MAXPOOL_SIZE']))) HPARAMS.append( hp.HParam('INIT_FILTERS', hp.Discrete(hp_ranges['INIT_FILTERS']))) HPARAMS.append( hp.HParam( 'FILTER_EXP_BASE', hp.IntInterval(hp_ranges['FILTER_EXP_BASE'][0], hp_ranges['FILTER_EXP_BASE'][1]))) HPARAMS.append( hp.HParam('NODES_DENSE0', hp.Discrete(hp_ranges['NODES_DENSE0']))) HPARAMS.append( hp.HParam( 'CONV_BLOCKS', hp.IntInterval(hp_ranges['CONV_BLOCKS'][0], hp_ranges['CONV_BLOCKS'][1]))) HPARAMS.append(hp.HParam('DROPOUT', hp.Discrete(hp_ranges['DROPOUT']))) HPARAMS.append( hp.HParam('LR', hp.RealInterval(hp_ranges['LR'][0], hp_ranges['LR'][1]))) HPARAMS.append(hp.HParam('OPTIMIZER', hp.Discrete(hp_ranges['OPTIMIZER']))) HPARAMS.append(hp.HParam('L2_LAMBDA', hp.Discrete(hp_ranges['L2_LAMBDA']))) HPARAMS.append( hp.HParam('BATCH_SIZE', hp.Discrete(hp_ranges['BATCH_SIZE']))) HPARAMS.append( hp.HParam('IMB_STRATEGY', hp.Discrete(hp_ranges['IMB_STRATEGY']))) # Define test set metrics that we wish to log to TensorBoard for each training run HP_METRICS = [ hp.Metric(metric, display_name='Test ' + metric) for metric in cfg['HP_SEARCH']['METRICS'] ] # Configure TensorBoard to log the results with tf.summary.create_file_writer(log_dir).as_default(): hp.hparams_config(hparams=HPARAMS, metrics=HP_METRICS) # Complete a number of training runs at different hparam values and log the results. repeats_per_combo = cfg['HP_SEARCH'][ 'REPEATS'] # Number of times to train the model per combination of hparams num_combos = cfg['HP_SEARCH'][ 'COMBINATIONS'] # Number of random combinations of hparams to attempt num_sessions = num_combos * repeats_per_combo # Total number of runs in this experiment model_type = 'DCNN_BINARY' if cfg['TRAIN'][ 'CLASS_MODE'] == 'binary' else 'DCNN_MULTICLASS' trial_id = 0 for group_idx in range(num_combos): rand = random.Random() HPARAMS = {h: h.domain.sample_uniform(rand) for h in HPARAMS} hparams = {h.name: HPARAMS[h] for h in HPARAMS} # To pass to model definition for repeat_idx in range(repeats_per_combo): trial_id += 1 print("Running training session %d/%d" % (trial_id, num_sessions)) print("Hparam values: ", {h.name: HPARAMS[h] for h in HPARAMS}) trial_logdir = os.path.join( log_dir, str(trial_id)) # Need specific logdir for each trial callbacks_hp = callbacks + [ TensorBoard( log_dir=trial_logdir, profile_batch=0, write_graph=False) ] # Set values of hyperparameters for this run in config file. for h in hparams: if h in ['LR', 'L2_LAMBDA']: val = 10**hparams[ h] # These hyperparameters are sampled on the log scale. else: val = hparams[h] cfg['NN'][model_type][h] = val # Set some hyperparameters that are not specified in model definition. cfg['TRAIN']['BATCH_SIZE'] = hparams['BATCH_SIZE'] cfg['TRAIN']['IMB_STRATEGY'] = hparams['IMB_STRATEGY'] # Run a training session and log the performance metrics on the test set to HParams dashboard in TensorBoard with tf.summary.create_file_writer(trial_logdir).as_default(): hp.hparams(HPARAMS, trial_id=str(trial_id)) model, test_metrics, test_generator = train_model(cfg, data, callbacks_hp, verbose=0) for metric in HP_METRICS: if metric._tag in test_metrics: tf.summary.scalar(metric._tag, test_metrics[metric._tag], step=1) # Log test metric return
def learn(self, symbol_list): if self.args.image == "generate": for symbol in symbol_list: start_time = time.time() print("Starting to generate images for: ", symbol) GenerateImagesNoHold.generate_buy_sell_images( symbol, str(self.from_date), str(self.to_date)) end_time = time.time() print("Generated images for stocks: ", symbol, " in: ", (end_time - start_time), " seconds") training_data = [] for symbol in symbol_list: for category in self.categories: path = os.path.join(Constants.IMAGE_DIR, category.value + "/" + symbol) class_num = self.categories.index(category) for img in os.listdir(path): try: img_arr = cv2.imread(os.path.join(path, img), cv2.IMREAD_GRAYSCALE) resized_arr = cv2.resize( img_arr, (self.IMG_SIZE, self.IMG_SIZE)) training_data.append([resized_arr, class_num]) except Exception as e: print(e) random.shuffle(training_data) X = [] y = [] for features, labels in training_data: X.append(features) y.append(labels) X = np.array(X).reshape(-1, self.IMG_SIZE, self.IMG_SIZE, 1) y = np.array(y) self.X_train, self.X_test, self.y_train, self.y_test = train_test_split( X, y, test_size=.3) self.X_train = self.X_train / 255.0 model = Sequential() model.add(Conv2D(128, (3, 3), input_shape=X.shape[1:])) model.add(Activation("relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(128, (3, 3))) model.add(Activation("relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(128, (3, 3))) model.add(Activation("relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(32)) model.add(Activation("relu")) model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile( loss="binary_crossentropy", optimizer="rmsprop", metrics=['accuracy'], ) model_save_path = Constants.MODEL_DIR + symbol + Constants.MODEL_EXTENSION model_cp = ModelCheckpoint(model_save_path, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1) lr_callback = LearningRateScheduler(self.lr_schedule) model.fit( self.X_train, self.y_train, batch_size=16, epochs=self.EPOCHS, validation_split=0.2, callbacks=[self.tensorboard, model_cp, self.early_stopping]) val_loss, val_acc = model.evaluate(self.X_test, self.y_test) print('Accuracy for ', symbol, ": ", val_loss, val_acc) model.save(model_save_path) accuracy = False if accuracy: # TENSORBOARD CODE train_dataset = tf.data.Dataset.from_tensor_slices( (self.X_train, self.y_train / 1.0)) test_dataset = tf.data.Dataset.from_tensor_slices( (self.X_test / 255.0, self.y_test / 1.0)) train_dataset = train_dataset.shuffle(60000).batch(64) test_dataset = test_dataset.batch(64) current_time = datetime.datetime.now().strftime( "%Y%m%d-%H%M%S") train_summary_writer = tf.summary.create_file_writer( 'logs/{}'.format(self.NAME) + '/train') test_summary_writer = tf.summary.create_file_writer( 'logs/{}'.format(self.NAME) + '/test') for epoch in range(self.EPOCHS): for (x_train, y_train) in train_dataset: self.train_step(model, self.optimizer, x_train, y_train) with train_summary_writer.as_default(): tf.summary.scalar('loss', self.train_loss.result(), step=epoch) tf.summary.scalar('accuracy', self.train_accuracy.result(), step=epoch) for (x_test, y_test) in test_dataset: self.test_step(model, x_test, y_test) with test_summary_writer.as_default(): tf.summary.scalar('loss', self.test_loss.result(), step=epoch) tf.summary.scalar('accuracy', self.test_accuracy.result(), step=epoch) template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}' print( template.format(epoch + 1, self.train_loss.result(), self.train_accuracy.result() * 100, self.test_loss.result(), self.test_accuracy.result() * 100)) # Reset metrics every epoch self.train_loss.reset_states() self.test_loss.reset_states() self.train_accuracy.reset_states() self.test_accuracy.reset_states() tuning = False if tuning: with tf.summary.create_file_writer( 'logs/hparam_tuning').as_default(): hp.hparams_config( hparams=[ self.HP_NUM_UNITS, self.HP_DROPOUT, self.HP_OPTIMIZER, self.HP_ACTIVATION ], metrics=[ hp.Metric(self.METRIC_ACCURACY, display_name='Accuracy') ], ) session_num = 0 for num_units in self.HP_NUM_UNITS.domain.values: for dropout_rate in (self.HP_DROPOUT.domain.min_value, self.HP_DROPOUT.domain.max_value): for optimizer in self.HP_OPTIMIZER.domain.values: for activation in self.HP_ACTIVATION.domain.values: hparams = { self.HP_NUM_UNITS: num_units, self.HP_DROPOUT: dropout_rate, self.HP_OPTIMIZER: optimizer, self.HP_ACTIVATION: activation } run_name = "run-%d" % session_num print('--- Starting trial: %s' % run_name) print({h.name: hparams[h] for h in hparams}) self.run('logs/hparam_tuning/' + run_name, hparams) session_num += 1
def param_tune(self,): """ Tuning the hyperparameters ****Helper Variables**** self.HP_NUM_UNITS:tensorboard.plugins.hparams.summary_v2.HParam Number of LSTM units self.HP_DROPOUT:tensorboard.plugins.hparams.summary_v2.HParam Dropout rate self.HP_OPTIMIZER:tensorboard.plugins.hparams.summary_v2.HParam Optimizer self.HP_LEARNING_RATE:tensorboard.plugins.hparams.summary_v2.HParam Learning Rate self.name: str name of the data file self.model_choice: str input for chosing model to tune eventlog : str file to be processed F: Datahandler Obj File reading and processing divisor: float average time between current and first events divisor2: float average time between current and first events char_indices : dict ascii coded characters of the unique activities to integer indices indices_char: dict integer indices to ascii coded characters of the unique activities target_char_indices: dict ascii coded characters of the target unique activities to integer indices (target includes one excess activity '!' case end) target_indices_char: dict integer indices to ascii coded characters of the target unique activities maxlen maximum length of cases chars:list ascii coded characters of the activities. target_chars: list ascii coded characters of the target activities. (target includes one excess activity '!' case end) X: ndarray training_set data y_a: list training_set labels for next activity y_t: list training_set list METRIC_LOSS: str Keeping track of the model loss num_units: int LSTM units parameter dropout_rate:float Dropoutrate optimizer:str Optimizer learning_rate:float learning_rate cw: ComputeCW obj class_weights: dict class weights for activities hparams: dict hyperparameter for the run session_num: int session number of the run run_dir: str path to store tensorboard files run_name: str name of the tensorboard file run_stat: str hyperparameters used hist: dict history of the training loss: list val_loss of the last epoch """ eventlog=input("Enter the name of the file to be executed from data folder: ") print(' ') #Reading the Data F = Datahandler() self.name=F.read_data(eventlog) #F.name is the name of the file #F.spamread is the data of the file spamreader,max_task = F.log2np() D=Preprocess() divisor,divisor2,divisor3 = D.divisor_cal(spamreader) maxlen,chars,target_chars,char_indices,indices_char,target_char_indices,target_indices_char= D.dict_cal() num_features = len(chars) + 5 X, y_a, y_t, d_t = D.training_set(num_features) self.model_choice = input( "Please Select model: 1.CS, 2.TLSTM, 3.CS_TLSTM ") self.HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([64,100])) self.HP_DROPOUT = hp.HParam('dropout', hp.Discrete([0.0, 0.2])) self.HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['nadam'])) self.HP_LEARNING_RATE = hp.HParam('learning_rate', hp.Discrete([0.0001,0.0002,0.001,0.002,0.01])) METRIC_LOSS = 'loss' path=os.path.abspath(os.curdir) with tf.summary.create_file_writer(path+'/logs_'+self.name+'/'+self.model_choice+'/').as_default(): hp.hparams_config( hparams=[self.HP_NUM_UNITS,self.HP_DROPOUT, self.HP_OPTIMIZER,self.HP_LEARNING_RATE], metrics=[hp.Metric(METRIC_LOSS, display_name='Loss')],) #Training if self.model_choice == '1': cw=ComputeCW() class_weights=cw.compute_class_weight(F.spamread) print('class_weights are: ', class_weights) def run(run_dir,hparams): with tf.summary.create_file_writer(run_dir).as_default(): M=CSModel(maxlen, max_task, target_chars, self.name, num_features) loss,hist=M.train(X,y_a,y_t, class_weights, hparams,self.HP_NUM_UNITS,self.HP_DROPOUT, self.HP_OPTIMIZER,self.HP_LEARNING_RATE) tf.summary.scalar(METRIC_LOSS, loss, step=1) return loss,hist elif self.model_choice == '2': def run(run_dir,hparams): with tf.summary.create_file_writer(run_dir).as_default(): M=ALL_TLSTM_Model(maxlen, max_task, target_chars, self.name, num_features) loss,hist=M.train(X,d_t,y_a,y_t,hparams,self.HP_NUM_UNITS,self.HP_DROPOUT, self.HP_OPTIMIZER,self.HP_LEARNING_RATE) tf.summary.scalar(METRIC_LOSS, loss, step=1) return loss,hist elif self.model_choice == '3': cw=ComputeCW() class_weights=cw.compute_class_weight(F.spamread) print('class_weights are: ', class_weights) def run(run_dir,hparams): with tf.summary.create_file_writer(run_dir).as_default(): M=CS_TLSTM_Model(maxlen, max_task, target_chars, self.name, num_features) loss,hist=M.train(X,d_t,y_a,y_t,class_weights,hparams,self.HP_NUM_UNITS,self.HP_DROPOUT, self.HP_OPTIMIZER,self.HP_LEARNING_RATE) tf.summary.scalar(METRIC_LOSS, loss, step=1) return loss,hist session_num = 0 for num_units in (self.HP_NUM_UNITS.domain.values): for dropout_rate in (self.HP_DROPOUT.domain.values): for optimizer in self.HP_OPTIMIZER.domain.values: for learning_rate in self.HP_LEARNING_RATE.domain.values: hparams = { self.HP_NUM_UNITS:num_units, self.HP_DROPOUT: dropout_rate, self.HP_OPTIMIZER: optimizer, self.HP_LEARNING_RATE:learning_rate, } #run_name = "run-%d" % session_num run_stat={h.name: hparams[h] for h in hparams} print('Session',session_num) run_name = str(run_stat.values()) #print('--- Starting trial: %s' % run_name) run_stat={h.name: hparams[h] for h in hparams} print(run_stat) loss,hist = run(path+'/logs_'+self.name+'/'+ self.model_choice+'/', hparams) with open(path+'/history/'+self.model_choice+'/'+self.name+run_name, "wb") as fp: pickle.dump(hist, fp) session_num += 1
print(X_train.shape, y_train.shape) return X_train, y_train # Part 2 - Building the RNN HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([100])) HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.11)) HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['rmsprop'])) HP_OUTPUT = hp.HParam('output_number', hp.Discrete(list(range(1)))) HP_WINDOW = hp.HParam('window_size', hp.Discrete([1])) #HP_OUTPUT = hp.HParam('output_number',hp.Discrete(list(range(n_outputs-1)))) METRIC_ACCURACY = 'accuracy' hp.hparams_config( hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER, HP_OUTPUT, HP_WINDOW], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')], ) def run(run_dir, hparams): hp.hparams(hparams) # record the values used in this trial predicted = train_test_model(run_dir, hparams) return predicted optimizers.RMSprop(lr=0.001, rho=0.9, decay=0.9) optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True) def get_weighted_loss(weights): def weighted_loss(y_true, y_pred):
print(X_scaled.shape,y_scaled.shape) return X_scaled, y_scaled # Part 2 - Building the RNN HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([100])) HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1,0.11)) HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['rmsprop'])) HP_OUTPUT = hp.HParam('output_number',hp.Discrete(list(range(0,12)))) HP_WINDOW = hp.HParam('window_size',hp.Discrete([4])) HP_HORIZON = hp.HParam('horizon_size',hp.Discrete([4])) #HP_OUTPUT = hp.HParam('output_number',hp.Discrete(list(range(n_outputs-1)))) METRIC_ACCURACY = 'loss' hp.hparams_config(hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER, HP_OUTPUT, HP_WINDOW, HP_HORIZON], metrics=[hp.Metric(METRIC_ACCURACY, display_name='loss')],) optimizers.RMSprop(lr=0.001,rho=0.9,decay=0.9) optimizers.SGD(lr=0.001,decay=1e-6,momentum=0.9,nesterov=True) n_inputs = max(HP_OUTPUT.domain.values)# total number of inputs minus 1 as it starts from 0 def train_model(run_dir,hparams): hp.hparams(hparams) [X_train, y_train] = create_data(data_unscaled=data, start_train=start_train, end_train=end_train, n_windows=hparams[HP_WINDOW], n_outputs=hparams[HP_OUTPUT]) [X_test, y_test] = create_data(data_unscaled=data, start_train=end_train, end_train=end_test, n_windows=hparams[HP_WINDOW], n_outputs=hparams[HP_OUTPUT]) tf.compat.v1.keras.backend.clear_session() model = Sequential() model.add(TimeDistributed(Masking(mask_value=0., input_shape=(hparams[HP_WINDOW], n_inputs+1)), input_shape=(n_company, hparams[HP_WINDOW], n_inputs+1))) model.add(TimeDistributed(LSTM(hparams[HP_NUM_UNITS], stateful=False, activation='tanh', return_sequences=True, input_shape=(hparams[HP_WINDOW], n_inputs+1), kernel_initializer='TruncatedNormal' ,bias_initializer=initializers.Constant(value=0.1), dropout=hparams[HP_DROPOUT] ,recurrent_dropout=hparams[HP_DROPOUT])))
from input_pipeline.data_preprocessing_visualization import r_train_ds, r_val_s, r_test_ds #Define the hyperparameters HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'RMSProp'])) HP_LSTM_NEURONS = hp.HParam('LSTMNeurons', hp.Discrete([128,256])) HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.2)) HP_EPOCHS = hp.HParam('epochs',hp.Discrete([5, 10])) METRIC_ACCURACY = 'accuracy' path_hparams = input('Enter the path to save the tuning logs: ') #The logs will be created and stored in the following path with tf.summary.create_file_writer(path_hparams + 'logs/hparam_tuning').as_default(): hp.hparams_config( hparams = [HP_LSTM_NEURONS, HP_DROPOUT, HP_OPTIMIZER, HP_EPOCHS], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')], ) #Hyperparamter model definition def HParams_mdl(): inputs = keras.Input(shape=(N_WindowSize,N_Features)) x = layers.LSTM(hparams[HP_LSTM_NEURONS],return_sequences=True)(inputs) x = layers.Dropout(hparams[HP_DROPOUT])(x) x = layers.LSTM(128,return_sequences=True)(x) outputs = layers.Dense(N_CLASSES, activation='softmax')(x) mdl = keras.Model(inputs=inputs, outputs=outputs, name='HAR_model') return mdl def run(run_dir, hparams): with tf.summary.create_file_writer(run_dir).as_default(): hp.hparams(hparams)
subset='training') val_generator = datagen.flow_from_directory(base_dir, target_size=(IMAGE_SIZE, IMAGE_SIZE), batch_size=BATCH_SIZE, subset='validation') HP_CONVO_FILTER = hp.HParam('filter', hp.Discrete([64, 128])) HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.2)) METRIC_ACCURACY = 'accuracy' with tf.summary.create_file_writer('logs/hparam_tuning').as_default(): hp.hparams_config( hparams=[HP_CONVO_FILTER, HP_DROPOUT], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')], ) def train_test_model(hparams): IMG_SHAPE = (IMAGE_SIZE, IMAGE_SIZE, 3) # Create the base model from the pre-trained model MobileNet V2 base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE, alpha=1.0, include_top=False, weights='imagenet') base_model.trainable = False model = tf.keras.Sequential([
gpu = args.gpu os.environ["CUDA_VISIBLE_DEVICES"] = gpu config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config = config) K.set_session(sess) HP_STRIDES = hp.HParam("strides", hp.Discrete([1,2,3])) HP_DROPOUT = hp.HParam("cnn_dropout", hp.Discrete([0.3, 0.4, 0.5])) # HP_FILTER = hp.HParam("filter_size", hp.Discrete([3,3],[2,2])) METRIC_ACCURACY = "accuracy" hp.hparams_config( hparams = [HP_STRIDES, HP_DROPOUT], metrics = [hp.Metric(METRIC_ACCURACY, display_name = "Accuarcy")] ) class LearningRateSchedule(tf.keras.callbacks.Callback): def __init__(self, schedule): super(LearningRateSchedule, self).__init__() self.schedule = schedule def on_epoch_begin(self, epoch, logs=None): if not hasattr(self.model.optimizer, 'lr'): raise ValueError("Optimizer must have a lr attribute") lr = float(tf.keras.backend.get_value(self.model.optimizer.lr)) scheduled_lr = self.schedule(epoch, lr) tf.keras.backend.set_value(self.model.optimizer.lr, scheduled_lr) print("\nEpoch {}: Learning Rate is {}.".format(epoch, scheduled_lr))
def objective(trial): standard_object = PrunableEvaluateMNIST( train_images=train_images, test_images=test_images, train_labels=train_labels, test_labels=test_labels, validation_data_proportion=(1-JUN_SHAO_TRAINING_PROPORTION), ) session_hparams = {} hparams_list = [] # Generate hyper-parameters standard_object.number_of_conv_2d_filters = trial.suggest_int( 'number_of_conv_2d_filters', 4, 8, ) session_hparams['number_of_conv_2d_filters'] = standard_object.number_of_conv_2d_filters hparams_list.append(hp.HParam('number_of_conv_2d_filters', hp.RealInterval(float(4), float(8)))) standard_object.first_conv2d_kernel_dim = trial.suggest_int( 'first_conv2d_kernel_dim', 1, 2, ) session_hparams['first_conv2d_kernel_dim'] = standard_object.first_conv2d_kernel_dim hparams_list.append(hp.HParam('first_conv2d_kernel_dim', hp.RealInterval(float(1), float(2)))) standard_object.second_conv2d_kernel_dim = trial.suggest_int( 'second_conv2d_kernel_dim', 1, 2, ) session_hparams['second_conv2d_kernel_dim'] = standard_object.second_conv2d_kernel_dim hparams_list.append(hp.HParam('second_conv2d_kernel_dim', hp.RealInterval(float(1), float(2)))) standard_object.conv2d_layers_activation_func = trial.suggest_categorical( 'conv2d_layers_activation_func', [ 'relu', 'sigmoid', 'softplus', ] ) session_hparams['conv2d_layers_activation_func'] = standard_object.conv2d_layers_activation_func hparams_list.append(hp.HParam('second_conv2d_kernel_dim', hp.Discrete([ 'relu', 'sigmoid', 'softplus', ]))) standard_object.number_hidden_conv_layers = trial.suggest_int( 'number_hidden_conv_layers', 0, 2, ) session_hparams['number_hidden_conv_layers'] = standard_object.number_hidden_conv_layers hparams_list.append(hp.HParam('number_hidden_conv_layers', hp.RealInterval(float(0), float(2)))) standard_object.batch_size_base_two_logarithm = trial.suggest_int( 'batch_size_base_two_logarithm', 0, MAXIMUM_BATCH_SIZE_POWER_OF_TWO, ) session_hparams['batch_size_base_two_logarithm'] = standard_object.batch_size_base_two_logarithm hparams_list.append(hp.HParam('batch_size_base_two_logarithm', hp.RealInterval(float(0), float(MAXIMUM_BATCH_SIZE_POWER_OF_TWO)))) standard_object.adam_learning_rate = trial.suggest_uniform( 'adam_learning_rate', 0, 2, ) session_hparams['adam_learning_rate'] = standard_object.adam_learning_rate hparams_list.append(hp.HParam('adam_learning_rate', hp.RealInterval(float(0), float(2)))) standard_object.adam_beta_1 = trial.suggest_uniform( 'adam_beta_1', 0, 1, ) session_hparams['adam_beta_1'] = standard_object.adam_beta_1 hparams_list.append(hp.HParam('adam_beta_1', hp.RealInterval(float(0), float(1)))) standard_object.adam_beta_2 = trial.suggest_uniform( 'adam_beta_2', 0, 1, ) session_hparams['adam_beta_2'] = standard_object.adam_beta_2 hparams_list.append(hp.HParam('adam_beta_2', hp.RealInterval(float(0), float(1)))) standard_object.adam_epsilon = trial.suggest_uniform( 'adam_epsilon', 0, 1 ) session_hparams['adam_epsilon'] = standard_object.adam_epsilon hparams_list.append(hp.HParam('adam_epsilon', hp.RealInterval(float(0), float(1)))) # Train and validate using hyper-parameters generated above clear_session() classifier_model = models.Sequential() classifier_model.add(layers.Conv2D( standard_object.number_of_conv_2d_filters, (standard_object.first_conv2d_kernel_dim, standard_object.second_conv2d_kernel_dim), activation=standard_object.conv2d_layers_activation_func, input_shape=(28, 28, 1), )) classifier_model.add(layers.MaxPooling2D((2, 2), strides=2)) for level in range(standard_object.number_hidden_conv_layers): classifier_model.add(layers.Conv2D( standard_object.number_of_conv_2d_filters, (standard_object.first_conv2d_kernel_dim, standard_object.second_conv2d_kernel_dim), activation=standard_object.conv2d_layers_activation_func, )) classifier_model.add(layers.MaxPooling2D((2, 2), strides=2)) classifier_model.add(layers.Flatten()) classifier_model.add(layers.Dense(10, activation='softmax')) standard_object.optimizer = Adam( learning_rate=standard_object.adam_learning_rate, beta_1=standard_object.adam_beta_1, beta_2=standard_object.adam_beta_2, epsilon=standard_object.adam_epsilon, amsgrad=0, # False ) classifier_model.compile( optimizer=standard_object.optimizer, loss=CategoricalCrossentropy(), metrics=[CategoricalAccuracy()], ) output_dir = os.path.join(stem_output_dir, ('trial_' + datetime.now().strftime('%Y.%m.%d_%H.%M.%S'))) if not(os.path.exists(output_dir)): os.mkdir(output_dir) tb_callback = TbCallback( log_dir=output_dir, histogram_freq=1, write_graph=1, # True ) standard_object.callbacks.append(tb_callback) # Append to callbacks list hp.hparams_config( hparams=hparams_list, metrics=[hp.Metric(CategoricalAccuracy().name, display_name=CategoricalAccuracy().name)], ) hp_callback = hp.KerasCallback(writer=output_dir, hparams=session_hparams) standard_object.callbacks.append(hp_callback) # Append to callbacks list es_callback = EarlyStopping( monitor='val_loss', min_delta=EARLY_STOPPING_SIGNIFICANT_DELTA, patience=EARLY_STOPPING_PATIENCE_PARAMETER, verbose=VERBOSITY_LEVEL_FOR_TENSORFLOW, mode='auto', baseline=None, restore_best_weights=1, # True ) standard_object.callbacks.append(es_callback) # Append to callbacks list # Append tf.keras pruner for later use during study keras_pruner = optuna.integration.TFKerasPruningCallback( trial, 'val_categorical_accuracy', ) standard_object.callbacks.append(keras_pruner) # Append to callbacks list standard_object.set_batch_size(standard_object.batch_size_base_two_logarithm) standard_object.stratified_split_for_training_and_validation() classifier_model.fit( standard_object.train_split_images, standard_object.train_split_labels, epochs=MAXIMUM_EPOCHS_TO_TRAIN, validation_data=standard_object.validate_split_data, verbose=VERBOSITY_LEVEL_FOR_TENSORFLOW, callbacks=standard_object.callbacks, batch_size=standard_object.batch_size, use_multiprocessing=1, # True workers=4, ) # Evaluate performance on test data and report score test_metrics = classifier_model.evaluate( standard_object.test_images, standard_object.test_labels, batch_size=standard_object.batch_size, ) test_results_dict = {output: test_metrics[i] for i, output in enumerate(classifier_model.metrics_names)} return(test_results_dict['categorical_accuracy'])
tf.summary.scalar('critic loss', self.critic_loss, step=self.n_train) #tf.summary.histogram('critic weights',self.critic.trainable_weights[0],step=self.n_train) #Start script here mv_envs = [] n = 0 trials = 1 if linux: with tf.summary.create_file_writer(os.getcwd() + '/logs').as_default(): hp.hparams_config( hparams=[ FORCE_SCALE, UPDATE_FREQ, TOLERANCE, MAX_STEPS, MAX_SESSIONS, CDIV, VAL_SCALE, LR_CTBG, LR_CRITIC, UNIT_1, UNIT_2, UNIT_3, TAU ], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Reward')], ) else: with tf.summary.create_file_writer(os.getcwd() + '\\logs').as_default(): hp.hparams_config( hparams=[ FORCE_SCALE, UPDATE_FREQ, TOLERANCE, MAX_STEPS, MAX_SESSIONS, CDIV, VAL_SCALE, LR_CTBG, LR_CRITIC, UNIT_1, UNIT_2, UNIT_3, TAU ], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Reward')], ) for a in FORCE_SCALE.domain.values:
def train_models( input_dir, basenames, clades, merge_behaviour='evenly', split_specifications={ 'train': { 'name': 'train', 'wanted_models': [0, 1], 'interweave_models': [.67, .33], 'repeat_models': [True, True] }, 'val': { 'name': 'val', 'wanted_models': [0, 1], 'interweave_models': True, 'repeat_models': [False, False] }, 'test': { 'name': 'test', 'wanted_models': [0, 1], 'interweave_models': True, 'repeat_models': [False, False] }, }, tuple_length=1, use_amino_acids=False, used_codons=False, model_hyperparameters={ "tcmc_rnn": { "tcmc_models": [ 8, ], "rnn_type": [ "gru", ], "rnn_units": [ 32, ], "dense_dimension": [ 16, ], }, "tcmc_mean_log": { "tcmc_models": [ 8, ], "sequence_length_as_feature": [ False, ], "dense1_dimension": [ 16, ], "dense2_dimension": [ 16, ], }, }, model_training_callbacks={ "tcmc_rnn": {}, "tcmc_mean_log": {}, }, batch_size=30, batches_per_epoch=100, epochs=40, save_model_weights=True, log_basedir='logs', saved_weights_basedir='saved_weights', verbose=True, ): """ TODO: Write Docstring """ print("type of clades:", type(clades)) # calculate some features from the input num_leaves = database_reader.num_leaves(clades) tuple_length = 3 if used_codons else tuple_length alphabet_size = 4**tuple_length if not use_amino_acids else 20**tuple_length # evaluate the split specifications splits = {'train': None, 'val': None, 'test': None} num_classes = 0 for k in split_specifications: if k in splits.keys(): try: splits[k] = database_reader.DatasetSplitSpecification( **split_specifications[k]) num_classes = len( splits[k].wanted_models) if num_classes < len( splits[k].wanted_models) else num_classes except TypeError as te: raise Exception( f"Invalid split specification for '{k}': {split_specifications[k]}" ) from te # read the datasets for each wanted basename wanted_splits = [split for split in splits.values() if split != None] unmerged_datasets = { b: database_reader.get_datasets(input_dir, b, wanted_splits, num_leaves=num_leaves, alphabet_size=alphabet_size, seed=None, buffer_size=1000, should_shuffle=True) for b in basenames } if any(['train' not in unmerged_datasets[b] for b in basenames]): raise Exception("A 'train' split must be specified!") # merge the respective splits of each basename datasets = {} merge_behaviour = merge_behaviour if len( merge_behaviour) > 1 else merge_behaviour[0] weights = len(basenames) * [1 / len(basenames)] # evenly is default if isinstance(merge_behaviour, str): if merge_behaviour != "evenly": print(f'Unknown merge beheaviour, merging evenly.') else: if len(merge_behaviour) > 1: # check whether the custom weights are correct # expecting a list of weights try: merge_behaviour = [float(w) for w in merge_behaviour] except ValueError: print( f'Expected a list of floats in merge_behaviour. However merge_behaviour = {merge_behaviour}' ) print(f'Will use even merge_behaviour = {weights} instead.') if len(merge_behaviour) == len(basenames) \ and all([isinstance(x, float) for x in merge_behaviour]) \ and all([x >= 0 for x in merge_behaviour]) \ and sum(merge_behaviour) == 1: weights = merge_behaviour merge_ds = tf.data.experimental.sample_from_datasets datasets = { s: merge_ds([unmerged_datasets[b][s] for b in basenames], weights) for s in splits.keys() } # prepare datasets for batching for split in datasets: ds = datasets[split] # batch and reshape sequences to match the input specification of tcmc ds = database_reader.padded_batch(ds, batch_size, num_leaves, alphabet_size) ds = ds.prefetch(tf.data.experimental.AUTOTUNE) #ds = ds.map(database_reader.concatenate_dataset_entries, num_parallel_calls = 4) # TODO: Pass the variable "num_classes" to database_reader.concatenate_dataset_entries(). if num_classes == 2: ds = ds.map(database_reader.concatenate_dataset_entries, num_parallel_calls=4) elif num_classes == 3: ds = ds.map(database_reader.concatenate_dataset_entries2, num_parallel_calls=4) else: raise Exception( f'Currently we only support two and three output classes. Your number of classes:{num_classes}' ) datasets[split] = ds if verbose: print(f'Example batch of the "train" dataset:\n') for t in datasets['train'].take(1): (sequences, clade_ids, sequence_lengths), models = t # extract the clade ids per batch padding = [[1, 0]] ind = tf.pad(tf.cumsum(sequence_lengths), padding)[:-1] clade_ids = tf.gather(clade_ids, ind) # extract the model ids model_ids = tf.argmax(models, axis=1) print(f'model_ids: {model_ids}') print(f'clade_ids: {clade_ids}') print(f'sequence_length: {sequence_lengths}') print(f'sequence_onehot.shape: {sequences.shape}') # to debug the sequence first transform it to its # original shape S = tf.transpose(sequences, perm=[1, 0, 2]) # decode the sequence and print some columns if use_amino_acids: alphabet = "ARNDCEQGHILKMFPSTWYV" dec = ote.OnehotTupleEncoder.decode_tfrecord_entry( S.numpy(), alphabet=alphabet, tuple_length=tuple_length, use_bucket_alphabet=False) else: dec = ote.OnehotTupleEncoder.decode_tfrecord_entry( S.numpy(), tuple_length=tuple_length) print( f'first (up to) 8 alignment columns of decoded reshaped sequence: \n{dec[:,:8]}' ) # obtain the model creation functions for the wanted models model_creaters = {} model_training_callbacks = {} for model_name in model_hyperparameters.keys(): try: model_module = import_module(f"models.{model_name}", package=__name__) except ModuleNotFoundError as err: raise Exception( f'The module "models/{model_name}.py" for the model "{model_name}" does not exist.' ) from err try: model_creaters[model_name] = getattr(model_module, "create_model") except AttributeError as err: raise Exception( f'The model "{model_name}" has no creation function "create_model" in "models/{model_name}.py".' ) try: model_training_callbacks[model_name] = getattr( model_module, "training_callbacks") except AttributeError as err: raise Exception( f'The model "{model_name}" has no training callbacks function "training_callbacks" in "models/{model_name}.py".' ) #prepare the hyperparams for training for model_name in model_hyperparameters: model_hps = model_hyperparameters[model_name] model_hps = { h: hp.HParam(h, hp.Discrete(model_hps[h])) for h in model_hps } model_hyperparameters[model_name] = model_hps accuracy_metric = 'accuracy' auroc_metric = tf.keras.metrics.AUC(num_thresholds=500, dtype=tf.float32, name='auroc') for model_name in model_hyperparameters: if verbose: print( '==================================================================================================' ) print(f'Current model name: "{model_name}"\n') # prepare model hyperparameters for iteration hps = model_hyperparameters[model_name] hp_names = list(hps.keys()) hp_values = [hps[k].domain.values for k in hp_names] # log the wanted hyperparams and metrics str_basenames = '_'.join(basenames) logdir = f'{log_basedir}/{str_basenames}/{model_name}' with tf.summary.create_file_writer(logdir).as_default(): hp.hparams_config( hparams=list(hps.values()), metrics=[ hp.Metric('accuracy', group='test', display_name='Accuracy'), hp.Metric('auroc', group='test', display_name="AUROC"), ], ) # iterate over all hyperparameter combinations for the current model for hp_config in itertools.product(*hp_values): # hp for tensorboard callback hp_current = {hps[k]: hp_config[i] for i, k in enumerate(hp_names)} # hp for model creation creation_params = {k: hp_config[i] for i, k in enumerate(hp_names)} # determine logging and saving paths now_str = datetime.datetime.now().strftime('%Y.%m.%d--%H.%M.%S') rundir = f'{logdir}/{now_str}' save_weights_dir = f'{saved_weights_basedir}/{str_basenames}/{model_name}' Path(save_weights_dir).mkdir(parents=True, exist_ok=True) save_weights_path = f'{save_weights_dir}/{now_str}.h5' if verbose: print(f"\n\n") print(f"Current set of hyperparameters: {creation_params}") print(f"Training information will be stored in: {rundir}") print( f"Weights for the best model will be stored in: {save_weights_path}" ) with tf.summary.create_file_writer(rundir).as_default(): hp.hparams(hp_current, trial_id=now_str) create_model = model_creaters[model_name] model = create_model(clades, alphabet_size, **creation_params) if verbose: print( f'Architecture of the model "{model_name}" with the current hyperparameters:' ) model.summary() # compile the model for training loss = tf.keras.losses.CategoricalCrossentropy() optimizer = tf.keras.optimizers.Adam(0.0005) model.compile( optimizer=optimizer, loss=loss, metrics=[accuracy_metric, auroc_metric], ) # define callbacks during training checkpoint_cb = tf.keras.callbacks.ModelCheckpoint( filepath=save_weights_path, monitor='val_loss', mode='min', save_best_only=True, verbose=1, ) # Function to decrease learning rate by 'factor' learnrate_cb = tf.keras.callbacks.ReduceLROnPlateau( monitor='val_loss', mode='min', factor=0.75, patience=4, verbose=1, ) tensorboard_cb = tf.keras.callbacks.TensorBoard(rundir) callbacks = [ tensorboard_cb, ] if datasets['val'] != None: callbacks = callbacks + [checkpoint_cb, learnrate_cb] training_callbacks = model_training_callbacks[model_name] callbacks = callbacks + training_callbacks( model, rundir, wanted_callbacks=None) model.fit( datasets['train'], validation_data=datasets['val'], callbacks=callbacks, epochs=epochs, steps_per_epoch=batches_per_epoch, verbose=verbose, ) # load 'best' model weights and eval the test dataset if datasets['test'] != None: if verbose: print("Evaluating the 'test' dataset:") model.load_weights(save_weights_path) test_loss, test_acc, test_auroc = model.evaluate( datasets['test']) with tf.summary.create_file_writer( f'{rundir}/test').as_default(): tf.summary.scalar('accuracy', test_acc, step=1) tf.summary.scalar('auroc', test_auroc, step=1) tf.summary.scalar('loss', test_auroc, step=1) return 0
def train(self): """Main training function.""" self.timing.start() dimensions = self._create_dimensions() hyperparameters = self._create_hyprparameters_domain() with tf.summary.create_file_writer( str(Path.cwd().joinpath("output", "logs", "hparam_tuning"))).as_default(): hp.hparams_config( hparams=hyperparameters, metrics=[hp.Metric("accuracy", display_name="Accuracy")], ) ( network_settings, train_settings, preprocess_settings, ) = parseConfigsFile(["network", "train", "preprocess"]) BATCH_SIZE = train_settings[ "batch_size"] * self.strategy.num_replicas_in_sync ( synthetic_train, synthetic_dataset_len, synthetic_num_classes, ) = self._get_datasets(BATCH_SIZE) validation_dataset = self._get_validation_dataset(BATCH_SIZE) srfr_model = self._instantiate_models(synthetic_num_classes, network_settings, preprocess_settings, train_settings) train_model_sr_only_use_case = TrainModelFrOnlyUseCase( self.strategy, TimingLogger(), self.logger, BATCH_SIZE, synthetic_dataset_len, ) _training = partial( self._fitness_function, train_model_use_case=train_model_sr_only_use_case, srfr_model=srfr_model, batch_size=BATCH_SIZE, synthetic_train=synthetic_train, validation_dataset=validation_dataset, num_classes=synthetic_num_classes, train_settings=train_settings, hparams=hyperparameters, ) _train = use_named_args(dimensions=dimensions)(_training) initial_parameters = [0.0002, 0.9, 10_000, 0.0005] search_result = gp_minimize( func=_train, dimensions=dimensions, acq_func="EI", n_calls=20, x0=initial_parameters, ) self.logger.info(f"Best hyperparameters: {search_result.x}")
def hp_search(working_dir, dataset='56_sunspots', epochs=100, metric='eval_mae_result', stopping_patience=5, stopping_delta=1): working_dir = os.path.join("./checkpoints", working_dir) # define domains for HP search HP_EMB_DIM = hp.HParam('emb_dim', hp.Discrete([32, 64, 128])) HP_LSTM_DIM = hp.HParam('lstm_dim', hp.Discrete([32, 64, 128])) HP_DROPOUT = hp.HParam('lstm_dropout', hp.Discrete([0.1, 0.2, 0.3])) HP_LR = hp.HParam('learning_rate', hp.Discrete([.0001, .001, .01])) HP_BS = hp.HParam('batch_size', hp.Discrete([32, 64, 128])) HP_WINDOW = hp.HParam('window_size', hp.Discrete([20, 40, 60])) # set up config with tf.summary.create_file_writer(working_dir).as_default(): hp.hparams_config(hparams=[ HP_EMB_DIM, HP_LSTM_DIM, HP_DROPOUT, HP_LR, HP_BS, HP_WINDOW ], metrics=[hp.Metric(metric, display_name=metric)]) # read in training df df = pd.read_csv( f'../../datasets/seed_datasets_current/{dataset}/TRAIN/dataset_TRAIN/tables/learningData.csv' ) df = _multi_index_prep(df, dataset) df = _time_col_to_seconds(df, dataset) # create TimeSeries and Learner objects ds = _create_ts_object(df, dataset) # grid search over parameters run_num = 0 total_run_count = len(HP_EMB_DIM.domain.values) * \ len(HP_LSTM_DIM.domain.values) * \ len(HP_DROPOUT.domain.values) * \ len(HP_LR.domain.values) * \ len(HP_BS.domain.values) * \ len(HP_WINDOW.domain.values) # outfile for saving hp config and runtimes outfile = open(os.path.join(working_dir, "metrics.txt"), "w+", buffering=1) for emb_dim in HP_EMB_DIM.domain.values: for lstm_dim in HP_LSTM_DIM.domain.values: for dropout in HP_DROPOUT.domain.values: for lr in HP_LR.domain.values: for bs in HP_BS.domain.values: for window_size in HP_WINDOW.domain.values: # create dict of parameters hp_dict = { 'emb_dim': emb_dim, 'lstm_dim': lstm_dim, 'lstm_dropout': dropout, 'learning_rate': lr, 'batch_size': bs, 'window_size': window_size, } run_name = f'run-{run_num}' logger.info( f'--- Starting Run: {run_name} of {total_run_count} ---' ) # print_dict = { # h.name: hp_dict[h] for h in hp_dict # } logger.info(f'HP Dict: {hp_dict}') hp.hparams(hp_dict) # create learner and fit with these HPs start_time = time.time() learner = DeepARLearner(ds, verbose=1, hparams=hp_dict) final_metric = learner.fit( epochs=epochs, stopping_patience=stopping_patience, stopping_delta=stopping_delta, checkpoint_dir=os.path.join( working_dir, run_name)) outfile.write( f'HPs: {hp_dict} ---- Metric: {final_metric} ---- Time: {round(time.time() - start_time,2)}\n' ) tf.summary.scalar(metric, final_metric, step=1) run_num += 1 outfile.close()
mse = run_training() tf.summary.scalar('mse', mse, step=1) if __name__ == '__main__': HP_NUM_FC_BLOCKS = hp.HParam('num_fc_blocks', hp.Discrete([2, 5])) HP_NUM_RES_BLOCKS = hp.HParam('num_res_blocks', hp.Discrete([2, 4])) HP_ACTIVATION = hp.HParam('activation_type', hp.Discrete(['leaky', 'mish'])) HP_INIT_NUM_UNITS = hp.HParam('init_num_units', hp.Discrete([8, 16, 32])) HP_FINAL_NUM_UNITS = hp.HParam('final_num_units', hp.Discrete([8, 16, 32])) HP_USE_DOWNSAMPLE = hp.HParam('use_downsample', hp.Discrete([True, False])) with tf.summary.create_file_writer('logs/hparam_tuning').as_default(): hp.hparams_config( hparams=[HP_NUM_FC_BLOCKS, HP_NUM_RES_BLOCKS, HP_ACTIVATION, HP_INIT_NUM_UNITS, HP_FINAL_NUM_UNITS, HP_USE_DOWNSAMPLE], metrics=[hp.Metric(tf.keras.metrics.MeanSquaredError(), display_name='MSE_LOSS')], )
#%% HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([128, 256, 512])) HP_DROPOUT = hp.HParam('dropout', hp.Discrete([0.5])) HP_LEARNING_RATE = hp.HParam('learning_rate', hp.Discrete([0.0001, 0.00001])) HP_CNN_FILTER_1 = hp.HParam('filter_1', hp.Discrete([16, 64, 256])) HP_CNN_FILTER_2 = hp.HParam('filter_2', hp.Discrete([16, 64, 256])) HP_BATCH_NORM = hp.HParam('batch_norm', hp.Discrete([False, True])) HP_CNN_KERNEL_X_1 = hp.HParam('kernel_1_x', hp.Discrete([80, 30, 5])) HP_CNN_KERNEL_Y_1 = hp.HParam('kernel_1_y', hp.Discrete([80, 30, 5])) HP_CNN_KERNEL_X_2 = hp.HParam('kernel_2_x', hp.Discrete([80, 30, 5])) HP_CNN_KERNEL_Y_2 = hp.HParam('kernel_2_y', hp.Discrete([80, 30, 5])) with tf.summary.create_file_writer('logs/78-hparam-tuning-v3').as_default(): hp.hparams_config( hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_LEARNING_RATE, HP_CNN_KERNEL_X_1, HP_CNN_KERNEL_Y_1, HP_CNN_KERNEL_X_2, HP_CNN_KERNEL_Y_2, HP_CNN_FILTER_1, HP_CNN_FILTER_2, HP_BATCH_NORM], metrics=[hp.Metric('accuracy', display_name='Accuracy')], ) #%% def train_test_model(logdir, hparams): classifier = tf.keras.Sequential() classifier.add(tf.keras.layers.Conv2D(filters=hparams[HP_CNN_FILTER_1], kernel_size=(hparams[HP_CNN_KERNEL_X_1], hparams[HP_CNN_KERNEL_Y_1]), padding='same', activation='relu', input_shape=(x_train[0].shape[0], x_train[0].shape[1],1))) if hparams[HP_BATCH_NORM]: classifier.add(tf.keras.layers.BatchNormalization()) classifier.add(tf.keras.layers.MaxPooling2D(pool_size=2)) classifier.add(tf.keras.layers.Dropout(hparams[HP_DROPOUT])) classifier.add(tf.keras.layers.Conv2D(filters=hparams[HP_CNN_FILTER_2], kernel_size=(hparams[HP_CNN_KERNEL_X_2], hparams[HP_CNN_KERNEL_Y_2]), padding='same', activation='relu')) classifier.add(tf.keras.layers.MaxPooling2D(pool_size=2)) classifier.add(tf.keras.layers.Dropout(hparams[HP_DROPOUT])) classifier.add(tf.keras.layers.Flatten())
my_inp=np.concatenate((inp_reno[:,None],inp_aoa[:,None],inp_para[:,:]),axis=1) my_out=np.concatenate((out_cd[:,None],out_cl[:,None]),axis=1) ## Training sets xtr0 = my_inp[I][:n] ttr1 = my_out[I][:n] #Hyperparameters HP_NEURONS = hp.HParam('neurons', hp.Discrete(range(10,101,10))) HP_LAYERS = hp.HParam('layer', hp.Discrete([1,2,3,4,5,6,7,8,9,10])) HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam'])) HP_L_RATE= hp.HParam('learning_rate', hp.Discrete([2.5e-4])) METRIC_MSE = 'Mean Squared Error' with tf.summary.create_file_writer('logs/1 hidden layer').as_default(): hp.hparams_config( hparams=[HP_LAYERS, HP_NEURONS, HP_OPTIMIZER], metrics=[hp.Metric(METRIC_MSE, display_name='val_loss')], ) epochs = list(range(0,gg)) def train_test_model(hparams, layer,rundir): model = tf.keras.models.Sequential() model.add(tf.keras.layers.InputLayer(input_shape=(5,))) model.add(tf.keras.layers.Dense(hparams[HP_NEURONS], kernel_initializer='random_normal', activation=tf.keras.activations.relu)) if layer > 1: for i in range(layer-1): model.add(tf.keras.layers.Dense(hparams[HP_NEURONS], activation=tf.keras.activations.relu)) model.add(tf.keras.layers.Dense(2, activation=None)) model.summary() optimizer_name = hparams[HP_OPTIMIZER] learning_rate = hparams[HP_L_RATE]
import datetime import tensorflow as tf from tensorboard.plugins.hparams import api as hp HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([8, 64])) HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.3)) HP_LEARNING_RATE= hp.HParam('learning_rate', hp.Discrete([0.001, 0.0005, 0.0001])) HP_OPTIMIZER=hp.HParam('optimizer', hp.Discrete(['adam', 'sgd', 'rmsprop'])) METRIC_ACCURACY = 'RootMeanSquaredError' log_dir ='\\logs\\fit\\' + datetime.datetime.now().strftime('%Y%m%d-%H%M%S') with tf.summary.create_file_writer(log_dir).as_default(): hp.hparams_config( hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER, HP_LEARNING_RATE], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Rmse')], ) def train_and_evaluate(model, hparams): tensorboard = TensorBoard(log_dir = log_dir, histogram_freq=10, write_graph=True, write_images=False) hp_tuning = hp.KerasCallback(log_dir,hparams) cbs = [tensorboard,hp_tuning] history = model.fit(gen, validation_split=0.15, epochs=p.epochs, batch_size=p.batch_size, callbacks=cbs, verbose=1, shuffle = True) mse, rmse = model.evaluate(test_gen) return rmse, history def run(run_dir, hparams): with tf.summary.create_file_writer(run_dir).as_default(): hp.hparams(hparams) model = create_model(hparams)
HP_UNITS_2 = hp.HParam('UNITS_2', hp.Discrete([0, 16, 64])) HP_LEARNING_RATE = hp.HParam('learning_rate', hp.Discrete([0.0001])) HP_CNN_FILTER_1 = hp.HParam('filter_1', hp.Discrete([8, 32, 64])) HP_CNN_KERNEL_1 = hp.HParam('kernel_1', hp.Discrete([5])) HP_CNN_FILTER_KERNEL_2 = hp.HParam( 'filter_kernel_2', hp.Discrete([ '[0, 0]', '[8, 5]', '[8, 10]', '[32, 5]', '[32, 10]', '[64, 5]', '[64, 10]' ])) with tf.summary.create_file_writer('logs/81-small-v1').as_default(): hp.hparams_config( hparams=[ HP_UNITS_1, HP_UNITS_2, HP_LEARNING_RATE, HP_CNN_KERNEL_1, HP_CNN_FILTER_KERNEL_2, HP_CNN_FILTER_1 ], metrics=[hp.Metric('accuracy', display_name='Accuracy')], ) #%% def train_test_model(logdir, hparams): start = timer() classifier = tf.keras.Sequential() classifier.add( tf.keras.layers.Conv2D(filters=int(hparams[HP_CNN_FILTER_1]), kernel_size=int(hparams[HP_CNN_KERNEL_1]), activation='relu', input_shape=(x_train[0].shape[0], x_train[0].shape[1], 1)))
# 1. Number of units in the first dense layer # 2. Dropout rate in the dropout layer # 3. Optimizer # # List the values to try, and log an experiment configuration to TensorBoard. This step is optional: you can provide domain information to enable more precise filtering of hyperparameters in the UI, and you can specify which metrics should be displayed. # %% HP_NUM_UNITS = hp.HParam("num_units", hp.Discrete([16, 32])) HP_DROPOUT = hp.HParam("dropout", hp.RealInterval(0.1, 0.2)) HP_OPTIMIZER = hp.HParam("optimizer", hp.Discrete(["adam", "sgd"])) METRIC_ACCURACY = "accuracy" with tf.summary.create_file_writer("logs/hparam_tuning").as_default(): hp.hparams_config( hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER], metrics=[hp.Metric(METRIC_ACCURACY, display_name="Accuracy")], ) # %% [markdown] # If you choose to skip this step, you can use a string literal wherever you would otherwise use an `HParam` value: e.g., `hparams['dropout']` instead of `hparams[HP_DROPOUT]`. # %% [markdown] # ## 2. Adapt TensorFlow runs to log hyperparameters and metrics # # The model will be quite simple: two dense layers with a dropout layer between them. The training code will look familiar, although the hyperparameters are no longer hardcoded. Instead, the hyperparameters are provided in an `hparams` dictionary and used throughout the training function: # %% def train_test_model(hparams): """ """ model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(),
# Defining constants EPOCHS = 15 BATCH_SIZE = 64 # Defining the hyperparameters we would tune, and their values to be tested HP_FILTER_SIZE = hp.HParam('filter_size', hp.Discrete([3, 5, 7])) HP_FILTER_NUM = hp.HParam('filters_number', hp.Discrete([32, 64, 96, 128])) METRIC_ACCURACY = 'accuracy' # Logging setup info with tf.summary.create_file_writer( r'logs/Model_1_All/hparam_tuning/').as_default(): hp.hparams_config( hparams=[HP_FILTER_SIZE, HP_FILTER_NUM], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')], ) # Wrapping our model and training in a function print( '=============Wrapping our model and training in a function==================' ) def train_test_model(hparams, session_num): # Outlining the model/architecture of our CNN ### Removed the 2nd Con2D, and All the MaxPooling layers. ### ### Changed the Dense to 4. ### model = tf.keras.Sequential([ tf.keras.layers.Conv2D(hparams[HP_FILTER_NUM], hparams[HP_FILTER_SIZE],
(X_train, y_train) = preprocessing(train_tweets) X_test = process_test_data(test_tweets) y_test = to_categorical(y_test) HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([15])) HP_BATCH_SIZE = hp.HParam('batch_size', hp.Discrete([32, 64])) HP_ACT_FUNC = hp.HParam('activation_function', hp.Discrete(["softmax"])) HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(["rmsprop"])) METRIC_ACCURACY = 'accuracy' with tf.summary.create_file_writer('logs/hparam_tuning/' + dateAndTimeNow).as_default(): hp.hparams_config( hparams=[HP_NUM_UNITS, HP_BATCH_SIZE, HP_ACT_FUNC], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')], ) def train_test_model2(hparams): start_time = time.time() input = Input(shape=(X_train.shape[1:])) output = tf.expand_dims(input, axis=-1) output = GRU(units=hparams[HP_NUM_UNITS], activation="relu", return_sequences=True)(output) output = GRU( units=hparams[HP_NUM_UNITS], activation="relu", )(output)
def main(argv): # set up Grappler for graph optimization # Ref: https://www.tensorflow.org/guide/graph_optimization @contextlib.contextmanager def options(options): old_opts = tf.config.optimizer.get_experimental_options() tf.config.optimizer.set_experimental_options(options) try: yield finally: tf.config.optimizer.set_experimental_options(old_opts) # ----------------------------------------------------------------- # Creating dataloaders for training and validation logging.info("Creating the dataloader from: %s..." % FLAGS.tfrecord_dir) train_dataset = dataset_coco.prepare_dataloader(tfrecord_dir=FLAGS.tfrecord_dir, img_size=256, batch_size=FLAGS.batch_size, subset='train') valid_dataset = dataset_coco.prepare_dataloader(tfrecord_dir=FLAGS.tfrecord_dir, img_size=256, batch_size=1, subset='val') # ----------------------------------------------------------------- # Creating the instance of the model specified. logging.info("Creating the model instance of YOLACT") YOLACT = lite.MyYolact(input_size=256, fpn_channels=96, feature_map_size=[32, 16, 8, 4, 2], num_class=13, # 12 classes + 1 background num_mask=32, aspect_ratio=[1, 0.5, 2], scales=[24, 48, 96, 192, 384]) model = YOLACT.gen() # add weight decay # for layer in model.layers: # if isinstance(layer, tf.keras.layers.Conv2D) or isinstance(layer, tf.keras.layers.Dense): # layer.add_loss(lambda: tf.keras.regularizers.l2(FLAGS.weight_decay)(layer.kernel)) # if hasattr(layer, 'bias_regularizer') and layer.use_bias: # layer.add_loss(lambda: tf.keras.regularizers.l2(FLAGS.weight_decay)(layer.bias)) # ----------------------------------------------------------------- # Choose the Optimizor, Loss Function, and Metrics, learning rate schedule logging.info("Initiate the Optimizer and Loss function...") lr_schedule = learning_rate_schedule.Yolact_LearningRateSchedule(warmup_steps=500, warmup_lr=1e-4, initial_lr=FLAGS.lr) HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete([ 'SGD' ])) with tf.summary.create_file_writer('logs/hparam_tuning').as_default(): hp.hparams_config( hparams=[HP_OPTIMIZER], metrics=[hp.Metric('train_loss', display_name='Train Loss'), hp.Metric('valid_loss', display_name='Valid Loss')], ) # ----------------------------------------------------------------- # Setup the TensorBoard for better visualization # Ref: https://www.tensorflow.org/tensorboard/get_started # ----------------------------------------------------------------- for trial_idx, optimizer_name in enumerate(HP_OPTIMIZER.domain.values): logging.info("Setup the TensorBoard...") current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") train_log_dir = './logs/gradient_tape/' + current_time + f'-{optimizer_name}' + '/train' test_log_dir = './logs/gradient_tape/' + current_time + f'-{optimizer_name}' + '/test' train_summary_writer = tf.summary.create_file_writer(train_log_dir) test_summary_writer = tf.summary.create_file_writer(test_log_dir) hparams = { HP_OPTIMIZER: optimizer_name } optimizer_map = { 'SGD': tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=FLAGS.momentum), 'NesterovSGD': tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=FLAGS.momentum, nesterov=True), 'Adam': tf.keras.optimizers.Adam(learning_rate=0.001) } optimizer = optimizer_map[hparams[HP_OPTIMIZER]] # Start the Training and Validation Process logging.info("Start the training process...") # setup checkpoints manager checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, model=model) manager = tf.train.CheckpointManager(checkpoint, directory=f'./checkpoints-{optimizer_name}', max_to_keep=5) # restore from latest checkpoint and iteration status = checkpoint.restore(manager.latest_checkpoint) if manager.latest_checkpoint: logging.info("Restored from {}".format(manager.latest_checkpoint)) else: logging.info("Initializing from scratch.") # Prepare Trainer trainer = Trainer(model, optimizer) best_val = 1e10 iterations = checkpoint.step.numpy() for image, labels in train_dataset: # check iteration and change the learning rate if iterations > FLAGS.train_iter: break logging.info(f'Check iterations: {iterations}') checkpoint.step.assign_add(1) iterations += 1 trainer.valid_step(image, labels)
('train_rois_per_image', hp.choice('train_rois_per_image', [50, 100, 200])), ('detection_min_confidence', hp.choice('detection_min_confidence', [0.6, 0.7, 0.8])), ('optimizer', hp.choice('optimizer', ['ADAM', 'SGD'])) ]) METRIC_MAP = 'mean average precision' METRIC_F1S = 'f1 score' with tf.summary.create_file_writer('logs/hparam_tuning_sund3').as_default(): tbhp.hparams_config(hparams=[ HP_BACKBONE, HP_DETECTION_MIN_CONFIDENCE, HP_TRAIN_ROIS_PER_IMAGE, HP_OPTIMIZER ], metrics=[ tbhp.Metric(METRIC_MAP, display_name='mean average precision'), tbhp.Metric(METRIC_F1S, display_name='f1 score') ]) def inference_calculation(config, model_dir, model_path, dataset_val): model = modellib.MaskRCNN(mode="inference", config=config, model_dir=model_dir, unique_log_name=False) model.load_weights(model_path, by_name=True) mean_average_precision, f1_score = calc_mean_average_precision( dataset_val=dataset_val, inference_config=config, model=model)
HP_connected_weights: HP_connected_weights.domain.values, HP_samples: HP_samples.domain.values, }) with tf.summary.create_file_writer('logs/hparam_tuning').as_default(): hp.hparams_config( hparams=[ HP_seed, HP_components, HP_encoder_dims, HP_mixture_dims, HP_bn, HP_connected_weights, HP_samples, ], metrics=[ METRIC_AMI_TRAIN, METRIC_AMI_TEST, METRIC_PURITY_TRAIN, METRIC_PURITY_TEST, METRIC_max_cluster_attachment_test, METRIC_loss, METRIC_likelihood, METRIC_z_prior_entropy, METRIC_y_prior_entropy, ], ) def train_test_model(run_id, hparams, X_train, y_train, X_test, y_test): #hp.hparams(hparams) # record the values used in this trial
def main(self): # DATA will contain a list of the paths to different binary data files. There should be one for each of the # "different types of systems" (e.g. z3, z2, high temp, etc). There is no guarantee that there are the same # number of configurations in each file though but the function below takes care of all of that for us to make # sure we get a balanced dataset and that it meets some basic requirements. # all_data = np.load(DATA) all_data, data_labels = self.import_data(self.data) n_records = len(all_data) x_train = all_data[:n_records - int(n_records / 4)] x_test = all_data[int(n_records / 4):] x_train = np.reshape(x_train, (len(x_train), self.L * 2, self.L * 2, 1)) x_test = np.reshape(x_test, (len(x_test), self.L * 2, self.L * 2, 1)) np.save(self.training_data_location, x_train) np.save(self.testing_data_location, x_test) np.save(self.data_label_location, data_labels) with tf.summary.create_file_writer( os.path.join(self.run_location, 'tensorboard_raw', self.tensorboard_sub_dir)).as_default(): hp.hparams_config(hparams=[ self.hp_batch_size, self.hp_n_layers, self.hp_feature_map_step, self.hp_stride_size, self.hp_use_batch_normalization, self.hp_use_dropout ], metrics=METRICS) c = 0 autoencoder = None for batch_size in self.hp_batch_size.domain.values: for n_layers in self.hp_n_layers.domain.values: for f_map_step in self.hp_feature_map_step.domain.values: for stride in self.hp_stride_size.domain.values: for use_batch_normalization in self.hp_use_batch_normalization.domain.values: for use_dropout in self.hp_use_dropout.domain.values: hyper_params = { self.hp_batch_size: batch_size, self.hp_n_layers: n_layers, self.hp_feature_map_step: f_map_step, self.hp_stride_size: stride, self.hp_use_batch_normalization: use_batch_normalization, self.hp_use_dropout: use_dropout } c += 1 run_name = f"run-{c}" print('--- Starting trial: %s' % run_name) print({ h.name: hyper_params[h] for h in hyper_params }) self.run( os.path.join(self.run_location, 'tensorboard_raw', self.tensorboard_sub_dir, run_name), hyper_params, x_test, x_train) # After each run lets attempt to log a sample of activations for the different layers best_autoencoder = keras.models.load_model(self.checkpoint_file) best_autoencoder.save(self.best_model_file) # Get the encoder piece of the autoencoder. We call this the "activation model". This is the full model up to # the bottle neck. layers_to_encoded = int(len(best_autoencoder.layers) / 2) print(layers_to_encoded) layer_activations = [ layer.output for layer in best_autoencoder.layers[:layers_to_encoded] ] activation_model = models.Model(inputs=best_autoencoder.input, outputs=layer_activations) activation_model.save(self.best_activations_file)
def set_tensorboard(self,hparams): with tf.summary.create_file_writer('logs/hparam_tunning').as_default(): hp.hparams_config( hparams=hparams, metrics=[hp.Metric('accuracy', display_name='Accuracy')] )
HP_NUM_UNITS_DLl = hp.HParam('dl1_num_units', hp.Discrete([512])) HP_DROPOUT_DL1 = hp.HParam('dl1_dropout', hp.Discrete([0.25])) HP_NUM_UNITS_DL2 = hp.HParam('dl2_num_units', hp.Discrete([1024])) HP_DROPOUT_DL2 = hp.HParam('dl2_dropout', hp.Discrete([0.5])) HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['rmsprop'])) METRIC_ACCURACY = 'accuracy' with tf.summary.create_file_writer('logs/hparam_tuning_DL').as_default(): hp.hparams_config( hparams=[ HP_NUM_UNITS_CLl, HP_KERNEL_SIZE_CLl, HP_STRIDES_CLl, HP_NUM_UNITS_CL2, HP_KERNEL_SIZE_CL2, HP_STRIDES_CL2, HP_POOL_SIZE_PL1, HP_DROPOUT_CL2, HP_NUM_UNITS_CL3, HP_KERNEL_SIZE_CL3, HP_STRIDES_CL3, HP_NUM_UNITS_CL4, HP_KERNEL_SIZE_CL4, HP_STRIDES_CL4, HP_POOL_SIZE_PL2, HP_DROPOUT_CL4, HP_NUM_UNITS_DLl, HP_DROPOUT_DL1, HP_NUM_UNITS_DL2, HP_DROPOUT_DL2, HP_OPTIMIZER ], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')], ) session_num = 1 for a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u in product( HP_NUM_UNITS_CLl.domain.values, HP_KERNEL_SIZE_CLl.domain.values, HP_STRIDES_CLl.domain.values, HP_NUM_UNITS_CL2.domain.values, HP_KERNEL_SIZE_CL2.domain.values, HP_STRIDES_CL2.domain.values, HP_POOL_SIZE_PL1.domain.values, HP_DROPOUT_CL2.domain.values, HP_NUM_UNITS_CL3.domain.values, HP_KERNEL_SIZE_CL3.domain.values, HP_STRIDES_CL3.domain.values, HP_NUM_UNITS_CL4.domain.values, HP_KERNEL_SIZE_CL4.domain.values, HP_STRIDES_CL4.domain.values,