x_train = np.asarray(x_train)
y_train = np.asarray(y_train)
x_predic = x_test.iloc[[1]]
x_test = np.asarray(x_test)
y_test = np.asarray(y_test)
#hparam tunning
HP_NUM_UNITS_ONE = hp.HParam('num_units_one', hp.Discrete([5, 10, 20]))
HP_NUM_UNITS_TWO = hp.HParam('num_units_two', hp.Discrete([10, 20, 40]))
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd']))
METRIC_ACCURACY = 'accuracy'
with tf.summary.create_file_writer(output_dir).as_default():
hp.hparams_config(
hparams=[HP_NUM_UNITS_ONE, HP_NUM_UNITS_TWO, HP_OPTIMIZER],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
)
def train_test_model(hparams):
model = tf.keras.models.Sequential()
model.add(
layers.Dense(hparams[HP_NUM_UNITS_ONE],
activation='relu',
input_shape=(13, )))
model.add(layers.Dense(hparams[HP_NUM_UNITS_TWO], activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(
optimizer=hparams[HP_OPTIMIZER],
loss='binary_crossentropy',
metrics=['accuracy'],
def random_hparam_search(cfg, data, callbacks, log_dir):
'''
Conduct a random hyperparameter search over the ranges given for the hyperparameters in config.yml and log results
in TensorBoard. Model is trained x times for y random combinations of hyperparameters.
:param cfg: Project config
:param data: Dict containing the partitioned datasets
:param callbacks: List of callbacks for Keras model (excluding TensorBoard)
:param log_dir: Base directory in which to store logs
:return: (Last model trained, resultant test set metrics, test data generator)
'''
# Define HParam objects for each hyperparameter we wish to tune.
hp_ranges = cfg['HP_SEARCH']['RANGES']
HPARAMS = []
HPARAMS.append(
hp.HParam('KERNEL_SIZE', hp.Discrete(hp_ranges['KERNEL_SIZE'])))
HPARAMS.append(
hp.HParam('MAXPOOL_SIZE', hp.Discrete(hp_ranges['MAXPOOL_SIZE'])))
HPARAMS.append(
hp.HParam('INIT_FILTERS', hp.Discrete(hp_ranges['INIT_FILTERS'])))
HPARAMS.append(
hp.HParam(
'FILTER_EXP_BASE',
hp.IntInterval(hp_ranges['FILTER_EXP_BASE'][0],
hp_ranges['FILTER_EXP_BASE'][1])))
HPARAMS.append(
hp.HParam('NODES_DENSE0', hp.Discrete(hp_ranges['NODES_DENSE0'])))
HPARAMS.append(
hp.HParam(
'CONV_BLOCKS',
hp.IntInterval(hp_ranges['CONV_BLOCKS'][0],
hp_ranges['CONV_BLOCKS'][1])))
HPARAMS.append(hp.HParam('DROPOUT', hp.Discrete(hp_ranges['DROPOUT'])))
HPARAMS.append(
hp.HParam('LR', hp.RealInterval(hp_ranges['LR'][0],
hp_ranges['LR'][1])))
HPARAMS.append(hp.HParam('OPTIMIZER', hp.Discrete(hp_ranges['OPTIMIZER'])))
HPARAMS.append(hp.HParam('L2_LAMBDA', hp.Discrete(hp_ranges['L2_LAMBDA'])))
HPARAMS.append(
hp.HParam('BATCH_SIZE', hp.Discrete(hp_ranges['BATCH_SIZE'])))
HPARAMS.append(
hp.HParam('IMB_STRATEGY', hp.Discrete(hp_ranges['IMB_STRATEGY'])))
# Define test set metrics that we wish to log to TensorBoard for each training run
HP_METRICS = [
hp.Metric(metric, display_name='Test ' + metric)
for metric in cfg['HP_SEARCH']['METRICS']
]
# Configure TensorBoard to log the results
with tf.summary.create_file_writer(log_dir).as_default():
hp.hparams_config(hparams=HPARAMS, metrics=HP_METRICS)
# Complete a number of training runs at different hparam values and log the results.
repeats_per_combo = cfg['HP_SEARCH'][
'REPEATS'] # Number of times to train the model per combination of hparams
num_combos = cfg['HP_SEARCH'][
'COMBINATIONS'] # Number of random combinations of hparams to attempt
num_sessions = num_combos * repeats_per_combo # Total number of runs in this experiment
model_type = 'DCNN_BINARY' if cfg['TRAIN'][
'CLASS_MODE'] == 'binary' else 'DCNN_MULTICLASS'
trial_id = 0
for group_idx in range(num_combos):
rand = random.Random()
HPARAMS = {h: h.domain.sample_uniform(rand) for h in HPARAMS}
hparams = {h.name: HPARAMS[h]
for h in HPARAMS} # To pass to model definition
for repeat_idx in range(repeats_per_combo):
trial_id += 1
print("Running training session %d/%d" % (trial_id, num_sessions))
print("Hparam values: ", {h.name: HPARAMS[h] for h in HPARAMS})
trial_logdir = os.path.join(
log_dir, str(trial_id)) # Need specific logdir for each trial
callbacks_hp = callbacks + [
TensorBoard(
log_dir=trial_logdir, profile_batch=0, write_graph=False)
]
# Set values of hyperparameters for this run in config file.
for h in hparams:
if h in ['LR', 'L2_LAMBDA']:
val = 10**hparams[
h] # These hyperparameters are sampled on the log scale.
else:
val = hparams[h]
cfg['NN'][model_type][h] = val
# Set some hyperparameters that are not specified in model definition.
cfg['TRAIN']['BATCH_SIZE'] = hparams['BATCH_SIZE']
cfg['TRAIN']['IMB_STRATEGY'] = hparams['IMB_STRATEGY']
# Run a training session and log the performance metrics on the test set to HParams dashboard in TensorBoard
with tf.summary.create_file_writer(trial_logdir).as_default():
hp.hparams(HPARAMS, trial_id=str(trial_id))
model, test_metrics, test_generator = train_model(cfg,
data,
callbacks_hp,
verbose=0)
for metric in HP_METRICS:
if metric._tag in test_metrics:
tf.summary.scalar(metric._tag,
test_metrics[metric._tag],
step=1) # Log test metric
return
def learn(self, symbol_list):
if self.args.image == "generate":
for symbol in symbol_list:
start_time = time.time()
print("Starting to generate images for: ", symbol)
GenerateImagesNoHold.generate_buy_sell_images(
symbol, str(self.from_date), str(self.to_date))
end_time = time.time()
print("Generated images for stocks: ", symbol, " in: ",
(end_time - start_time), " seconds")
training_data = []
for symbol in symbol_list:
for category in self.categories:
path = os.path.join(Constants.IMAGE_DIR,
category.value + "/" + symbol)
class_num = self.categories.index(category)
for img in os.listdir(path):
try:
img_arr = cv2.imread(os.path.join(path, img),
cv2.IMREAD_GRAYSCALE)
resized_arr = cv2.resize(
img_arr, (self.IMG_SIZE, self.IMG_SIZE))
training_data.append([resized_arr, class_num])
except Exception as e:
print(e)
random.shuffle(training_data)
X = []
y = []
for features, labels in training_data:
X.append(features)
y.append(labels)
X = np.array(X).reshape(-1, self.IMG_SIZE, self.IMG_SIZE, 1)
y = np.array(y)
self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
X, y, test_size=.3)
self.X_train = self.X_train / 255.0
model = Sequential()
model.add(Conv2D(128, (3, 3), input_shape=X.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, (3, 3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, (3, 3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(32))
model.add(Activation("relu"))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(
loss="binary_crossentropy",
optimizer="rmsprop",
metrics=['accuracy'],
)
model_save_path = Constants.MODEL_DIR + symbol + Constants.MODEL_EXTENSION
model_cp = ModelCheckpoint(model_save_path,
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
lr_callback = LearningRateScheduler(self.lr_schedule)
model.fit(
self.X_train,
self.y_train,
batch_size=16,
epochs=self.EPOCHS,
validation_split=0.2,
callbacks=[self.tensorboard, model_cp, self.early_stopping])
val_loss, val_acc = model.evaluate(self.X_test, self.y_test)
print('Accuracy for ', symbol, ": ", val_loss, val_acc)
model.save(model_save_path)
accuracy = False
if accuracy:
# TENSORBOARD CODE
train_dataset = tf.data.Dataset.from_tensor_slices(
(self.X_train, self.y_train / 1.0))
test_dataset = tf.data.Dataset.from_tensor_slices(
(self.X_test / 255.0, self.y_test / 1.0))
train_dataset = train_dataset.shuffle(60000).batch(64)
test_dataset = test_dataset.batch(64)
current_time = datetime.datetime.now().strftime(
"%Y%m%d-%H%M%S")
train_summary_writer = tf.summary.create_file_writer(
'logs/{}'.format(self.NAME) + '/train')
test_summary_writer = tf.summary.create_file_writer(
'logs/{}'.format(self.NAME) + '/test')
for epoch in range(self.EPOCHS):
for (x_train, y_train) in train_dataset:
self.train_step(model, self.optimizer, x_train,
y_train)
with train_summary_writer.as_default():
tf.summary.scalar('loss',
self.train_loss.result(),
step=epoch)
tf.summary.scalar('accuracy',
self.train_accuracy.result(),
step=epoch)
for (x_test, y_test) in test_dataset:
self.test_step(model, x_test, y_test)
with test_summary_writer.as_default():
tf.summary.scalar('loss',
self.test_loss.result(),
step=epoch)
tf.summary.scalar('accuracy',
self.test_accuracy.result(),
step=epoch)
template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'
print(
template.format(epoch + 1, self.train_loss.result(),
self.train_accuracy.result() * 100,
self.test_loss.result(),
self.test_accuracy.result() * 100))
# Reset metrics every epoch
self.train_loss.reset_states()
self.test_loss.reset_states()
self.train_accuracy.reset_states()
self.test_accuracy.reset_states()
tuning = False
if tuning:
with tf.summary.create_file_writer(
'logs/hparam_tuning').as_default():
hp.hparams_config(
hparams=[
self.HP_NUM_UNITS, self.HP_DROPOUT,
self.HP_OPTIMIZER, self.HP_ACTIVATION
],
metrics=[
hp.Metric(self.METRIC_ACCURACY,
display_name='Accuracy')
],
)
session_num = 0
for num_units in self.HP_NUM_UNITS.domain.values:
for dropout_rate in (self.HP_DROPOUT.domain.min_value,
self.HP_DROPOUT.domain.max_value):
for optimizer in self.HP_OPTIMIZER.domain.values:
for activation in self.HP_ACTIVATION.domain.values:
hparams = {
self.HP_NUM_UNITS: num_units,
self.HP_DROPOUT: dropout_rate,
self.HP_OPTIMIZER: optimizer,
self.HP_ACTIVATION: activation
}
run_name = "run-%d" % session_num
print('--- Starting trial: %s' % run_name)
print({h.name: hparams[h] for h in hparams})
self.run('logs/hparam_tuning/' + run_name,
hparams)
session_num += 1
def param_tune(self,):
"""
Tuning the hyperparameters
****Helper Variables****
self.HP_NUM_UNITS:tensorboard.plugins.hparams.summary_v2.HParam
Number of LSTM units
self.HP_DROPOUT:tensorboard.plugins.hparams.summary_v2.HParam
Dropout rate
self.HP_OPTIMIZER:tensorboard.plugins.hparams.summary_v2.HParam
Optimizer
self.HP_LEARNING_RATE:tensorboard.plugins.hparams.summary_v2.HParam
Learning Rate
self.name: str
name of the data file
self.model_choice: str
input for chosing model to tune
eventlog : str
file to be processed
F: Datahandler Obj
File reading and processing
divisor: float
average time between current and first events
divisor2: float
average time between current and first events
char_indices : dict
ascii coded characters of the unique activities to integer indices
indices_char: dict
integer indices to ascii coded characters of the unique activities
target_char_indices: dict
ascii coded characters of the target unique activities to integer indices
(target includes one excess activity '!' case end)
target_indices_char: dict
integer indices to ascii coded characters of the target unique activities
maxlen
maximum length of cases
chars:list
ascii coded characters of the activities.
target_chars: list
ascii coded characters of the target activities.
(target includes one excess activity '!' case end)
X: ndarray
training_set data
y_a: list
training_set labels for next activity
y_t: list
training_set list
METRIC_LOSS: str
Keeping track of the model loss
num_units: int
LSTM units parameter
dropout_rate:float
Dropoutrate
optimizer:str
Optimizer
learning_rate:float
learning_rate
cw: ComputeCW obj
class_weights: dict
class weights for activities
hparams: dict
hyperparameter for the run
session_num: int
session number of the run
run_dir: str
path to store tensorboard files
run_name: str
name of the tensorboard file
run_stat: str
hyperparameters used
hist: dict
history of the training
loss: list
val_loss of the last epoch
"""
eventlog=input("Enter the name of the file to be executed from data folder: ")
print(' ')
#Reading the Data
F = Datahandler()
self.name=F.read_data(eventlog)
#F.name is the name of the file
#F.spamread is the data of the file
spamreader,max_task = F.log2np()
D=Preprocess()
divisor,divisor2,divisor3 = D.divisor_cal(spamreader)
maxlen,chars,target_chars,char_indices,indices_char,target_char_indices,target_indices_char= D.dict_cal()
num_features = len(chars) + 5
X, y_a, y_t, d_t = D.training_set(num_features)
self.model_choice = input( "Please Select model: 1.CS, 2.TLSTM, 3.CS_TLSTM ")
self.HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([64,100]))
self.HP_DROPOUT = hp.HParam('dropout', hp.Discrete([0.0, 0.2]))
self.HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['nadam']))
self.HP_LEARNING_RATE = hp.HParam('learning_rate', hp.Discrete([0.0001,0.0002,0.001,0.002,0.01]))
METRIC_LOSS = 'loss'
path=os.path.abspath(os.curdir)
with tf.summary.create_file_writer(path+'/logs_'+self.name+'/'+self.model_choice+'/').as_default():
hp.hparams_config(
hparams=[self.HP_NUM_UNITS,self.HP_DROPOUT, self.HP_OPTIMIZER,self.HP_LEARNING_RATE],
metrics=[hp.Metric(METRIC_LOSS, display_name='Loss')],)
#Training
if self.model_choice == '1':
cw=ComputeCW()
class_weights=cw.compute_class_weight(F.spamread)
print('class_weights are: ', class_weights)
def run(run_dir,hparams):
with tf.summary.create_file_writer(run_dir).as_default():
M=CSModel(maxlen,
max_task,
target_chars,
self.name,
num_features)
loss,hist=M.train(X,y_a,y_t, class_weights,
hparams,self.HP_NUM_UNITS,self.HP_DROPOUT,
self.HP_OPTIMIZER,self.HP_LEARNING_RATE)
tf.summary.scalar(METRIC_LOSS, loss, step=1)
return loss,hist
elif self.model_choice == '2':
def run(run_dir,hparams):
with tf.summary.create_file_writer(run_dir).as_default():
M=ALL_TLSTM_Model(maxlen,
max_task,
target_chars,
self.name,
num_features)
loss,hist=M.train(X,d_t,y_a,y_t,hparams,self.HP_NUM_UNITS,self.HP_DROPOUT,
self.HP_OPTIMIZER,self.HP_LEARNING_RATE)
tf.summary.scalar(METRIC_LOSS, loss, step=1)
return loss,hist
elif self.model_choice == '3':
cw=ComputeCW()
class_weights=cw.compute_class_weight(F.spamread)
print('class_weights are: ', class_weights)
def run(run_dir,hparams):
with tf.summary.create_file_writer(run_dir).as_default():
M=CS_TLSTM_Model(maxlen,
max_task,
target_chars,
self.name,
num_features)
loss,hist=M.train(X,d_t,y_a,y_t,class_weights,hparams,self.HP_NUM_UNITS,self.HP_DROPOUT,
self.HP_OPTIMIZER,self.HP_LEARNING_RATE)
tf.summary.scalar(METRIC_LOSS, loss, step=1)
return loss,hist
session_num = 0
for num_units in (self.HP_NUM_UNITS.domain.values):
for dropout_rate in (self.HP_DROPOUT.domain.values):
for optimizer in self.HP_OPTIMIZER.domain.values:
for learning_rate in self.HP_LEARNING_RATE.domain.values:
hparams = {
self.HP_NUM_UNITS:num_units,
self.HP_DROPOUT: dropout_rate,
self.HP_OPTIMIZER: optimizer,
self.HP_LEARNING_RATE:learning_rate,
}
#run_name = "run-%d" % session_num
run_stat={h.name: hparams[h] for h in hparams}
print('Session',session_num)
run_name = str(run_stat.values())
#print('--- Starting trial: %s' % run_name)
run_stat={h.name: hparams[h] for h in hparams}
print(run_stat)
loss,hist = run(path+'/logs_'+self.name+'/'+ self.model_choice+'/', hparams)
with open(path+'/history/'+self.model_choice+'/'+self.name+run_name, "wb") as fp:
pickle.dump(hist, fp)
session_num += 1
print(X_train.shape, y_train.shape)
return X_train, y_train
# Part 2 - Building the RNN
HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([100]))
HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.11))
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['rmsprop']))
HP_OUTPUT = hp.HParam('output_number', hp.Discrete(list(range(1))))
HP_WINDOW = hp.HParam('window_size', hp.Discrete([1]))
#HP_OUTPUT = hp.HParam('output_number',hp.Discrete(list(range(n_outputs-1))))
METRIC_ACCURACY = 'accuracy'
hp.hparams_config(
hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER, HP_OUTPUT, HP_WINDOW],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
)
def run(run_dir, hparams):
hp.hparams(hparams) # record the values used in this trial
predicted = train_test_model(run_dir, hparams)
return predicted
optimizers.RMSprop(lr=0.001, rho=0.9, decay=0.9)
optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
def get_weighted_loss(weights):
def weighted_loss(y_true, y_pred):
print(X_scaled.shape,y_scaled.shape)
return X_scaled, y_scaled
# Part 2 - Building the RNN
HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([100]))
HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1,0.11))
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['rmsprop']))
HP_OUTPUT = hp.HParam('output_number',hp.Discrete(list(range(0,12))))
HP_WINDOW = hp.HParam('window_size',hp.Discrete([4]))
HP_HORIZON = hp.HParam('horizon_size',hp.Discrete([4]))
#HP_OUTPUT = hp.HParam('output_number',hp.Discrete(list(range(n_outputs-1))))
METRIC_ACCURACY = 'loss'
hp.hparams_config(hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER, HP_OUTPUT, HP_WINDOW, HP_HORIZON], metrics=[hp.Metric(METRIC_ACCURACY, display_name='loss')],)
optimizers.RMSprop(lr=0.001,rho=0.9,decay=0.9)
optimizers.SGD(lr=0.001,decay=1e-6,momentum=0.9,nesterov=True)
n_inputs = max(HP_OUTPUT.domain.values)# total number of inputs minus 1 as it starts from 0
def train_model(run_dir,hparams):
hp.hparams(hparams)
[X_train, y_train] = create_data(data_unscaled=data, start_train=start_train, end_train=end_train, n_windows=hparams[HP_WINDOW], n_outputs=hparams[HP_OUTPUT])
[X_test, y_test] = create_data(data_unscaled=data, start_train=end_train, end_train=end_test, n_windows=hparams[HP_WINDOW], n_outputs=hparams[HP_OUTPUT])
tf.compat.v1.keras.backend.clear_session()
model = Sequential()
model.add(TimeDistributed(Masking(mask_value=0., input_shape=(hparams[HP_WINDOW], n_inputs+1)), input_shape=(n_company, hparams[HP_WINDOW], n_inputs+1)))
model.add(TimeDistributed(LSTM(hparams[HP_NUM_UNITS], stateful=False, activation='tanh', return_sequences=True, input_shape=(hparams[HP_WINDOW], n_inputs+1), kernel_initializer='TruncatedNormal' ,bias_initializer=initializers.Constant(value=0.1), dropout=hparams[HP_DROPOUT] ,recurrent_dropout=hparams[HP_DROPOUT])))
from input_pipeline.data_preprocessing_visualization import r_train_ds, r_val_s, r_test_ds
#Define the hyperparameters
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'RMSProp']))
HP_LSTM_NEURONS = hp.HParam('LSTMNeurons', hp.Discrete([128,256]))
HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.2))
HP_EPOCHS = hp.HParam('epochs',hp.Discrete([5, 10]))
METRIC_ACCURACY = 'accuracy'
path_hparams = input('Enter the path to save the tuning logs: ')
#The logs will be created and stored in the following path
with tf.summary.create_file_writer(path_hparams + 'logs/hparam_tuning').as_default():
hp.hparams_config(
hparams = [HP_LSTM_NEURONS, HP_DROPOUT, HP_OPTIMIZER, HP_EPOCHS],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
)
#Hyperparamter model definition
def HParams_mdl():
inputs = keras.Input(shape=(N_WindowSize,N_Features))
x = layers.LSTM(hparams[HP_LSTM_NEURONS],return_sequences=True)(inputs)
x = layers.Dropout(hparams[HP_DROPOUT])(x)
x = layers.LSTM(128,return_sequences=True)(x)
outputs = layers.Dense(N_CLASSES, activation='softmax')(x)
mdl = keras.Model(inputs=inputs, outputs=outputs, name='HAR_model')
return mdl
def run(run_dir, hparams):
with tf.summary.create_file_writer(run_dir).as_default():
hp.hparams(hparams)
subset='training')
val_generator = datagen.flow_from_directory(base_dir,
target_size=(IMAGE_SIZE,
IMAGE_SIZE),
batch_size=BATCH_SIZE,
subset='validation')
HP_CONVO_FILTER = hp.HParam('filter', hp.Discrete([64, 128]))
HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.2))
METRIC_ACCURACY = 'accuracy'
with tf.summary.create_file_writer('logs/hparam_tuning').as_default():
hp.hparams_config(
hparams=[HP_CONVO_FILTER, HP_DROPOUT],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
)
def train_test_model(hparams):
IMG_SHAPE = (IMAGE_SIZE, IMAGE_SIZE, 3)
# Create the base model from the pre-trained model MobileNet V2
base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,
alpha=1.0,
include_top=False,
weights='imagenet')
base_model.trainable = False
model = tf.keras.Sequential([
gpu = args.gpu
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config = config)
K.set_session(sess)
HP_STRIDES = hp.HParam("strides", hp.Discrete([1,2,3]))
HP_DROPOUT = hp.HParam("cnn_dropout", hp.Discrete([0.3, 0.4, 0.5]))
# HP_FILTER = hp.HParam("filter_size", hp.Discrete([3,3],[2,2]))
METRIC_ACCURACY = "accuracy"
hp.hparams_config(
hparams = [HP_STRIDES, HP_DROPOUT],
metrics = [hp.Metric(METRIC_ACCURACY, display_name = "Accuarcy")]
)
class LearningRateSchedule(tf.keras.callbacks.Callback):
def __init__(self, schedule):
super(LearningRateSchedule, self).__init__()
self.schedule = schedule
def on_epoch_begin(self, epoch, logs=None):
if not hasattr(self.model.optimizer, 'lr'):
raise ValueError("Optimizer must have a lr attribute")
lr = float(tf.keras.backend.get_value(self.model.optimizer.lr))
scheduled_lr = self.schedule(epoch, lr)
tf.keras.backend.set_value(self.model.optimizer.lr, scheduled_lr)
print("\nEpoch {}: Learning Rate is {}.".format(epoch, scheduled_lr))
def objective(trial):
standard_object = PrunableEvaluateMNIST(
train_images=train_images,
test_images=test_images,
train_labels=train_labels,
test_labels=test_labels,
validation_data_proportion=(1-JUN_SHAO_TRAINING_PROPORTION),
)
session_hparams = {}
hparams_list = []
# Generate hyper-parameters
standard_object.number_of_conv_2d_filters = trial.suggest_int(
'number_of_conv_2d_filters',
4,
8,
)
session_hparams['number_of_conv_2d_filters'] = standard_object.number_of_conv_2d_filters
hparams_list.append(hp.HParam('number_of_conv_2d_filters', hp.RealInterval(float(4), float(8))))
standard_object.first_conv2d_kernel_dim = trial.suggest_int(
'first_conv2d_kernel_dim',
1,
2,
)
session_hparams['first_conv2d_kernel_dim'] = standard_object.first_conv2d_kernel_dim
hparams_list.append(hp.HParam('first_conv2d_kernel_dim', hp.RealInterval(float(1), float(2))))
standard_object.second_conv2d_kernel_dim = trial.suggest_int(
'second_conv2d_kernel_dim',
1,
2,
)
session_hparams['second_conv2d_kernel_dim'] = standard_object.second_conv2d_kernel_dim
hparams_list.append(hp.HParam('second_conv2d_kernel_dim', hp.RealInterval(float(1), float(2))))
standard_object.conv2d_layers_activation_func = trial.suggest_categorical(
'conv2d_layers_activation_func',
[
'relu',
'sigmoid',
'softplus',
]
)
session_hparams['conv2d_layers_activation_func'] = standard_object.conv2d_layers_activation_func
hparams_list.append(hp.HParam('second_conv2d_kernel_dim', hp.Discrete([
'relu',
'sigmoid',
'softplus',
])))
standard_object.number_hidden_conv_layers = trial.suggest_int(
'number_hidden_conv_layers',
0,
2,
)
session_hparams['number_hidden_conv_layers'] = standard_object.number_hidden_conv_layers
hparams_list.append(hp.HParam('number_hidden_conv_layers', hp.RealInterval(float(0), float(2))))
standard_object.batch_size_base_two_logarithm = trial.suggest_int(
'batch_size_base_two_logarithm',
0,
MAXIMUM_BATCH_SIZE_POWER_OF_TWO,
)
session_hparams['batch_size_base_two_logarithm'] = standard_object.batch_size_base_two_logarithm
hparams_list.append(hp.HParam('batch_size_base_two_logarithm', hp.RealInterval(float(0), float(MAXIMUM_BATCH_SIZE_POWER_OF_TWO))))
standard_object.adam_learning_rate = trial.suggest_uniform(
'adam_learning_rate',
0,
2,
)
session_hparams['adam_learning_rate'] = standard_object.adam_learning_rate
hparams_list.append(hp.HParam('adam_learning_rate', hp.RealInterval(float(0), float(2))))
standard_object.adam_beta_1 = trial.suggest_uniform(
'adam_beta_1',
0,
1,
)
session_hparams['adam_beta_1'] = standard_object.adam_beta_1
hparams_list.append(hp.HParam('adam_beta_1', hp.RealInterval(float(0), float(1))))
standard_object.adam_beta_2 = trial.suggest_uniform(
'adam_beta_2',
0,
1,
)
session_hparams['adam_beta_2'] = standard_object.adam_beta_2
hparams_list.append(hp.HParam('adam_beta_2', hp.RealInterval(float(0), float(1))))
standard_object.adam_epsilon = trial.suggest_uniform(
'adam_epsilon',
0,
1
)
session_hparams['adam_epsilon'] = standard_object.adam_epsilon
hparams_list.append(hp.HParam('adam_epsilon', hp.RealInterval(float(0), float(1))))
# Train and validate using hyper-parameters generated above
clear_session()
classifier_model = models.Sequential()
classifier_model.add(layers.Conv2D(
standard_object.number_of_conv_2d_filters,
(standard_object.first_conv2d_kernel_dim, standard_object.second_conv2d_kernel_dim),
activation=standard_object.conv2d_layers_activation_func,
input_shape=(28, 28, 1),
))
classifier_model.add(layers.MaxPooling2D((2, 2), strides=2))
for level in range(standard_object.number_hidden_conv_layers):
classifier_model.add(layers.Conv2D(
standard_object.number_of_conv_2d_filters,
(standard_object.first_conv2d_kernel_dim, standard_object.second_conv2d_kernel_dim),
activation=standard_object.conv2d_layers_activation_func,
))
classifier_model.add(layers.MaxPooling2D((2, 2), strides=2))
classifier_model.add(layers.Flatten())
classifier_model.add(layers.Dense(10, activation='softmax'))
standard_object.optimizer = Adam(
learning_rate=standard_object.adam_learning_rate,
beta_1=standard_object.adam_beta_1,
beta_2=standard_object.adam_beta_2,
epsilon=standard_object.adam_epsilon,
amsgrad=0, # False
)
classifier_model.compile(
optimizer=standard_object.optimizer,
loss=CategoricalCrossentropy(),
metrics=[CategoricalAccuracy()],
)
output_dir = os.path.join(stem_output_dir, ('trial_' + datetime.now().strftime('%Y.%m.%d_%H.%M.%S')))
if not(os.path.exists(output_dir)):
os.mkdir(output_dir)
tb_callback = TbCallback(
log_dir=output_dir,
histogram_freq=1,
write_graph=1, # True
)
standard_object.callbacks.append(tb_callback) # Append to callbacks list
hp.hparams_config(
hparams=hparams_list,
metrics=[hp.Metric(CategoricalAccuracy().name, display_name=CategoricalAccuracy().name)],
)
hp_callback = hp.KerasCallback(writer=output_dir, hparams=session_hparams)
standard_object.callbacks.append(hp_callback) # Append to callbacks list
es_callback = EarlyStopping(
monitor='val_loss',
min_delta=EARLY_STOPPING_SIGNIFICANT_DELTA,
patience=EARLY_STOPPING_PATIENCE_PARAMETER,
verbose=VERBOSITY_LEVEL_FOR_TENSORFLOW,
mode='auto',
baseline=None,
restore_best_weights=1, # True
)
standard_object.callbacks.append(es_callback) # Append to callbacks list
# Append tf.keras pruner for later use during study
keras_pruner = optuna.integration.TFKerasPruningCallback(
trial,
'val_categorical_accuracy',
)
standard_object.callbacks.append(keras_pruner) # Append to callbacks list
standard_object.set_batch_size(standard_object.batch_size_base_two_logarithm)
standard_object.stratified_split_for_training_and_validation()
classifier_model.fit(
standard_object.train_split_images,
standard_object.train_split_labels,
epochs=MAXIMUM_EPOCHS_TO_TRAIN,
validation_data=standard_object.validate_split_data,
verbose=VERBOSITY_LEVEL_FOR_TENSORFLOW,
callbacks=standard_object.callbacks,
batch_size=standard_object.batch_size,
use_multiprocessing=1, # True
workers=4,
)
# Evaluate performance on test data and report score
test_metrics = classifier_model.evaluate(
standard_object.test_images,
standard_object.test_labels,
batch_size=standard_object.batch_size,
)
test_results_dict = {output: test_metrics[i] for i, output in enumerate(classifier_model.metrics_names)}
return(test_results_dict['categorical_accuracy'])
tf.summary.scalar('critic loss',
self.critic_loss,
step=self.n_train)
#tf.summary.histogram('critic weights',self.critic.trainable_weights[0],step=self.n_train)
#Start script here
mv_envs = []
n = 0
trials = 1
if linux:
with tf.summary.create_file_writer(os.getcwd() + '/logs').as_default():
hp.hparams_config(
hparams=[
FORCE_SCALE, UPDATE_FREQ, TOLERANCE, MAX_STEPS, MAX_SESSIONS,
CDIV, VAL_SCALE, LR_CTBG, LR_CRITIC, UNIT_1, UNIT_2, UNIT_3,
TAU
],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Reward')],
)
else:
with tf.summary.create_file_writer(os.getcwd() + '\\logs').as_default():
hp.hparams_config(
hparams=[
FORCE_SCALE, UPDATE_FREQ, TOLERANCE, MAX_STEPS, MAX_SESSIONS,
CDIV, VAL_SCALE, LR_CTBG, LR_CRITIC, UNIT_1, UNIT_2, UNIT_3,
TAU
],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Reward')],
)
for a in FORCE_SCALE.domain.values:
def train_models(
input_dir,
basenames,
clades,
merge_behaviour='evenly',
split_specifications={
'train': {
'name': 'train',
'wanted_models': [0, 1],
'interweave_models': [.67, .33],
'repeat_models': [True, True]
},
'val': {
'name': 'val',
'wanted_models': [0, 1],
'interweave_models': True,
'repeat_models': [False, False]
},
'test': {
'name': 'test',
'wanted_models': [0, 1],
'interweave_models': True,
'repeat_models': [False, False]
},
},
tuple_length=1,
use_amino_acids=False,
used_codons=False,
model_hyperparameters={
"tcmc_rnn": {
"tcmc_models": [
8,
],
"rnn_type": [
"gru",
],
"rnn_units": [
32,
],
"dense_dimension": [
16,
],
},
"tcmc_mean_log": {
"tcmc_models": [
8,
],
"sequence_length_as_feature": [
False,
],
"dense1_dimension": [
16,
],
"dense2_dimension": [
16,
],
},
},
model_training_callbacks={
"tcmc_rnn": {},
"tcmc_mean_log": {},
},
batch_size=30,
batches_per_epoch=100,
epochs=40,
save_model_weights=True,
log_basedir='logs',
saved_weights_basedir='saved_weights',
verbose=True,
):
"""
TODO: Write Docstring
"""
print("type of clades:", type(clades))
# calculate some features from the input
num_leaves = database_reader.num_leaves(clades)
tuple_length = 3 if used_codons else tuple_length
alphabet_size = 4**tuple_length if not use_amino_acids else 20**tuple_length
# evaluate the split specifications
splits = {'train': None, 'val': None, 'test': None}
num_classes = 0
for k in split_specifications:
if k in splits.keys():
try:
splits[k] = database_reader.DatasetSplitSpecification(
**split_specifications[k])
num_classes = len(
splits[k].wanted_models) if num_classes < len(
splits[k].wanted_models) else num_classes
except TypeError as te:
raise Exception(
f"Invalid split specification for '{k}': {split_specifications[k]}"
) from te
# read the datasets for each wanted basename
wanted_splits = [split for split in splits.values() if split != None]
unmerged_datasets = {
b: database_reader.get_datasets(input_dir,
b,
wanted_splits,
num_leaves=num_leaves,
alphabet_size=alphabet_size,
seed=None,
buffer_size=1000,
should_shuffle=True)
for b in basenames
}
if any(['train' not in unmerged_datasets[b] for b in basenames]):
raise Exception("A 'train' split must be specified!")
# merge the respective splits of each basename
datasets = {}
merge_behaviour = merge_behaviour if len(
merge_behaviour) > 1 else merge_behaviour[0]
weights = len(basenames) * [1 / len(basenames)] # evenly is default
if isinstance(merge_behaviour, str):
if merge_behaviour != "evenly":
print(f'Unknown merge beheaviour, merging evenly.')
else:
if len(merge_behaviour) > 1:
# check whether the custom weights are correct
# expecting a list of weights
try:
merge_behaviour = [float(w) for w in merge_behaviour]
except ValueError:
print(
f'Expected a list of floats in merge_behaviour. However merge_behaviour = {merge_behaviour}'
)
print(f'Will use even merge_behaviour = {weights} instead.')
if len(merge_behaviour) == len(basenames) \
and all([isinstance(x, float) for x in merge_behaviour]) \
and all([x >= 0 for x in merge_behaviour]) \
and sum(merge_behaviour) == 1:
weights = merge_behaviour
merge_ds = tf.data.experimental.sample_from_datasets
datasets = {
s: merge_ds([unmerged_datasets[b][s] for b in basenames], weights)
for s in splits.keys()
}
# prepare datasets for batching
for split in datasets:
ds = datasets[split]
# batch and reshape sequences to match the input specification of tcmc
ds = database_reader.padded_batch(ds, batch_size, num_leaves,
alphabet_size)
ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
#ds = ds.map(database_reader.concatenate_dataset_entries, num_parallel_calls = 4)
# TODO: Pass the variable "num_classes" to database_reader.concatenate_dataset_entries().
if num_classes == 2:
ds = ds.map(database_reader.concatenate_dataset_entries,
num_parallel_calls=4)
elif num_classes == 3:
ds = ds.map(database_reader.concatenate_dataset_entries2,
num_parallel_calls=4)
else:
raise Exception(
f'Currently we only support two and three output classes. Your number of classes:{num_classes}'
)
datasets[split] = ds
if verbose:
print(f'Example batch of the "train" dataset:\n')
for t in datasets['train'].take(1):
(sequences, clade_ids, sequence_lengths), models = t
# extract the clade ids per batch
padding = [[1, 0]]
ind = tf.pad(tf.cumsum(sequence_lengths), padding)[:-1]
clade_ids = tf.gather(clade_ids, ind)
# extract the model ids
model_ids = tf.argmax(models, axis=1)
print(f'model_ids: {model_ids}')
print(f'clade_ids: {clade_ids}')
print(f'sequence_length: {sequence_lengths}')
print(f'sequence_onehot.shape: {sequences.shape}')
# to debug the sequence first transform it to its
# original shape
S = tf.transpose(sequences, perm=[1, 0, 2])
# decode the sequence and print some columns
if use_amino_acids:
alphabet = "ARNDCEQGHILKMFPSTWYV"
dec = ote.OnehotTupleEncoder.decode_tfrecord_entry(
S.numpy(),
alphabet=alphabet,
tuple_length=tuple_length,
use_bucket_alphabet=False)
else:
dec = ote.OnehotTupleEncoder.decode_tfrecord_entry(
S.numpy(), tuple_length=tuple_length)
print(
f'first (up to) 8 alignment columns of decoded reshaped sequence: \n{dec[:,:8]}'
)
# obtain the model creation functions for the wanted models
model_creaters = {}
model_training_callbacks = {}
for model_name in model_hyperparameters.keys():
try:
model_module = import_module(f"models.{model_name}",
package=__name__)
except ModuleNotFoundError as err:
raise Exception(
f'The module "models/{model_name}.py" for the model "{model_name}" does not exist.'
) from err
try:
model_creaters[model_name] = getattr(model_module, "create_model")
except AttributeError as err:
raise Exception(
f'The model "{model_name}" has no creation function "create_model" in "models/{model_name}.py".'
)
try:
model_training_callbacks[model_name] = getattr(
model_module, "training_callbacks")
except AttributeError as err:
raise Exception(
f'The model "{model_name}" has no training callbacks function "training_callbacks" in "models/{model_name}.py".'
)
#prepare the hyperparams for training
for model_name in model_hyperparameters:
model_hps = model_hyperparameters[model_name]
model_hps = {
h: hp.HParam(h, hp.Discrete(model_hps[h]))
for h in model_hps
}
model_hyperparameters[model_name] = model_hps
accuracy_metric = 'accuracy'
auroc_metric = tf.keras.metrics.AUC(num_thresholds=500,
dtype=tf.float32,
name='auroc')
for model_name in model_hyperparameters:
if verbose:
print(
'=================================================================================================='
)
print(f'Current model name: "{model_name}"\n')
# prepare model hyperparameters for iteration
hps = model_hyperparameters[model_name]
hp_names = list(hps.keys())
hp_values = [hps[k].domain.values for k in hp_names]
# log the wanted hyperparams and metrics
str_basenames = '_'.join(basenames)
logdir = f'{log_basedir}/{str_basenames}/{model_name}'
with tf.summary.create_file_writer(logdir).as_default():
hp.hparams_config(
hparams=list(hps.values()),
metrics=[
hp.Metric('accuracy',
group='test',
display_name='Accuracy'),
hp.Metric('auroc', group='test', display_name="AUROC"),
],
)
# iterate over all hyperparameter combinations for the current model
for hp_config in itertools.product(*hp_values):
# hp for tensorboard callback
hp_current = {hps[k]: hp_config[i] for i, k in enumerate(hp_names)}
# hp for model creation
creation_params = {k: hp_config[i] for i, k in enumerate(hp_names)}
# determine logging and saving paths
now_str = datetime.datetime.now().strftime('%Y.%m.%d--%H.%M.%S')
rundir = f'{logdir}/{now_str}'
save_weights_dir = f'{saved_weights_basedir}/{str_basenames}/{model_name}'
Path(save_weights_dir).mkdir(parents=True, exist_ok=True)
save_weights_path = f'{save_weights_dir}/{now_str}.h5'
if verbose:
print(f"\n\n")
print(f"Current set of hyperparameters: {creation_params}")
print(f"Training information will be stored in: {rundir}")
print(
f"Weights for the best model will be stored in: {save_weights_path}"
)
with tf.summary.create_file_writer(rundir).as_default():
hp.hparams(hp_current, trial_id=now_str)
create_model = model_creaters[model_name]
model = create_model(clades, alphabet_size, **creation_params)
if verbose:
print(
f'Architecture of the model "{model_name}" with the current hyperparameters:'
)
model.summary()
# compile the model for training
loss = tf.keras.losses.CategoricalCrossentropy()
optimizer = tf.keras.optimizers.Adam(0.0005)
model.compile(
optimizer=optimizer,
loss=loss,
metrics=[accuracy_metric, auroc_metric],
)
# define callbacks during training
checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(
filepath=save_weights_path,
monitor='val_loss',
mode='min',
save_best_only=True,
verbose=1,
)
# Function to decrease learning rate by 'factor'
learnrate_cb = tf.keras.callbacks.ReduceLROnPlateau(
monitor='val_loss',
mode='min',
factor=0.75,
patience=4,
verbose=1,
)
tensorboard_cb = tf.keras.callbacks.TensorBoard(rundir)
callbacks = [
tensorboard_cb,
]
if datasets['val'] != None:
callbacks = callbacks + [checkpoint_cb, learnrate_cb]
training_callbacks = model_training_callbacks[model_name]
callbacks = callbacks + training_callbacks(
model, rundir, wanted_callbacks=None)
model.fit(
datasets['train'],
validation_data=datasets['val'],
callbacks=callbacks,
epochs=epochs,
steps_per_epoch=batches_per_epoch,
verbose=verbose,
)
# load 'best' model weights and eval the test dataset
if datasets['test'] != None:
if verbose:
print("Evaluating the 'test' dataset:")
model.load_weights(save_weights_path)
test_loss, test_acc, test_auroc = model.evaluate(
datasets['test'])
with tf.summary.create_file_writer(
f'{rundir}/test').as_default():
tf.summary.scalar('accuracy', test_acc, step=1)
tf.summary.scalar('auroc', test_auroc, step=1)
tf.summary.scalar('loss', test_auroc, step=1)
return 0
def train(self):
"""Main training function."""
self.timing.start()
dimensions = self._create_dimensions()
hyperparameters = self._create_hyprparameters_domain()
with tf.summary.create_file_writer(
str(Path.cwd().joinpath("output", "logs",
"hparam_tuning"))).as_default():
hp.hparams_config(
hparams=hyperparameters,
metrics=[hp.Metric("accuracy", display_name="Accuracy")],
)
(
network_settings,
train_settings,
preprocess_settings,
) = parseConfigsFile(["network", "train", "preprocess"])
BATCH_SIZE = train_settings[
"batch_size"] * self.strategy.num_replicas_in_sync
(
synthetic_train,
synthetic_dataset_len,
synthetic_num_classes,
) = self._get_datasets(BATCH_SIZE)
validation_dataset = self._get_validation_dataset(BATCH_SIZE)
srfr_model = self._instantiate_models(synthetic_num_classes,
network_settings,
preprocess_settings,
train_settings)
train_model_sr_only_use_case = TrainModelFrOnlyUseCase(
self.strategy,
TimingLogger(),
self.logger,
BATCH_SIZE,
synthetic_dataset_len,
)
_training = partial(
self._fitness_function,
train_model_use_case=train_model_sr_only_use_case,
srfr_model=srfr_model,
batch_size=BATCH_SIZE,
synthetic_train=synthetic_train,
validation_dataset=validation_dataset,
num_classes=synthetic_num_classes,
train_settings=train_settings,
hparams=hyperparameters,
)
_train = use_named_args(dimensions=dimensions)(_training)
initial_parameters = [0.0002, 0.9, 10_000, 0.0005]
search_result = gp_minimize(
func=_train,
dimensions=dimensions,
acq_func="EI",
n_calls=20,
x0=initial_parameters,
)
self.logger.info(f"Best hyperparameters: {search_result.x}")
def hp_search(working_dir,
dataset='56_sunspots',
epochs=100,
metric='eval_mae_result',
stopping_patience=5,
stopping_delta=1):
working_dir = os.path.join("./checkpoints", working_dir)
# define domains for HP search
HP_EMB_DIM = hp.HParam('emb_dim', hp.Discrete([32, 64, 128]))
HP_LSTM_DIM = hp.HParam('lstm_dim', hp.Discrete([32, 64, 128]))
HP_DROPOUT = hp.HParam('lstm_dropout', hp.Discrete([0.1, 0.2, 0.3]))
HP_LR = hp.HParam('learning_rate', hp.Discrete([.0001, .001, .01]))
HP_BS = hp.HParam('batch_size', hp.Discrete([32, 64, 128]))
HP_WINDOW = hp.HParam('window_size', hp.Discrete([20, 40, 60]))
# set up config
with tf.summary.create_file_writer(working_dir).as_default():
hp.hparams_config(hparams=[
HP_EMB_DIM, HP_LSTM_DIM, HP_DROPOUT, HP_LR, HP_BS, HP_WINDOW
],
metrics=[hp.Metric(metric, display_name=metric)])
# read in training df
df = pd.read_csv(
f'../../datasets/seed_datasets_current/{dataset}/TRAIN/dataset_TRAIN/tables/learningData.csv'
)
df = _multi_index_prep(df, dataset)
df = _time_col_to_seconds(df, dataset)
# create TimeSeries and Learner objects
ds = _create_ts_object(df, dataset)
# grid search over parameters
run_num = 0
total_run_count = len(HP_EMB_DIM.domain.values) * \
len(HP_LSTM_DIM.domain.values) * \
len(HP_DROPOUT.domain.values) * \
len(HP_LR.domain.values) * \
len(HP_BS.domain.values) * \
len(HP_WINDOW.domain.values)
# outfile for saving hp config and runtimes
outfile = open(os.path.join(working_dir, "metrics.txt"), "w+", buffering=1)
for emb_dim in HP_EMB_DIM.domain.values:
for lstm_dim in HP_LSTM_DIM.domain.values:
for dropout in HP_DROPOUT.domain.values:
for lr in HP_LR.domain.values:
for bs in HP_BS.domain.values:
for window_size in HP_WINDOW.domain.values:
# create dict of parameters
hp_dict = {
'emb_dim': emb_dim,
'lstm_dim': lstm_dim,
'lstm_dropout': dropout,
'learning_rate': lr,
'batch_size': bs,
'window_size': window_size,
}
run_name = f'run-{run_num}'
logger.info(
f'--- Starting Run: {run_name} of {total_run_count} ---'
)
# print_dict = {
# h.name: hp_dict[h] for h in hp_dict
# }
logger.info(f'HP Dict: {hp_dict}')
hp.hparams(hp_dict)
# create learner and fit with these HPs
start_time = time.time()
learner = DeepARLearner(ds,
verbose=1,
hparams=hp_dict)
final_metric = learner.fit(
epochs=epochs,
stopping_patience=stopping_patience,
stopping_delta=stopping_delta,
checkpoint_dir=os.path.join(
working_dir, run_name))
outfile.write(
f'HPs: {hp_dict} ---- Metric: {final_metric} ---- Time: {round(time.time() - start_time,2)}\n'
)
tf.summary.scalar(metric, final_metric, step=1)
run_num += 1
outfile.close()
mse = run_training()
tf.summary.scalar('mse', mse, step=1)
if __name__ == '__main__':
HP_NUM_FC_BLOCKS = hp.HParam('num_fc_blocks', hp.Discrete([2, 5]))
HP_NUM_RES_BLOCKS = hp.HParam('num_res_blocks', hp.Discrete([2, 4]))
HP_ACTIVATION = hp.HParam('activation_type', hp.Discrete(['leaky', 'mish']))
HP_INIT_NUM_UNITS = hp.HParam('init_num_units', hp.Discrete([8, 16, 32]))
HP_FINAL_NUM_UNITS = hp.HParam('final_num_units', hp.Discrete([8, 16, 32]))
HP_USE_DOWNSAMPLE = hp.HParam('use_downsample', hp.Discrete([True, False]))
with tf.summary.create_file_writer('logs/hparam_tuning').as_default():
hp.hparams_config(
hparams=[HP_NUM_FC_BLOCKS,
HP_NUM_RES_BLOCKS,
HP_ACTIVATION,
HP_INIT_NUM_UNITS,
HP_FINAL_NUM_UNITS,
HP_USE_DOWNSAMPLE],
metrics=[hp.Metric(tf.keras.metrics.MeanSquaredError(), display_name='MSE_LOSS')],
)
#%%
HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([128, 256, 512]))
HP_DROPOUT = hp.HParam('dropout', hp.Discrete([0.5]))
HP_LEARNING_RATE = hp.HParam('learning_rate', hp.Discrete([0.0001, 0.00001]))
HP_CNN_FILTER_1 = hp.HParam('filter_1', hp.Discrete([16, 64, 256]))
HP_CNN_FILTER_2 = hp.HParam('filter_2', hp.Discrete([16, 64, 256]))
HP_BATCH_NORM = hp.HParam('batch_norm', hp.Discrete([False, True]))
HP_CNN_KERNEL_X_1 = hp.HParam('kernel_1_x', hp.Discrete([80, 30, 5]))
HP_CNN_KERNEL_Y_1 = hp.HParam('kernel_1_y', hp.Discrete([80, 30, 5]))
HP_CNN_KERNEL_X_2 = hp.HParam('kernel_2_x', hp.Discrete([80, 30, 5]))
HP_CNN_KERNEL_Y_2 = hp.HParam('kernel_2_y', hp.Discrete([80, 30, 5]))
with tf.summary.create_file_writer('logs/78-hparam-tuning-v3').as_default():
hp.hparams_config(
hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_LEARNING_RATE, HP_CNN_KERNEL_X_1, HP_CNN_KERNEL_Y_1, HP_CNN_KERNEL_X_2, HP_CNN_KERNEL_Y_2, HP_CNN_FILTER_1, HP_CNN_FILTER_2, HP_BATCH_NORM],
metrics=[hp.Metric('accuracy', display_name='Accuracy')],
)
#%%
def train_test_model(logdir, hparams):
classifier = tf.keras.Sequential()
classifier.add(tf.keras.layers.Conv2D(filters=hparams[HP_CNN_FILTER_1], kernel_size=(hparams[HP_CNN_KERNEL_X_1], hparams[HP_CNN_KERNEL_Y_1]), padding='same', activation='relu', input_shape=(x_train[0].shape[0], x_train[0].shape[1],1)))
if hparams[HP_BATCH_NORM]:
classifier.add(tf.keras.layers.BatchNormalization())
classifier.add(tf.keras.layers.MaxPooling2D(pool_size=2))
classifier.add(tf.keras.layers.Dropout(hparams[HP_DROPOUT]))
classifier.add(tf.keras.layers.Conv2D(filters=hparams[HP_CNN_FILTER_2], kernel_size=(hparams[HP_CNN_KERNEL_X_2], hparams[HP_CNN_KERNEL_Y_2]), padding='same', activation='relu'))
classifier.add(tf.keras.layers.MaxPooling2D(pool_size=2))
classifier.add(tf.keras.layers.Dropout(hparams[HP_DROPOUT]))
classifier.add(tf.keras.layers.Flatten())
my_inp=np.concatenate((inp_reno[:,None],inp_aoa[:,None],inp_para[:,:]),axis=1)
my_out=np.concatenate((out_cd[:,None],out_cl[:,None]),axis=1)
## Training sets
xtr0 = my_inp[I][:n]
ttr1 = my_out[I][:n]
#Hyperparameters
HP_NEURONS = hp.HParam('neurons', hp.Discrete(range(10,101,10)))
HP_LAYERS = hp.HParam('layer', hp.Discrete([1,2,3,4,5,6,7,8,9,10]))
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam']))
HP_L_RATE= hp.HParam('learning_rate', hp.Discrete([2.5e-4]))
METRIC_MSE = 'Mean Squared Error'
with tf.summary.create_file_writer('logs/1 hidden layer').as_default():
hp.hparams_config(
hparams=[HP_LAYERS, HP_NEURONS, HP_OPTIMIZER],
metrics=[hp.Metric(METRIC_MSE, display_name='val_loss')],
)
epochs = list(range(0,gg))
def train_test_model(hparams, layer,rundir):
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.InputLayer(input_shape=(5,)))
model.add(tf.keras.layers.Dense(hparams[HP_NEURONS], kernel_initializer='random_normal', activation=tf.keras.activations.relu))
if layer > 1:
for i in range(layer-1):
model.add(tf.keras.layers.Dense(hparams[HP_NEURONS], activation=tf.keras.activations.relu))
model.add(tf.keras.layers.Dense(2, activation=None))
model.summary()
optimizer_name = hparams[HP_OPTIMIZER]
learning_rate = hparams[HP_L_RATE]
import datetime
import tensorflow as tf
from tensorboard.plugins.hparams import api as hp
HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([8, 64]))
HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.3))
HP_LEARNING_RATE= hp.HParam('learning_rate', hp.Discrete([0.001, 0.0005, 0.0001]))
HP_OPTIMIZER=hp.HParam('optimizer', hp.Discrete(['adam', 'sgd', 'rmsprop']))
METRIC_ACCURACY = 'RootMeanSquaredError'
log_dir ='\\logs\\fit\\' + datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
with tf.summary.create_file_writer(log_dir).as_default():
hp.hparams_config(
hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER, HP_LEARNING_RATE],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Rmse')],
)
def train_and_evaluate(model, hparams):
tensorboard = TensorBoard(log_dir = log_dir, histogram_freq=10, write_graph=True, write_images=False)
hp_tuning = hp.KerasCallback(log_dir,hparams)
cbs = [tensorboard,hp_tuning]
history = model.fit(gen, validation_split=0.15, epochs=p.epochs, batch_size=p.batch_size, callbacks=cbs, verbose=1, shuffle = True)
mse, rmse = model.evaluate(test_gen)
return rmse, history
def run(run_dir, hparams):
with tf.summary.create_file_writer(run_dir).as_default():
hp.hparams(hparams)
model = create_model(hparams)
HP_UNITS_2 = hp.HParam('UNITS_2', hp.Discrete([0, 16, 64]))
HP_LEARNING_RATE = hp.HParam('learning_rate', hp.Discrete([0.0001]))
HP_CNN_FILTER_1 = hp.HParam('filter_1', hp.Discrete([8, 32, 64]))
HP_CNN_KERNEL_1 = hp.HParam('kernel_1', hp.Discrete([5]))
HP_CNN_FILTER_KERNEL_2 = hp.HParam(
'filter_kernel_2',
hp.Discrete([
'[0, 0]', '[8, 5]', '[8, 10]', '[32, 5]', '[32, 10]', '[64, 5]',
'[64, 10]'
]))
with tf.summary.create_file_writer('logs/81-small-v1').as_default():
hp.hparams_config(
hparams=[
HP_UNITS_1, HP_UNITS_2, HP_LEARNING_RATE, HP_CNN_KERNEL_1,
HP_CNN_FILTER_KERNEL_2, HP_CNN_FILTER_1
],
metrics=[hp.Metric('accuracy', display_name='Accuracy')],
)
#%%
def train_test_model(logdir, hparams):
start = timer()
classifier = tf.keras.Sequential()
classifier.add(
tf.keras.layers.Conv2D(filters=int(hparams[HP_CNN_FILTER_1]),
kernel_size=int(hparams[HP_CNN_KERNEL_1]),
activation='relu',
input_shape=(x_train[0].shape[0],
x_train[0].shape[1], 1)))
# 1. Number of units in the first dense layer
# 2. Dropout rate in the dropout layer
# 3. Optimizer
#
# List the values to try, and log an experiment configuration to TensorBoard. This step is optional: you can provide domain information to enable more precise filtering of hyperparameters in the UI, and you can specify which metrics should be displayed.
# %%
HP_NUM_UNITS = hp.HParam("num_units", hp.Discrete([16, 32]))
HP_DROPOUT = hp.HParam("dropout", hp.RealInterval(0.1, 0.2))
HP_OPTIMIZER = hp.HParam("optimizer", hp.Discrete(["adam", "sgd"]))
METRIC_ACCURACY = "accuracy"
with tf.summary.create_file_writer("logs/hparam_tuning").as_default():
hp.hparams_config(
hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER],
metrics=[hp.Metric(METRIC_ACCURACY, display_name="Accuracy")],
)
# %% [markdown]
# If you choose to skip this step, you can use a string literal wherever you would otherwise use an `HParam` value: e.g., `hparams['dropout']` instead of `hparams[HP_DROPOUT]`.
# %% [markdown]
# ## 2. Adapt TensorFlow runs to log hyperparameters and metrics
#
# The model will be quite simple: two dense layers with a dropout layer between them. The training code will look familiar, although the hyperparameters are no longer hardcoded. Instead, the hyperparameters are provided in an `hparams` dictionary and used throughout the training function:
# %%
def train_test_model(hparams):
""" """
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(),
# Defining constants
EPOCHS = 15
BATCH_SIZE = 64
# Defining the hyperparameters we would tune, and their values to be tested
HP_FILTER_SIZE = hp.HParam('filter_size', hp.Discrete([3, 5, 7]))
HP_FILTER_NUM = hp.HParam('filters_number', hp.Discrete([32, 64, 96, 128]))
METRIC_ACCURACY = 'accuracy'
# Logging setup info
with tf.summary.create_file_writer(
r'logs/Model_1_All/hparam_tuning/').as_default():
hp.hparams_config(
hparams=[HP_FILTER_SIZE, HP_FILTER_NUM],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
)
# Wrapping our model and training in a function
print(
'=============Wrapping our model and training in a function=================='
)
def train_test_model(hparams, session_num):
# Outlining the model/architecture of our CNN
### Removed the 2nd Con2D, and All the MaxPooling layers. ###
### Changed the Dense to 4. ###
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(hparams[HP_FILTER_NUM],
hparams[HP_FILTER_SIZE],
(X_train, y_train) = preprocessing(train_tweets)
X_test = process_test_data(test_tweets)
y_test = to_categorical(y_test)
HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([15]))
HP_BATCH_SIZE = hp.HParam('batch_size', hp.Discrete([32, 64]))
HP_ACT_FUNC = hp.HParam('activation_function', hp.Discrete(["softmax"]))
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(["rmsprop"]))
METRIC_ACCURACY = 'accuracy'
with tf.summary.create_file_writer('logs/hparam_tuning/' +
dateAndTimeNow).as_default():
hp.hparams_config(
hparams=[HP_NUM_UNITS, HP_BATCH_SIZE, HP_ACT_FUNC],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
)
def train_test_model2(hparams):
start_time = time.time()
input = Input(shape=(X_train.shape[1:]))
output = tf.expand_dims(input, axis=-1)
output = GRU(units=hparams[HP_NUM_UNITS],
activation="relu",
return_sequences=True)(output)
output = GRU(
units=hparams[HP_NUM_UNITS],
activation="relu",
)(output)
def main(argv):
# set up Grappler for graph optimization
# Ref: https://www.tensorflow.org/guide/graph_optimization
@contextlib.contextmanager
def options(options):
old_opts = tf.config.optimizer.get_experimental_options()
tf.config.optimizer.set_experimental_options(options)
try:
yield
finally:
tf.config.optimizer.set_experimental_options(old_opts)
# -----------------------------------------------------------------
# Creating dataloaders for training and validation
logging.info("Creating the dataloader from: %s..." % FLAGS.tfrecord_dir)
train_dataset = dataset_coco.prepare_dataloader(tfrecord_dir=FLAGS.tfrecord_dir,
img_size=256,
batch_size=FLAGS.batch_size,
subset='train')
valid_dataset = dataset_coco.prepare_dataloader(tfrecord_dir=FLAGS.tfrecord_dir,
img_size=256,
batch_size=1,
subset='val')
# -----------------------------------------------------------------
# Creating the instance of the model specified.
logging.info("Creating the model instance of YOLACT")
YOLACT = lite.MyYolact(input_size=256,
fpn_channels=96,
feature_map_size=[32, 16, 8, 4, 2],
num_class=13, # 12 classes + 1 background
num_mask=32,
aspect_ratio=[1, 0.5, 2],
scales=[24, 48, 96, 192, 384])
model = YOLACT.gen()
# add weight decay
# for layer in model.layers:
# if isinstance(layer, tf.keras.layers.Conv2D) or isinstance(layer, tf.keras.layers.Dense):
# layer.add_loss(lambda: tf.keras.regularizers.l2(FLAGS.weight_decay)(layer.kernel))
# if hasattr(layer, 'bias_regularizer') and layer.use_bias:
# layer.add_loss(lambda: tf.keras.regularizers.l2(FLAGS.weight_decay)(layer.bias))
# -----------------------------------------------------------------
# Choose the Optimizor, Loss Function, and Metrics, learning rate schedule
logging.info("Initiate the Optimizer and Loss function...")
lr_schedule = learning_rate_schedule.Yolact_LearningRateSchedule(warmup_steps=500, warmup_lr=1e-4, initial_lr=FLAGS.lr)
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete([ 'SGD' ]))
with tf.summary.create_file_writer('logs/hparam_tuning').as_default():
hp.hparams_config(
hparams=[HP_OPTIMIZER],
metrics=[hp.Metric('train_loss', display_name='Train Loss'), hp.Metric('valid_loss', display_name='Valid Loss')],
)
# -----------------------------------------------------------------
# Setup the TensorBoard for better visualization
# Ref: https://www.tensorflow.org/tensorboard/get_started
# -----------------------------------------------------------------
for trial_idx, optimizer_name in enumerate(HP_OPTIMIZER.domain.values):
logging.info("Setup the TensorBoard...")
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
train_log_dir = './logs/gradient_tape/' + current_time + f'-{optimizer_name}' + '/train'
test_log_dir = './logs/gradient_tape/' + current_time + f'-{optimizer_name}' + '/test'
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
test_summary_writer = tf.summary.create_file_writer(test_log_dir)
hparams = {
HP_OPTIMIZER: optimizer_name
}
optimizer_map = {
'SGD': tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=FLAGS.momentum),
'NesterovSGD': tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=FLAGS.momentum, nesterov=True),
'Adam': tf.keras.optimizers.Adam(learning_rate=0.001)
}
optimizer = optimizer_map[hparams[HP_OPTIMIZER]]
# Start the Training and Validation Process
logging.info("Start the training process...")
# setup checkpoints manager
checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, model=model)
manager = tf.train.CheckpointManager(checkpoint, directory=f'./checkpoints-{optimizer_name}', max_to_keep=5)
# restore from latest checkpoint and iteration
status = checkpoint.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
logging.info("Restored from {}".format(manager.latest_checkpoint))
else:
logging.info("Initializing from scratch.")
# Prepare Trainer
trainer = Trainer(model, optimizer)
best_val = 1e10
iterations = checkpoint.step.numpy()
for image, labels in train_dataset:
# check iteration and change the learning rate
if iterations > FLAGS.train_iter:
break
logging.info(f'Check iterations: {iterations}')
checkpoint.step.assign_add(1)
iterations += 1
trainer.valid_step(image, labels)
('train_rois_per_image', hp.choice('train_rois_per_image',
[50, 100, 200])),
('detection_min_confidence',
hp.choice('detection_min_confidence', [0.6, 0.7, 0.8])),
('optimizer', hp.choice('optimizer', ['ADAM', 'SGD']))
])
METRIC_MAP = 'mean average precision'
METRIC_F1S = 'f1 score'
with tf.summary.create_file_writer('logs/hparam_tuning_sund3').as_default():
tbhp.hparams_config(hparams=[
HP_BACKBONE, HP_DETECTION_MIN_CONFIDENCE, HP_TRAIN_ROIS_PER_IMAGE,
HP_OPTIMIZER
],
metrics=[
tbhp.Metric(METRIC_MAP,
display_name='mean average precision'),
tbhp.Metric(METRIC_F1S, display_name='f1 score')
])
def inference_calculation(config, model_dir, model_path, dataset_val):
model = modellib.MaskRCNN(mode="inference",
config=config,
model_dir=model_dir,
unique_log_name=False)
model.load_weights(model_path, by_name=True)
mean_average_precision, f1_score = calc_mean_average_precision(
dataset_val=dataset_val, inference_config=config, model=model)
HP_connected_weights: HP_connected_weights.domain.values,
HP_samples: HP_samples.domain.values,
})
with tf.summary.create_file_writer('logs/hparam_tuning').as_default():
hp.hparams_config(
hparams=[
HP_seed,
HP_components,
HP_encoder_dims,
HP_mixture_dims,
HP_bn,
HP_connected_weights,
HP_samples,
],
metrics=[
METRIC_AMI_TRAIN,
METRIC_AMI_TEST,
METRIC_PURITY_TRAIN,
METRIC_PURITY_TEST,
METRIC_max_cluster_attachment_test,
METRIC_loss,
METRIC_likelihood,
METRIC_z_prior_entropy,
METRIC_y_prior_entropy,
],
)
def train_test_model(run_id, hparams, X_train, y_train, X_test, y_test):
#hp.hparams(hparams) # record the values used in this trial
def main(self):
# DATA will contain a list of the paths to different binary data files. There should be one for each of the
# "different types of systems" (e.g. z3, z2, high temp, etc). There is no guarantee that there are the same
# number of configurations in each file though but the function below takes care of all of that for us to make
# sure we get a balanced dataset and that it meets some basic requirements.
# all_data = np.load(DATA)
all_data, data_labels = self.import_data(self.data)
n_records = len(all_data)
x_train = all_data[:n_records - int(n_records / 4)]
x_test = all_data[int(n_records / 4):]
x_train = np.reshape(x_train,
(len(x_train), self.L * 2, self.L * 2, 1))
x_test = np.reshape(x_test, (len(x_test), self.L * 2, self.L * 2, 1))
np.save(self.training_data_location, x_train)
np.save(self.testing_data_location, x_test)
np.save(self.data_label_location, data_labels)
with tf.summary.create_file_writer(
os.path.join(self.run_location, 'tensorboard_raw',
self.tensorboard_sub_dir)).as_default():
hp.hparams_config(hparams=[
self.hp_batch_size, self.hp_n_layers, self.hp_feature_map_step,
self.hp_stride_size, self.hp_use_batch_normalization,
self.hp_use_dropout
],
metrics=METRICS)
c = 0
autoencoder = None
for batch_size in self.hp_batch_size.domain.values:
for n_layers in self.hp_n_layers.domain.values:
for f_map_step in self.hp_feature_map_step.domain.values:
for stride in self.hp_stride_size.domain.values:
for use_batch_normalization in self.hp_use_batch_normalization.domain.values:
for use_dropout in self.hp_use_dropout.domain.values:
hyper_params = {
self.hp_batch_size: batch_size,
self.hp_n_layers: n_layers,
self.hp_feature_map_step: f_map_step,
self.hp_stride_size: stride,
self.hp_use_batch_normalization:
use_batch_normalization,
self.hp_use_dropout: use_dropout
}
c += 1
run_name = f"run-{c}"
print('--- Starting trial: %s' % run_name)
print({
h.name: hyper_params[h]
for h in hyper_params
})
self.run(
os.path.join(self.run_location,
'tensorboard_raw',
self.tensorboard_sub_dir,
run_name), hyper_params,
x_test, x_train)
# After each run lets attempt to log a sample of activations for the different layers
best_autoencoder = keras.models.load_model(self.checkpoint_file)
best_autoencoder.save(self.best_model_file)
# Get the encoder piece of the autoencoder. We call this the "activation model". This is the full model up to
# the bottle neck.
layers_to_encoded = int(len(best_autoencoder.layers) / 2)
print(layers_to_encoded)
layer_activations = [
layer.output
for layer in best_autoencoder.layers[:layers_to_encoded]
]
activation_model = models.Model(inputs=best_autoencoder.input,
outputs=layer_activations)
activation_model.save(self.best_activations_file)
def set_tensorboard(self,hparams):
with tf.summary.create_file_writer('logs/hparam_tunning').as_default():
hp.hparams_config(
hparams=hparams,
metrics=[hp.Metric('accuracy', display_name='Accuracy')]
)
HP_NUM_UNITS_DLl = hp.HParam('dl1_num_units', hp.Discrete([512]))
HP_DROPOUT_DL1 = hp.HParam('dl1_dropout', hp.Discrete([0.25]))
HP_NUM_UNITS_DL2 = hp.HParam('dl2_num_units', hp.Discrete([1024]))
HP_DROPOUT_DL2 = hp.HParam('dl2_dropout', hp.Discrete([0.5]))
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['rmsprop']))
METRIC_ACCURACY = 'accuracy'
with tf.summary.create_file_writer('logs/hparam_tuning_DL').as_default():
hp.hparams_config(
hparams=[
HP_NUM_UNITS_CLl, HP_KERNEL_SIZE_CLl, HP_STRIDES_CLl,
HP_NUM_UNITS_CL2, HP_KERNEL_SIZE_CL2, HP_STRIDES_CL2,
HP_POOL_SIZE_PL1, HP_DROPOUT_CL2, HP_NUM_UNITS_CL3,
HP_KERNEL_SIZE_CL3, HP_STRIDES_CL3, HP_NUM_UNITS_CL4,
HP_KERNEL_SIZE_CL4, HP_STRIDES_CL4, HP_POOL_SIZE_PL2,
HP_DROPOUT_CL4, HP_NUM_UNITS_DLl, HP_DROPOUT_DL1, HP_NUM_UNITS_DL2,
HP_DROPOUT_DL2, HP_OPTIMIZER
],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
)
session_num = 1
for a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u in product(
HP_NUM_UNITS_CLl.domain.values, HP_KERNEL_SIZE_CLl.domain.values,
HP_STRIDES_CLl.domain.values, HP_NUM_UNITS_CL2.domain.values,
HP_KERNEL_SIZE_CL2.domain.values, HP_STRIDES_CL2.domain.values,
HP_POOL_SIZE_PL1.domain.values, HP_DROPOUT_CL2.domain.values,
HP_NUM_UNITS_CL3.domain.values, HP_KERNEL_SIZE_CL3.domain.values,
HP_STRIDES_CL3.domain.values, HP_NUM_UNITS_CL4.domain.values,
HP_KERNEL_SIZE_CL4.domain.values, HP_STRIDES_CL4.domain.values,
