def model_build(image_shape=INPUT_SHAPE, classes=num_class):
input_img = Input(shape=INPUT_SHAPE, name="Image_Input")
x = Conv2D(64, (5,5), strides=(1,1),\
padding="valid", name="Conv_1_1")(input_img)
x = BatchNorm(name="BN_1_1")(x)
x = Activation("relu")(x)
x = MaxPooling2D(pool_size=(3, 3), padding="valid", name="Pool_1_1")(x)
x = Conv2D(128, (3, 3), strides=(1, 1), padding="valid",
name="Conv_2_1")(x)
x = BatchNorm(name="BN_2_1")(x)
x = Activation("relu")(x)
x = Conv2D(128, (3, 3), strides=(1, 1), padding="valid",
name="Conv_2_2")(x)
x = BatchNorm(name="BN_2_2")(x)
x = Activation("relu")(x)
x = Conv2D(128, (3, 3), strides=(1, 1), padding="valid",
name="Conv_2_3")(x)
x = BatchNorm(name="BN_2_3")(x)
x = Activation("relu")(x)
x = MaxPooling2D(pool_size=(2, 2), padding="valid", name="Pool_2_1")(x)
x = Conv2D(256, (3, 3), strides=(1, 1), padding="valid",
name="Conv_3_1")(x)
x = BatchNorm(name="BN_3_1")(x)
x = Activation("relu")(x)
x = Conv2D(256, (3, 3), strides=(1, 1), padding="valid",
name="Conv_3_2")(x)
x = BatchNorm(name="BN_3_2")(x)
x = Activation("relu")(x)
x = Conv2D(256, (3, 3), strides=(1, 1), padding="valid",
name="Conv_3_3")(x)
x = BatchNorm(name="BN_3_3")(x)
x = Activation("relu")(x)
x = MaxPooling2D(pool_size=(2, 2), name="Pool_3_1")(x)
x = Conv2D(512, (3, 3), strides=(1, 1), padding="valid",
name="Conv_4_1")(x)
x = BatchNorm(name="BN_4_1")(x)
x = Activation("relu")(x)
x = Conv2D(512, (3, 3), strides=(1, 1), padding="valid",
name="Conv_4_2")(x)
x = BatchNorm(name="BN_4_2")(x)
x = Activation("relu")(x)
x = Conv2D(512, (3, 3), strides=(1, 1), padding="valid",
name="Conv_4_3")(x)
x = BatchNorm(name="BN_4_3")(x)
x = Activation("relu")(x)
x = MaxPooling2D(pool_size=(2, 2), name="Pool_4_1")(x)
x = Flatten(name="Flatten")(x)
x = Dense(1024, activation="relu", name="FC_1")(x)
x = Dense(256, activation="relu", name="FC_2")(x)
x = Dense(classes, activation="softmax", name="FC_3")(x)
model = Model(inputs=input_img, outputs=x)
return model
def compute(self, config, budget, working_directory, epoch_cb, *args,
**kwargs):
"""
Simple example for a compute function using a feed forward network.
It is trained on the kin8nm dataset.
The input parameter "config" (dictionary) contains the sampled configurations passed by the bohb optimizer
"""
params = config_to_params(config)
n_iterations = budget
print("Total iterations:", n_iterations)
y_train = self.data['y_train']
y_valid = self.data['y_valid']
y_test = self.data['y_test']
if params['scaler'] != 'None':
scaler = eval("{}()".format(params['scaler']))
x_train_ = scaler.fit_transform(self.data['x_train'].astype(float))
x_valid_ = scaler.transform(self.data['x_valid'].astype(float))
x_test_ = scaler.transform(self.data['x_test'].astype(float))
else:
x_train_ = self.data['x_train']
x_valid_ = self.data['x_valid']
x_test_ = self.data['x_test']
input_dim = x_train_.shape[1]
model = Sequential()
model.add(
Dense(params['layer_1_size'],
init=params['init'],
activation=params['layer_1_activation'],
input_dim=input_dim))
for i in range(int(params['n_layers']) - 1):
extras = 'layer_{}_extras'.format(i + 1)
if params[extras]['name'] == 'dropout':
model.add(Dropout(params[extras]['rate']))
elif params[extras]['name'] == 'batchnorm':
model.add(BatchNorm())
model.add(
Dense(params['layer_{}_size'.format(i + 2)],
init=params['init'],
activation=params['layer_{}_activation'.format(i + 2)]))
model.add(Dense(1, init=params['init'], activation='linear'))
model.compile(optimizer=params['optimizer'], loss=params['loss'])
validation_data = (x_valid_, y_valid)
early_stopping = EarlyStopping(monitor='val_loss',
patience=5,
verbose=0)
h = model.fit(x_train_,
y_train,
epochs=int(round(n_iterations)),
batch_size=params['batch_size'],
shuffle=params['shuffle'],
validation_data=validation_data,
callbacks=[early_stopping, epoch_cb])
p = model.predict(x_test_, batch_size=params['batch_size'])
mse = MSE(y_test, p)
rmse = sqrt(mse)
mae = MAE(y_test, p)
print("# {} | RMSE: {:.4f}, MAE: {:.4f}".format(
self.run_id, rmse, mae))
loss = None
if self.loss_type == 'MAE':
loss = mae
elif self.loss_type == 'MSE':
loss = mse
else:
# set default loss
loss = rmse
self.loss_type = 'RMSE'
return ({
'cur_loss': loss,
'loss_type': self.loss_type,
'cur_iter': len(h.history['loss']),
'iter_unit': 'epoch',
'early_stop': model.stop_training,
'info': {
'params': params,
'rmse': rmse,
'mae': mae
}
})
def try_params(n_iterations, params):
print "iterations:", n_iterations
print_params(params)
y_train = data['y_train']
y_test = data['y_test']
if params['scaler']:
scaler = eval("{}()".format(params['scaler']))
x_train_ = scaler.fit_transform(data['x_train'].astype(float))
x_test_ = scaler.transform(data['x_test'].astype(float))
else:
x_train_ = data['x_train']
x_test_ = data['x_test']
input_dim = x_train_.shape[1]
model = Sequential()
model.add(
Dense(params['layer_1_size'],
init=params['init'],
activation=params['layer_1_activation'],
input_dim=input_dim))
for i in range(int(params['n_layers']) - 1):
extras = 'layer_{}_extras'.format(i + 1)
if params[extras]['name'] == 'dropout':
model.add(Dropout(params[extras]['rate']))
elif params[extras]['name'] == 'batchnorm':
model.add(BatchNorm())
model.add(
Dense(params['layer_{}_size'.format(i + 2)],
init=params['init'],
activation=params['layer_{}_activation'.format(i + 2)]))
model.add(Dense(1, init=params['init'], activation='linear'))
model.compile(optimizer=params['optimizer'], loss=params['loss'])
#print model.summary()
#
validation_data = (x_test_, y_test)
early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=0)
history = model.fit(x_train_,
y_train,
nb_epoch=int(round(n_iterations)),
batch_size=params['batch_size'],
shuffle=params['shuffle'],
validation_data=validation_data,
callbacks=[early_stopping])
#
p = model.predict(x_train_, batch_size=params['batch_size'])
nans_count = np.isnan(p).sum()
if nans_count > 0:
print "NULLS IN PREDICTIONS FOR TRAIN ({})".format(nans_count)
loss = np.iinfo(int).max
return {
'loss': loss,
'rmse': loss,
'mae': loss,
'early_stop': model.stop_training
}
mse = MSE(y_train, p)
rmse = sqrt(mse)
mae = MAE(y_train, p)
print "\n# training | RMSE: {:.4f}, MAE: {:.4f}".format(rmse, mae)
#
p = model.predict(x_test_, batch_size=params['batch_size'])
nans_count = np.isnan(p).sum()
if nans_count > 0:
print "NULLS IN PREDICTIONS FOR TEST ({})".format(nans_count)
loss = np.iinfo(int).max
return {
'loss': loss,
'rmse': loss,
'mae': loss,
'early_stop': model.stop_training
}
mse = MSE(y_test, p)
rmse = sqrt(mse)
mae = MAE(y_test, p)
print "# testing | RMSE: {:.4f}, MAE: {:.4f}".format(rmse, mae)
return {
'loss': rmse,
'rmse': rmse,
'mae': mae,
'early_stop': model.stop_training
}
def try_params(n_iterations, params):
print "iterations:", n_iterations
print_params(params)
y_train = data['y_train']
y_test = data['y_test']
if params['scaler']:
scaler = eval("{}()".format(params['scaler']))
x_train_ = scaler.fit_transform(data['x_train'].astype(float))
x_test_ = scaler.transform(data['x_test'].astype(float))
else:
x_train_ = data['x_train']
x_test_ = data['x_test']
input_dim = x_train_.shape[1]
model = Sequential()
model.add(
Dense(params['layer_1_size'],
init=params['init'],
activation=params['layer_1_activation'],
input_dim=input_dim))
for i in range(int(params['n_layers']) - 1):
extras = 'layer_{}_extras'.format(i + 1)
if params[extras]['name'] == 'dropout':
model.add(Dropout(params[extras]['rate']))
elif params[extras]['name'] == 'batchnorm':
model.add(BatchNorm())
model.add(
Dense(params['layer_{}_size'.format(i + 2)],
init=params['init'],
activation=params['layer_{}_activation'.format(i + 2)]))
model.add(Dense(1, init=params['init'], activation='sigmoid'))
model.compile(optimizer=params['optimizer'], loss='binary_crossentropy')
#print model.summary()
#
validation_data = (x_test_, y_test)
early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=0)
history = model.fit(x_train_,
y_train,
nb_epoch=int(round(n_iterations)),
batch_size=params['batch_size'],
shuffle=False,
validation_data=validation_data,
callbacks=[early_stopping])
#
p = model.predict_proba(x_train_, batch_size=params['batch_size'])
ll = log_loss(y_train, p)
auc = AUC(y_train, p)
acc = accuracy(y_train, np.round(p))
print "\n# training | log loss: {:.2%}, AUC: {:.2%}, accuracy: {:.2%}".format(
ll, auc, acc)
#
p = model.predict_proba(x_test_, batch_size=params['batch_size'])
ll = log_loss(y_test, p)
auc = AUC(y_test, p)
acc = accuracy(y_test, np.round(p))
print "# testing | log loss: {:.2%}, AUC: {:.2%}, accuracy: {:.2%}".format(
ll, auc, acc)
return {
'loss': ll,
'log_loss': ll,
'auc': auc,
'early_stop': model.stop_training
}
def try_params(n_iterations, params, data, return_model=False, early_stop=True):
n_iterations = int(n_iterations * iters_mult)
print("iterations:", n_iterations)
print_params(params)
y_train = data['y_train']
y_test = data['y_test']
if params['scaler']:
scaler_x = eval("{}()".format(params['scaler']))
x_train_ = scaler_x.fit_transform(data['x_train'].astype(float))
x_test_ = scaler_x.transform(data['x_test'].astype(float))
scaler_y = eval("{}()".format(params['scaler']))
y_train = scaler_y.fit_transform(data['y_train'].reshape(-1, 1).astype(float))
y_test = scaler_y.transform(data['y_test'].reshape(-1, 1).astype(float))
else:
x_train_ = data['x_train']
x_test_ = data['x_test']
input_dim = x_train_.shape[1]
k_reg, a_reg = _get_regularizations(params, 1)
model = Sequential()
model.add(Dense(params['layer_1_size'], kernel_initializer=params['init'],
activation=params['layer_1_activation'], input_dim=input_dim,
kernel_regularizer=k_reg, activity_regularizer=a_reg))
last = 1
for i in range(int(params['n_layers']) - 1):
extras = 'layer_{}_extras'.format(i + 1)
if params[extras]['name'] == 'dropout':
model.add(Dropout(params[extras]['rate']))
elif params[extras]['name'] == 'batchnorm':
model.add(BatchNorm())
k_reg, a_reg = _get_regularizations(params, i + 2)
model.add(Dense(params['layer_{}_size'.format(i + 2)], kernel_initializer=params['init'],
activation=params['layer_{}_activation'.format(i + 2)],
kernel_regularizer=k_reg, activity_regularizer=a_reg))
last = i + 2
extras = 'layer_{}_extras'.format(last)
if params[extras]['name'] == 'dropout':
model.add(Dropout(params[extras]['rate']))
elif params[extras]['name'] == 'batchnorm':
model.add(BatchNorm())
model.add(Dense(1, kernel_initializer=params['init'], activation='linear'))
model.compile(optimizer=params['optimizer'], loss=params['loss'])
#print(model.summary())
#
validation_data = (x_test_, y_test)
if early_stop:
early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0)
else: # Never stop...
early_stopping = EarlyStopping(monitor='train_loss', patience=10000, verbose=0)
history = model.fit(x_train_, y_train,
epochs=int(round(n_iterations)),
batch_size=params['batch_size'],
shuffle=params['shuffle'],
validation_data=validation_data,
callbacks=[early_stopping])
#
p = model.predict(x_train_, batch_size=params['batch_size'])
p = np.nan_to_num(p)
if params['scaler']:
p = scaler_y.inverse_transform(p)
y_train = scaler_y.inverse_transform(y_train)
mse = MSE(y_train, p)
rmse = sqrt(mse)
mae = MAE(y_train, p)
print("\n# training | RMSE: {:.4f}, MAE: {:.4f}".format(rmse, mae))
#
p = model.predict(x_test_, batch_size=params['batch_size'])
p = np.nan_to_num(p)
if params['scaler']:
p = scaler_y.inverse_transform(p)
y_test = scaler_y.inverse_transform(y_test)
mse = MSE(y_test, p)
rmse = sqrt(mse)
mae = MAE(y_test, p)
print("# testing | RMSE: {:.4f}, MAE: {:.4f}".format(rmse, mae))
if return_model:
return model
else:
return {'loss': rmse, 'rmse': rmse, 'mae': mae, 'early_stop': model.stop_training}