def create_model(train_gen, val_gen):
"""
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
"""
base_model = Xception(include_top=False, weights='imagenet')
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024,
activation='relu',
kernel_regularizer=regularizers.l2({{uniform(0, 0.1)}}))(x)
x = Dropout({{uniform(0, 1)}})(x)
predictions = Dense(2, activation='softmax')(x)
model = Model(input=base_model.input, output=predictions)
model.compile(loss='categorical_crossentropy',
metrics=['accuracy'],
optimizer='adam')
model.fit_generator(train_gen,
steps_per_epoch=1,
epochs=1,
verbose=2,
validation_data=val_gen)
score, acc = model.evaluate_generator(val_gen, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(X_train, Y_train, X_valid, Y_valid, X_test, Y_test):
# hyperparams of the model
n_layer1 = {{choice([128, 256, 512])}}
n_layer2 = {{choice([128, 256, 512])}}
dropout_1 = {{uniform(0, 0.5)}}
dropout_2 = {{uniform(0, 0.5)}}
optim = {{choice(['rmsprop', 'adam'])}}
n_batch = {{choice([64, 128, 256])}}
# an LSTM model that can learn character sequences
model = Sequential()
model.add(
LSTM(n_layer1,
input_shape=(X_train.shape[1], X_train.shape[2]),
return_sequences=True))
model.add(Dropout(dropout_1))
model.add(LSTM(n_layer2))
model.add(Dropout(dropout_2))
model.add(Dense(Y_train.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=optim)
# training the model
result = model.fit(X_train,
Y_train,
batch_size=n_batch,
epochs=10,
verbose=2,
validation_data=(X_valid, Y_valid),
shuffle=True)
validation_loss = np.amax(result.history['val_loss'])
return {'loss': validation_loss, 'status': STATUS_OK, 'model': model}
def create_model(x_train, y_train, x_test, y_test):
model = Sequential()
model.add(Dense(512, input_shape=(16384, )))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([16, 32, 64, 128, 256, 512])}}))
model.add(Activation({{choice(['relu', 'sigmoid'])}}))
model.add(Dropout({{uniform(0, 1)}}))
# If we choose 'four', add an additional fourth layer
if {{choice(['three', 'four'])}} == 'four':
model.add(Dense(100))
# We can also choose between complete sets of layers
model.add({{choice([Dropout(0.5), Activation('linear')])}})
model.add(Activation('relu'))
model.add(Dense(2))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
metrics=['accuracy'],
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})
result = model.fit(x_train,
y_train,
batch_size={{choice([30, 48, 64, 128])}},
epochs=8,
verbose=2,
validation_split=0.3)
#get the highest validation accuracy of the training epochs
validation_acc = np.amax(result.history['val_acc'])
print('Best validation acc of epoch:', validation_acc)
return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
def keras_model(X_train, X_test, y_train, y_test):
NUM_EPOCHS = 125
BATCH_SIZE = 128
inputs = Input(shape=(304, ))
x = Dropout({{uniform(0.1, 0.5)}})(inputs)
x = Dense({{choice([64, 128, 256])}})(x)
x = Activation("relu")(x)
x = Dropout({{uniform(0.1, 0.5)}})(x)
x = Dense({{choice([64, 128, 256])}})(x)
x = Activation("relu")(x)
x = Dropout({{uniform(0.1, 0.5)}})(x)
x = Dense({{choice([64, 128, 256])}})(x)
x = Activation("relu")(x)
x = Dropout({{uniform(0.1, 0.5)}})(x)
predictions = Dense(1)(x)
model = Model(inputs=[inputs], outputs=[predictions])
model.compile(loss="mse", optimizer={{choice(["adam", "RMSprop"])}})
model.fit(X_train, y_train,
batch_size=BATCH_SIZE, nb_epoch=NUM_EPOCHS,
verbose=2,
validation_data=(X_test, y_test))
score = model.evaluate(X_test, y_test, verbose=0)
return {'loss': -score, 'status': STATUS_OK, 'model': model}
def model(X_train, Y_train, X_test, Y_test):
model = Sequential()
model.add(Dense({{choice([256, 512, 1024])}}, input_shape=(7, )))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation({{choice(['relu', 'sigmoid'])}}))
model.add(Dropout({{uniform(0, 1)}}))
# If we choose 'four', add an additional fourth layer
if {{choice(['three', 'four'])}} == 'four':
model.add(Dense({{choice([256, 512, 1024])}}))
model.add({{choice([Dropout(0.5), Activation('linear')])}})
model.add(Activation('relu'))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}},
metrics=['accuracy'])
model.fit(X_train,
Y_train,
batch_size={{choice([5, 10, 15])}},
nb_epoch=20,
verbose=2,
validation_split=0.3)
score, acc = model.evaluate(X_test, Y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def keras_model():
from keras.regularizers import l2, activity_l2
from aiding_funcs.embeddings_handling import get_the_folds, join_folds
from keras.layers.recurrent import LSTM
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.layers.embeddings import Embedding
from keras.regularizers import l1, activity_l1
import pickle
embeddings = pickle.load( open( "/data/dpappas/personality/emb.p", "rb" ) )
train = pickle.load( open( "/data/dpappas/personality/train.p", "rb" ) )
no_of_folds = 10
folds = get_the_folds(train,no_of_folds)
train_data = join_folds(folds,folds.keys()[:-1])
validation_data = folds[folds.keys()[-1]]
max_input_length = validation_data['features'].shape[1]
LSTM_size = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
Dense_size = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
opt = {{choice([ 'adadelta','sgd','rmsprop', 'adagrad', 'adadelta', 'adam'])}}
is_trainable = {{choice([ True, False ])}}
D = embeddings.shape[-1]
out_dim = 5
model = Sequential()
model.add(Embedding(input_dim = embeddings.shape[0], output_dim=D, weights=[embeddings], trainable=is_trainable, input_length = max_input_length))
model.add(LSTM(LSTM_size, activation = 'sigmoid'))
model.add(Dense(Dense_size, activation = 'sigmoid',W_regularizer=l2({{uniform(0, 1)}}), activity_regularizer=activity_l2({{uniform(0, 1)}})))
model.add(Dense(out_dim, activation = 'linear',W_regularizer=l2({{uniform(0, 1)}}), activity_regularizer=activity_l2({{uniform(0, 1)}})))
model.compile(loss='mse', optimizer= opt) # kalutera leei rmsprop o fchollet enw adam leei enas allos
model.fit(train_data['features'], train_data['labels'], nb_epoch=50, show_accuracy=False, verbose=2)
score = model.evaluate( validation_data['features'], validation_data['labels'])
#score = model.evaluate( train_data['features'], train_data['labels'])
return {'loss': score, 'status': STATUS_OK}
def tuned_model(x_t, y_t, x_v, y_v):
model = Sequential()
model.add(
Dense(72, input_dim=72, kernel_initializer='normal',
activation='relu'))
model.add(Dropout({{uniform(0.15, 0.3)}}))
for i in range(1, {{choice([8, 10, 12])}}):
model.add(
Dense({{choice([36, 72])}},
kernel_initializer='normal',
activation='relu'))
model.add(Dropout({{uniform(0.15, 0.3)}}))
model.add(Dense(1, activation='relu'))
model.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer='adam')
model.fit(x_t,
y_t,
batch_size={{choice([8, 10, 15])}},
epochs=100,
verbose=2)
score, acc = model.evaluate(x_v, y_v, verbose=0)
print("Test accuracy: ", acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, X_test, Y_train, Y_test):
model = Sequential()
model.add(Dense(512, input_shape=(1, 64, 64, 64)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([400, 512, 600])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Flatten())
model.add(Dense(2))
model.add(Activation('softmax'))
#rms = RMSprop()
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
nb_epoch = 1
batch_size = 50
model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=nb_epoch,
verbose=2,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(trainX, trainY, valX, valY, vocab_size, GRU_config, report_folder_GRU, window_size):
logger = logging.getLogger(__name__)
contextEmbedding = Embedding(input_dim=vocab_size, output_dim={{choice([64, 128, 256])}},
input_length=window_size)
tensor = Input(shape=(window_size,))
c = contextEmbedding(tensor)
c = Dropout({{uniform(0, 0.5)}})(c)
c = GRU({{choice([50, 100, 200])}}, recurrent_dropout={{uniform(0, 0.5)}})(c)
c = Dropout({{uniform(0, 0.5)}})(c)
c = Dense({{choice([70, 100, 200])}}, activation={{choice(['relu', 'elu', 'selu'])}})(c)
c = Dropout({{uniform(0, 0.5)}})(c)
answer = Dense(vocab_size, activation='softmax')(c)
model = Model(tensor, answer)
optimizer = Adam(lr={{choice([0.001, 3e-4])}})
model.compile(optimizer=optimizer, loss="sparse_categorical_crossentropy", metrics=['acc'])
early_stopping = EarlyStopping(monitor='val_loss',
mode= 'min',
patience=5)
model.fit(trainX, trainY,
batch_size={{choice([64, 128])}},
epochs={{choice([10, 15, 20, 30])}},
verbose=2,
validation_data=(valX, valY),
callbacks=[early_stopping]
)
score, acc = model.evaluate(valX, valY, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(x_train, y_train, x_valid, y_valid):
model = Sequential()
model.add(Dense(512, input_shape=(69, ), activation='relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(
Dense({{choice([256, 512, 1024])}},
activation={{choice(['relu', 'sigmoid'])}}))
model.add(Dropout({{uniform(0, 1)}}))
if {{choice(['three', 'four'])}} == 'four':
model.add(
Dense({{choice([128, 256])}},
activation={{choice(['relu', 'sigmoid'])}}))
model.add(Dense(1))
model.compile(optimizer={{choice(['rmsprop', 'adam', 'sgd'])}}, loss='mse')
history = model.fit(x_train,
y_train,
batch_size={{choice([64, 128])}},
epochs={{choice([10, 15, 20])}},
validation_split=.1)
validation_loss = np.amax(history.history['val_loss'])
print('Best validation loss of epochs:', validation_loss)
return {'loss': -validation_loss, 'status': STATUS_OK, 'model': model}
def model(x_train, y_train, x_val, y_val, embedding_dim, maxlen, max_words):
model = Sequential()
model.add(Embedding(max_words, embedding_dim, input_length=maxlen))
model.add(Flatten())
model.add(Dense({{choice([8, 16, 32, 64, 128])}}, activation='relu'))
#model.add(layers.BatchNormalization())
model.add(layers.Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([4, 8, 16, 32, 64])}}, activation='relu'))
#model.add(layers.BatchNormalization())
model.add(layers.Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([2, 4, 8, 16, 32])}}, activation='relu'))
#model.add(layers.BatchNormalization())
#model.add(layers.Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
#model.summary()
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
model.fit(x_train, y_train,
epochs=50,
batch_size=32,
verbose=2,
validation_data=(x_val, y_val))
#callbacks=callbacks_list,
score, acc = model.evaluate(x_val, y_val, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, Y_train, X_test, Y_test):
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import RMSprop
model = Sequential()
model.add(Dense(512, input_shape=(784, )))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy', optimizer=rms)
model.fit(X_train,
Y_train,
batch_size={{choice([64, 128])}},
nb_epoch=1,
show_accuracy=True,
verbose=2,
validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
print('Test accuracy:', score[1])
return {'loss': -score[1], 'status': STATUS_OK}
def model(X_train, Y_train, X_test, Y_test):
'''
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
'''
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy', optimizer=rms, metrics=['accuracy'])
model.fit(X_train, Y_train,
batch_size={{choice([64, 128])}},
nb_epoch=1,
verbose=2,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, Y_train, X_test, Y_test):
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import RMSprop
model = Sequential()
model.add(Dense(512, input_shape=(784, )))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy',
optimizer=rms,
metrics=['acc'])
model.fit(X_train,
Y_train,
batch_size={{choice([64, 128])}},
epochs=1,
verbose=2,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
print('Test accuracy:', acc)
return {
'loss': -acc,
'status': STATUS_OK,
'model': model.to_yaml(),
'weights': pickle.dumps(model.get_weights())
}
def keras_model():
from keras.models import Sequential
from keras.layers.core import Dense, Reshape, Activation, Flatten, Dropout
from keras.regularizers import l1, activity_l1, l2, activity_l2
from aiding_funcs.embeddings_handling import get_the_folds, join_folds
from aiding_funcs.label_handling import MaxMin, myRMSE, MaxMinFit
import pickle
train = pickle.load( open( "/data/dpappas/personality/train.p", "rb" ) )
no_of_folds = 10
folds = get_the_folds(train,no_of_folds)
train_data = join_folds(folds,folds.keys()[:-1])
validation_data = folds[folds.keys()[-1]]
mins, maxs = MaxMin(train_data['AV'])
T_AV = MaxMinFit(train_data['AV'], mins, maxs)
Dense_size = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
Dense_size2 = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
Dense_size3 = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
opt = {{choice([ 'adadelta','sgd','rmsprop', 'adagrad', 'adadelta', 'adam'])}}
out_dim = 5
model = Sequential()
model.add(Dense(Dense_size, activation='sigmoid',W_regularizer=l2({{uniform(0, 1)}}),activity_regularizer=activity_l2({{uniform(0, 1)}}),input_dim = train_data['AV'].shape[-1] ))
model.add(Dense(Dense_size2, activation='sigmoid',W_regularizer=l2({{uniform(0, 1)}}),activity_regularizer=activity_l2({{uniform(0, 1)}})))
model.add(Dense(Dense_size3, activation='sigmoid',W_regularizer=l2({{uniform(0, 1)}}),activity_regularizer=activity_l2({{uniform(0, 1)}})))
model.add(Dense(out_dim, activation='linear',W_regularizer=l2({{uniform(0, 1)}}),activity_regularizer=activity_l2({{uniform(0, 1)}})))
model.compile(loss='rmse', optimizer=opt)
model.fit(T_AV, train_data['labels'], nb_epoch=500, show_accuracy=False, verbose=2)
#score = model.evaluate( validation_data['features'], validation_data['labels'])
score = model.evaluate( T_AV, train_data['labels'])
print("score : " +str(score))
return {'loss': score, 'status': STATUS_OK}
def model(x_train, y_train, x_test, y_test):
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([128, 256, 512, 1024])}}))
model.add(Activation({{choice(['relu', 'sigmoid', 'tanh'])}}))
model.add(Dropout({{uniform(0, 1)}}))
# If we choose 'fomv ur', add an additional fourth layer
if conditional({{choice(['three', 'four'])}}) == 'four':
model.add(Dense(100))
# We can also choose between complete sets of layers
model.add({{choice([Dropout(0.5), Activation('linear')])}})
model.add(Activation('relu'))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', metrics=['accuracy'],
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})
model.fit(x_train, y_train,
batch_size={{choice([128, 256])}},
epochs=5,
verbose=2,
validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(x_train, y_train, x_val, y_val, x_test, y_test):
ACTIONS_FILE = "sc.txt"
# read action names
ACTIONS = []
for l in codecs.open(ACTIONS_FILE, "r").readlines():
ACTIONS.append(l.split("\t")[1].replace("\n", "").replace("\r", ""))
model = Sequential()
# have several parameters to choose with meta-parametrization optimization
model.add(
LSTM({{choice([128, 256, 512])}}, input_shape=(None, len(ACTIONS))))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(len(ACTIONS)))
model.add(Activation('softmax'))
optimizer = Adam(lr={{uniform(0.001, 0.01)}})
model.compile(loss='categorical_crossentropy',
metrics=['accuracy'],
optimizer=optimizer)
result = model.fit(x_train,
y_train,
batch_size={{choice([16, 32, 64])}},
epochs=10,
verbose=2,
shuffle=True,
validation_data=(x_val, y_val))
# get the highest validation accuracy of the training epochs
validation_acc = np.amax(result.history['val_acc'])
print('Best validation acc of epoch:', validation_acc)
return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
def create_model(train_X, train_y, val_X, val_y):
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)))
# design network
model = Sequential()
model.add(
LSTM(units=128,
return_sequences=True,
input_shape=(train_X.shape[1], train_X.shape[2]),
bias_regularizer=L1L2(l1=0.1, l2=0.05)))
model.add(Dropout({{uniform(0, 1)}}))
model.add(LSTM(units=64))
model.add(Dropout({{uniform(0, 1)}}))
# model.add(Dense(16,init='uniform',activation='relu'))
model.add(Dense({{choice([64, 32, 16, 8])}}))
model.add(Dense(units=1))
model.compile(optimizer={{choice(['adam'])}}, loss=root_mean_squared_error)
# fit network
history = model.fit(train_X,
train_y,
epochs=500,
batch_size={{choice([25])}},
verbose=2,
shuffle=False,
validation_split=0.1)
#get the highest validation accuracy of the training epochs
validation_acc = np.amin(history.history['val_loss'])
print('Best validation loss of epoch:', validation_acc)
return {'loss': validation_acc, 'status': STATUS_OK, 'model': model}
def cnn_model():
model = Sequential()
model.add(
Conv2D(32, (3, 3),
padding='same',
input_shape=(3, 48, 48),
activation='relu'))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Conv2D(128, (3, 3), padding='same', activation='relu'))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(43, activation='softmax'))
return model
def create_model(X_trn, y_trn, X_val, y_val):
model = Sequential()
model.add(
Dense({{choice([np.power(2, 5),
np.power(2, 6),
np.power(2, 7)])}},
input_dim=X_trn.shape[1]))
model.add(LeakyReLU(alpha={{uniform(0, 0.5)}}))
model.add(Dropout({{uniform(0.5, 1)}}))
model.add(
Dense({{choice([np.power(2, 5),
np.power(2, 6),
np.power(2, 7)])}},
input_dim=X_trn.shape[1]))
model.add(LeakyReLU(alpha={{uniform(0, 0.5)}}))
model.add(Dropout({{uniform(0.5, 1)}}))
model.add(Dense(1, activation='sigmoid'))
reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_acc',
factor=0.2,
patience=5,
min_lr=0.0001)
model.compile(optimizer={{choice(['rmsprop', 'adam', 'sgd'])}},
loss='binary_crossentropy',
metrics=['accuracy'])
model.fit(X_trn,
y_trn,
epochs={{choice([25, 50, 75, 100])}},
batch_size={{choice([16, 32, 64])}},
validation_data=(X_val, y_val),
verbose=1,
callbacks=[reduce_lr])
score, acc = model.evaluate(X_val, y_val, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model_creation(x_train, y_train, x_test, y_test):
model = Sequential()
model.add(
Embedding(input_dim=210338,
output_dim={{choice([100, 150, 300, 500])}},
embeddings_initializer='uniform',
input_length=100))
model.add(Dropout({{uniform(0, 1)}}))
model.add(LSTM({{choice([150, 256, 512, 1024])}}, return_sequences=True))
model.add(Dropout({{uniform(0, 1)}}))
model.add(LSTM({{choice([150, 256, 512, 1024])}}, return_sequences=False))
model.add(Dropout({{uniform(0, 1)}}))
#if conditional({{choice(['three', 'four'])}}) == 'three':
# model.add(LSTM(1024, return_sequences=False))
# model.add(Dropout(0.32))
model.add(Dense(6, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})
print(model.summary())
model.fit(x_train,
y_train,
epochs=1,
batch_size={{choice([32, 64, 128, 256, 512])}})
score, acc = model.evaluate(x_test, y_test, verbose=0)
print('Test accuracy: ', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, Y_train, X_test, Y_test):
model = models.Sequential()
f = {{choice([[16, 16, 32, 32, 32], [32, 32, 64, 64, 64], [16, 32, 32, 64, 64]])}}
model.add(layers.Conv2D(f[0], {{choice([(3, 3), (5, 5)])}}, padding='same', activation='relu', input_shape=(28, 28, 1)))
model.add(layers.Conv2D(f[1], {{choice([(3, 3), (5, 5)])}}, padding='same', activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Dropout({{uniform(0, 1)}}, seed=7382))
model.add(layers.Conv2D(f[2], {{choice([(3, 3), (5, 5)])}}, padding='same', activation='relu'))
model.add(layers.Conv2D(f[3], {{choice([(3, 3), (5, 5)])}}, padding='same', activation='relu'))
model.add(layers.Conv2D(f[4], {{choice([(3, 3), (5, 5)])}}, padding='same', activation='relu'))
model.add(layers.BatchNormalization())
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Dropout({{uniform(0, 1)}}, seed=7382))
model.add(layers.Flatten())
model.add(layers.Dense({{choice([64, 84, 128])}}, activation='relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout({{uniform(0, 1)}}))
model.add(layers.Dense(10, activation='softmax'))
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
model.fit(X_train, Y_train,
batch_size={{choice([None, 64, 128])}},
epochs=10,
verbose=2,
validation_data=(X_test, Y_test),
callbacks=[WandbCallback()])
score, acc = model.evaluate(X_test, Y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, Y_train, X_test, Y_test):
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import RMSprop
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy', optimizer=rms)
model.fit(X_train, Y_train,
batch_size={{choice([64, 128])}},
nb_epoch=1,
show_accuracy=True,
verbose=2,
validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test,
show_accuracy=True, verbose=0)
print('Test accuracy:', score[1])
return {'loss': -score[1], 'status': STATUS_OK}
def create_model(X_train, y_train, X_val, y_val):
model = Sequential()
embedding_vector_length = 500
model.add(
layers.Embedding(vocab_size,
embedding_vector_length,
input_length=max_sequence_length))
model.add(
layers.LSTM(
units={{choice([np.power(2, 5),
np.power(2, 6),
np.power(2, 7)])}},
dropout={{uniform(0.5, 1)}},
recurrent_dropout={{uniform(0.5, 1)}}))
model.add(
Dense(units=10, activation="softmax", kernel_initializer="uniform"))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=5,
min_lr=0.001)
model.fit(X_train,
y_train,
epochs={{choice([25, 50, 75, 100])}},
batch_size={{choice([16, 32, 64])}},
validation_data=(X_val, y_val),
callbacks=[reduce_lr])
score, acc = model.evaluate(X_val, y_val, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(train_X, test_X, train_y, test_y):
model = Sequential()
model.add(Dense(500, input_shape=(60,),kernel_initializer= {{choice(['glorot_uniform','random_uniform'])}}))
model.add(BatchNormalization(epsilon=1e-06, mode=0, momentum=0.9, weights=None))
model.add(Activation({{choice(['relu','sigmoid','tanh'])}}))
model.add(Dropout({{uniform(0, 0.3)}}))
model.add(Dense({{choice([128,256])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 0.4)}}))
model.add(Dense({{choice([128,256])}}))
model.add(Activation({{choice(['relu','tanh'])}}))
model.add(Dropout(0.3))
model.add(Dense(41))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer={{choice(['rmsprop', 'adam'])}})
model.summary()
early_stops = EarlyStopping(patience=3, monitor='val_acc')
ckpt_callback = ModelCheckpoint('keras_model',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto')
model.fit(train_X, train_y, batch_size={{choice([128,264])}}, nb_epoch={{choice([10,20])}}, validation_data=(test_X, test_y), callbacks=[early_stops,ckpt_callback])
score, acc = model.evaluate(test_X, test_y, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(x_train, y_train, x_test, y_test):
model = Sequential()
model.add(
LSTM(units={{choice([16, 32, 64, 128, 256])}},
input_shape=(x_train.shape[1], x_train.shape[2])))
model.add(Dropout({{uniform(0, 1)}}))
if conditional({{choice(['three', 'four'])}}) == 'four':
model.add(Dense(units={{choice([2, 4, 8, 16, 32, 64, 128, 256])}}))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(1))
model.compile(
loss='mae', # because mse gives me NaN's some times
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})
model.fit(x_train,
y_train,
batch_size={{choice([8, 16, 32, 64, 128])}},
epochs=50,
verbose=0,
shuffle=False,
validation_data=(x_test, y_test))
mae = model.evaluate(x_test, y_test, verbose=0)
print('MAE:', mae)
return {'loss': mae, 'status': STATUS_OK, 'model': model}
def model(X_train, X_test, Y_train, Y_test):
model = Sequential()
model.add(Dense(512, input_shape=(784, )))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([400, 512, 600])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy',
optimizer=rms,
metrics=['accuracy'])
nb_epoch = 10
batch_size = 128
model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
verbose=2,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, Y_train, X_test, Y_test):
model = Sequential()
model.add(Dense(50, input_shape=(784, )))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([20, 30, 40])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy',
optimizer=rms,
metrics=['accuracy'])
model.fit(X_train,
Y_train,
batch_size={{choice([64, 128])}},
epochs=1,
verbose=2,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(x_train, y_train, x_test, y_test):
# 25 > 10 (relu) dropout > 10 (relu) dropout > 1 (relu)
model = Sequential()
model.add(
Dense(output_dim={{
choice([5, 10, 15, 20, 25, 30, 50, 75, 100, 200, 500])
}},
input_dim=x_train.shape[1]))
model.add(Activation({{choice(['relu', 'sigmoid', 'tanh', 'linear'])}}))
model.add(Dropout({{uniform(0, 1)}}))
# If we choose 'four', add an additional fourth layer
if conditional({{choice(['extra-layer', 'no'])}}) == 'extra-layer':
model.add(
Dense(output_dim={{
choice([5, 10, 15, 20, 25, 30, 50, 75, 100, 200, 500])
}}))
model.add(Activation({{choice(['relu', 'sigmoid', 'tanh',
'linear'])}}))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(output_dim=1))
model.add(Activation({{choice(['relu', 'sigmoid', 'tanh', 'linear'])}}))
model.compile(loss='mean_squared_error',
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}},
metrics=['accuracy'])
model.fit(x_train,
y_train,
batch_size={{choice([32, 64, 128])}},
nb_epoch=10,
validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(train_X, test_X, train_y, test_y):
model = Sequential()
model.add(Dense(500, input_shape=(238,),kernel_initializer= {{choice(['glorot_uniform','random_uniform'])}}))
model.add(BatchNormalization(epsilon=1e-06, mode=0, momentum=0.9, weights=None))
model.add(Activation({{choice(['relu','sigmoid','tanh'])}}))
model.add(Dropout({{uniform(0, 0.3)}}))
model.add(Dense({{choice([128,256])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 0.4)}}))
model.add(Dense({{choice([128,256])}}))
model.add(Activation({{choice(['relu','tanh'])}}))
model.add(Dropout(0.3))
model.add(Dense(41))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer={{choice(['rmsprop', 'adam'])}})
model.summary()
early_stops = EarlyStopping(patience=3, monitor='val_acc')
ckpt_callback = ModelCheckpoint('keras_model',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto')
model.fit(train_X, train_y, batch_size={{choice([128,264])}}, nb_epoch={{choice([10,20])}}, validation_data=(test_X, test_y), callbacks=[early_stops,ckpt_callback])
score, acc = model.evaluate(test_X, test_y, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def new_model(bendata, benlabel, benmir, benslabel):
n = len(bendata[0])
"""
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
"""
model = Sequential()
model.add(Dense({{choice([32, 64, 128, 256, 512])}}, input_shape=(n, )))
model.add(Activation({{choice(['relu', 'sigmoid'])}}))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([32, 64, 128, 256, 512])}}))
model.add(Activation({{choice(['relu', 'sigmoid'])}}))
#model.add({{choice([Dropout(0.1), Dropout(0.2), Dropout(0.3)])}})
model.add(Dropout({{uniform(0, 1)}}))
# If we choose 'four', add an additional fourth layer
if {{choice(['two', 'three'])}} == 'three':
model.add(Dense({{choice([32, 64, 128, 256, 512])}}))
model.add(Activation({{choice(['relu', 'sigmoid'])}}))
#model.add({{choice([Dropout(0.1), Dropout(0.2), Dropout(0.3)])}})
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(1))
model.add(Activation({{choice(['sigmoid', 'softmax', 'linear'])}}))
adam = Adam(lr={{choice([10**-4, 10**-3, 10**-2, 10**-1])}})
rmsprop = RMSprop(lr={{choice([10**-4, 10**-3, 10**-2, 10**-1])}})
sgd = SGD(lr={{choice([10**-4, 10**-3, 10**-2, 10**-1])}})
chosen_par = {{choice(['adam', 'rmsprop', 'sgd'])}}
if chosen_par == 'adam':
optm = adam
elif chosen_par == 'rmsprop':
optm = rmsprop
else:
optm = sgd
model.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=optm)
result = model.fit(
bendata,
benlabel,
batch_size={{choice([32, 64, 128])}},
epochs={{choice([2, 4, 10, 20])}},
#epochs=2,
verbose=2,
validation_split=0.1)
#get the highest validation accuracy of the training epochs
validation_acc = np.amax(result.history['val_acc'])
print('Best test accuracy of epoch:', validation_acc)
return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
def model(train, test):
'''
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
'''
model = Sequential()
model.add(Dense(512, input_shape=(666,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(666))
model.add(Activation('linear'))
rms = RMSprop()
model.compile(loss='mse', optimizer=rms, metrics=['mse'])
model.fit(train, train,
batch_size={{choice([64, 128])}},
nb_epoch=1,
verbose=2
)
score, mse = model.evaluate(test, test, verbose=0)
print('Test accuracy for new:', mse)
return {'loss': mse, 'status': STATUS_OK, 'model': model}
def model(X_train, X_test, y_train, y_test):
model = Sequential()
model.add(con_model())
model.add(Conv2D({{choice([64, 128, 256, 512])}}, (3, 3), activation='relu', padding='same', name="block_converge_2")) # ,input_shape=(IMG_SIZE, IMG_SIZE, 3)))
model.add(Dropout({{uniform(0,0.5)}}))
model.add(Conv2D({{choice([64, 128, 256, 512])}}, (3, 3), activation='relu', padding='same', name="block_converge_3"))
model.add(Dropout({{uniform(0, 0.5)}}))
model.add(Conv2D({{choice([64, 128, 256, 512])}}, (3, 3), activation='relu', padding='same', name="block_converge_4"))
model.add(Dropout({{uniform(0, 0.5)}}))
model.add(BatchNormalization())
model.add(Flatten())
model.add(Dense(2, activation='softmax', name="block_converge_5k"))
sgd = SGD(lr=1e-4, decay={{choice([1e-4, 1e-5, 1e-6])}}, momentum={{uniform(0, 0.9)}}, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
model.fit(
[X_train[0], X_train[1]],
y_train,
batch_size=16,
epochs=30,
validation_data=( [X_test[0], X_test[1]], y_test))
score, acc = model.evaluate([X_test[0], X_test[1]], y_test, verbose=0)
print('Test score:', score)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(x_train, y_train, x_test, y_test):
"""
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
"""
model = Sequential()
model.add(Dense({{choice([8, 16, 32])}}, input_shape=(19,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([8, 16, 32])}}))
model.add(Activation({{choice(['relu', 'linear'])}}))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(1, activation='linear'))
model.compile(loss='mae', optimizer="adam", metrics=['mae'])
result = model.fit(x_train, y_train,
batch_size={{choice([4,8,16,32, 64, 128])}},
epochs=50,
verbose=1,
validation_data=(x_test, y_test))
#get the highest validation accuracy of the training epochs
score, acc = model.evaluate(x_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def nn(X, Y):
model = Sequential()
M = Masking(mask_value=0, input_shape=(54, 1))
model.add(M)
model.add(LSTM(500, input_shape=(None, 1), activation="tanh"))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation({{choice(['relu', 'tanh', 'softmax'])}}))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation({{choice(['relu', 'tanh', 'softmax'])}}))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer={{choice(['rmsprop', 'adam', 'adadelta'])}},
loss='binary_crossentropy',
metrics=['accuracy'])
# Debugging: Predict training examples themselves (since no validation split)...
num_examples = 1
model.fit(X[:num_examples, :, :], Y[:num_examples], epochs=1, batch_size=32)
# for i in range(NUM_EXAMPLES):
# print(model.predict_classes(X[i].reshape(1, 54, 1)))
score, acc = model.evaluate(X[:num_examples, :, :], Y[:num_examples], verbose=0)
print('Test score:', score)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, X_test, Y_train, Y_test):
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([400, 512, 600])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy', optimizer=rms, metrics=['accuracy'])
nb_epoch = 10
batch_size = 128
model.fit(X_train, Y_train,
batch_size=batch_size, nb_epoch=nb_epoch,
verbose=2,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(data, labels, val_data, val_labels):
model = Sequential()
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10))
# Prior precision lower and upper bounds are from appendix I.1 tests
optimizer = Vadam(lr={{uniform(1e-4, 1e-2)}},
prior_prec={{uniform(1e-2, 25)}},
train_set_size=1000)
model.compile(loss='kld',
metrics=['mae'],
optimizer=optimizer)
result = model.fit(data,
labels,
epochs=10,
batch_size=32,
validation_data=(val_data, val_labels))
# get the lowest validation loss of each training epoch
validation_loss = np.amin(result.history['val_loss'])
print('Best loss of epoch:', validation_loss)
return {'loss': validation_loss, 'status': STATUS_OK, 'model': model}
def model(depnet_feat_dev1, depnet_feat_dev2, depnet_feat_val, img_feat_dev1, img_feat_dev2, img_feat_val, q_dev1, q_dev2, q_val, a_dev1, a_dev2, a_val, qdict_dev1, adict_dev1):
from keras.models import Sequential
from keras.layers.embeddings import Embedding
from keras.layers.core import Lambda, Dense, Activation, Merge, Dropout, Reshape
from keras.callbacks import EarlyStopping, ModelCheckpoint
import keras.backend as K
import os
path2outdir = os.environ.get('OUTDIR', 'no')
vocab_size = len(qdict_dev1)
nb_ans = len(adict_dev1) - 1
nb_epoch = 1000
quest_model = Sequential()
quest_model.add(Embedding(input_dim=vocab_size, output_dim={{choice([100, 200, 300, 500])}},
init={{choice(['uniform', 'normal', 'glorot_uniform', 'glorot_normal', 'he_normal', 'he_uniform'])}},
mask_zero=False, dropout={{uniform(0,1)}}
)
)
quest_model.add(Lambda(function=lambda x: K.sum(x, axis=1), output_shape=lambda shape: (shape[0], ) + shape[2:]))
nb_img_feat = img_feat_dev1.shape[1]
img_model = Sequential()
img_model.add(Reshape((nb_img_feat, ), input_shape=(nb_img_feat, )))
nb_depnet_feat = depnet_feat_dev1.shape[1]
depnet_model = Sequential()
depnet_model.add(Reshape((nb_depnet_feat, ), input_shape=(nb_depnet_feat, )))
multimodal = Sequential()
multimodal.add(Merge([img_model, depnet_model, quest_model], mode='concat', concat_axis=1))
multimodal.add(Dropout({{uniform(0, 1)}}))
multimodal.add(Dense(nb_ans))
multimodal.add(Activation('softmax'))
multimodal.compile(loss='categorical_crossentropy',
optimizer={{choice(['sgd', 'adam', 'rmsprop', 'adagrad', 'adadelta', 'adamax'])}},
metrics=['accuracy'])
print('##################################')
print('Train...')
early_stopping = EarlyStopping(monitor='val_loss', patience=10)
checkpointer = ModelCheckpoint(filepath=os.path.join(path2outdir, 'cnn_bow_weights.hdf5'), verbose=1, save_best_only=True)
multimodal.fit([img_feat_dev1, depnet_feat_dev1, q_dev1], a_dev1, batch_size={{choice([32, 64, 100])}}, nb_epoch=nb_epoch,
validation_data=([img_feat_dev2, depnet_feat_dev2, q_dev2], a_dev2),
callbacks=[early_stopping, checkpointer])
multimodal.load_weights(os.path.join(path2outdir, 'cnn_bow_weights.hdf5'))
score, acc = multimodal.evaluate([img_feat_val, depnet_feat_val, q_val], a_val, verbose=1)
print('##################################')
print('Test accuracy:%.4f' % acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': multimodal}
def model(train_X, train_Y, test_X, test_Y):
'''
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
'''
model = Sequential()
model.add(Dense(500,input_shape=(train_X.shape[1],)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([512, 1024])}}))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([512, 1024])}}))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout({{uniform(0, 1)}}))
# If we choose 'four', add an additional fourth layer
if conditional({{choice(['three', 'four'])}}) == 'four':
model.add(Dense(500))
# We can also choose between complete sets of layers
model.add({{choice([Dropout(0.5), Activation('linear')])}})
model.add(Activation('relu'))
model.add(Dense(train_Y.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', metrics=['accuracy'],
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})
model.fit(train_X, train_Y,
batch_size={{choice([128, 256])}},
nb_epoch=1,
verbose=2,
validation_data=(test_X, test_Y))
score, acc = model.evaluate(test_X, test_Y, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(datagen, X_train, Y_train, X_test, Y_test):
batch_size = 32
nb_epoch = 200
# input image dimensions
img_rows, img_cols = 32, 32
# the CIFAR10 images are RGB
img_channels = 3
model = Sequential()
model.add(Convolution2D(32, 3, 3, border_mode='same',
input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
# fit the model on the batches generated by datagen.flow()
model.fit_generator(datagen.flow(X_train, Y_train,
batch_size=batch_size),
samples_per_epoch=X_train.shape[0],
nb_epoch=nb_epoch,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def build_model(self, window_size, word_vector_size, activation_function, dense_layer_sizes,
hidden_dropout_rate, dropout, k_output, name, hyperparameter_search=False):
self.model_info = {}
self.model_info['window_size'] = window_size
self.model_info['activation_function'] = activation_function
self.model_info['k_output'] = k_output
self.model_info['dropout'] = dropout
self.model_info['hidden_dropout_rate'] = hidden_dropout_rate
self.model_info['dense_layer_sizes'] = dense_layer_sizes
self.model_info['name'] = name
self.model_info['word_vector_size'] = word_vector_size
self.model_info['hyperparameter_search'] = hyperparameter_search
model = Sequential()
model.add(Dense(100, input_dim=(window_size * 2 + 1) * word_vector_size))
model.add(Activation(activation_function))
for layer_size in dense_layer_sizes:
model.add(Dense(layer_size))
if dropout:
if hyperparameter_search:
hidden_dropout_rate = {{uniform(0, 1)}}
model.add(Dropout(hidden_dropout_rate))
model.add(Dense(k_output))
model.add(Activation('softmax'))
self.model = model
return model
def model(X_train, X_test, y_train, y_test, maxlen, max_features):
embedding_size = 300
pool_length = 4
lstm_output_size = 100
batch_size = 200
nb_epoch = 1
model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout({{uniform(0, 1)}}))
# Note that we use unnamed parameters here, which is bad style, but is used here
# to demonstrate that it works. Always prefer named parameters.
model.add(Convolution1D({{choice([64, 128])}},
{{choice([6, 8])}},
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_data=(X_test, y_test))
score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(x_train, y_train, x_test, y_test):
"""
Create your model...
"""
layer_1_size = {{quniform(12, 256, 4)}}
l1_dropout = {{uniform(0.001, 0.7)}}
params = {
'l1_size': layer_1_size,
'l1_dropout': l1_dropout
}
num_classes = 10
model = Sequential()
model.add(Dense(int(layer_1_size), activation='relu'))
model.add(Dropout(l1_dropout))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=128, epochs=10, validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, verbose=0)
out = {
'loss': -acc,
'score': score,
'status': STATUS_OK,
'model_params': params,
}
# optionally store a dump of your model here so you can get it from the database later
temp_name = tempfile.gettempdir()+'/'+next(tempfile._get_candidate_names()) + '.h5'
model.save(temp_name)
with open(temp_name, 'rb') as infile:
model_bytes = infile.read()
out['model_serial'] = model_bytes
return out
def model(X_train, X_test, y_train, y_test, max_features, maxlen):
model = Sequential()
model.add(Embedding(max_features, 128, input_length=maxlen))
model.add(LSTM(128))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
early_stopping = EarlyStopping(monitor='val_loss', patience=4)
checkpointer = ModelCheckpoint(filepath='keras_weights.hdf5',
verbose=1,
save_best_only=True)
model.fit(X_train, y_train,
batch_size={{choice([32, 64, 128])}},
nb_epoch=1,
validation_split=0.08,
callbacks=[early_stopping, checkpointer])
score, acc = model.evaluate(X_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def keras_model():
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import RMSprop
from keras.utils import np_utils
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
nb_classes = 10
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy', optimizer=rms)
model.fit(X_train, Y_train,
batch_size={{choice([64, 128])}},
nb_epoch=1,
show_accuracy=True,
verbose=2,
validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test,
show_accuracy=True, verbose=0)
print('Test accuracy:', score[1])
return {'loss': -score[1], 'status': STATUS_OK}
def model(X_train, Y_train, X_test, Y_test):
'''
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
'''
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation({{choice(['relu', 'sigmoid'])}}))
model.add(Dropout({{uniform(0, 1)}}))
# If we choose 'four', add an additional fourth layer
if conditional({{choice(['three', 'four'])}}) == 'four':
model.add(Dense(100))
model.add({{choice([Dropout(0.5), Activation('linear')])}})
model.add(Activation('relu'))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})
model.fit(X_train, Y_train,
batch_size={{choice([64, 128])}},
nb_epoch=1,
show_accuracy=True,
verbose=2,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, Y_train, X_test, Y_test):
model = Sequential()
model.add(Dense(50, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([20, 30, 40])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy', optimizer=rms, metrics=['accuracy'])
model.fit(X_train, Y_train,
batch_size={{choice([64, 128])}},
epochs=1,
verbose=2,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(x_train, y_train, x_test, y_test):
batch_size = 256
epochs = 1
learning_rate = 0.8713270582626444
momentum = 0.8671876498073315
decay = 0.0
early_stop_th = 10**-5
input_dim = (784,)
dropout_1 = 0.026079803111884514
dropout_2 = 0.4844455237320119
# Stop the training if the accuracy is not moving more than a delta
# keras.callbacks.History is by default added to all keras model
# callbacks = [EarlyStopping(monitor='acc', min_delta=early_stop_th, patience=5, verbose=0, mode='auto')]
# Code up the network
x_input = Input(input_dim)
x = Dropout(dropout_1)(x_input)
x = Dense(1024, activation='relu', name ="dense1",kernel_constraint=max_norm( {{uniform(0.9, 5)}} ) )(x)
x = Dropout(dropout_2)(x)
x = Dense(1024, activation='relu', name = "dense2",kernel_constraint=max_norm( {{uniform(0.9,5)}} ) )(x)
predictions = Dense(10, activation='softmax')(x)
# Optimizer
sgd = optimizers.SGD(lr=learning_rate, momentum=momentum, decay=0, nesterov=False)
# Create and train model
model = Model(inputs = x_input, outputs = predictions)
model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(x=x_train,y= y_train, validation_split = 0.1, batch_size = batch_size ,epochs = epochs, verbose = 1)
metrics = model.evaluate(x=x_test, y=y_test, batch_size=batch_size, verbose=0, sample_weight=None, steps=None)
accuracy = metrics[1]
return {'loss': 1-accuracy, 'status': STATUS_OK, 'model': model}
def keras_model():
from keras.models import Sequential
from keras.layers.embeddings import Embedding
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.layers.core import Dense, Reshape, Activation, Flatten, Dropout
from keras.regularizers import l1, activity_l1, l2, activity_l2
from aiding_funcs.embeddings_handling import get_the_folds, join_folds
import pickle
embeddings = pickle.load( open( "/data/dpappas/personality/emb.p", "rb" ) )
train = pickle.load( open( "/data/dpappas/personality/train.p", "rb" ) )
no_of_folds = 10
folds = get_the_folds(train,no_of_folds)
train_data = join_folds(folds,folds.keys()[:-1])
validation_data = folds[folds.keys()[-1]]
max_input_length = validation_data['features'].shape[1]
CNN_filters = {{choice([5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95])}}
CNN_rows = {{choice([1,2,3,4,5,6])}}
Dense_size = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
opt = {{choice([ 'adadelta','sgd','rmsprop', 'adagrad', 'adadelta', 'adam'])}}
is_trainable = {{choice([ True, False ])}}
D = embeddings.shape[-1]
cols = D
out_dim = 5
model = Sequential()
model.add(Embedding(input_dim = embeddings.shape[0], output_dim=D, weights=[embeddings], trainable=is_trainable, input_length = max_input_length))
model.add(Reshape((1, max_input_length, D)))
model.add(Convolution2D( CNN_filters, CNN_rows, cols, dim_ordering='th', activation='sigmoid' ))
sh = model.layers[-1].output_shape
model.add(MaxPooling2D(pool_size=(sh[-2], sh[-1]),dim_ordering = 'th'))
model.add(Flatten())
model.add(Dense(Dense_size, activation='sigmoid',W_regularizer=l2({{uniform(0, 1)}}),activity_regularizer=activity_l2({{uniform(0, 1)}})))
model.add(Dense(out_dim, activation='linear',W_regularizer=l2({{uniform(0, 1)}}),activity_regularizer=activity_l2({{uniform(0, 1)}})))
model.compile(loss='mse', optimizer=opt)
model.fit(train_data['features'], train_data['labels'], nb_epoch=50, show_accuracy=False, verbose=2)
#score = model.evaluate( validation_data['features'], validation_data['labels'])
score = model.evaluate( train_data['features'], train_data['labels'])
return {'loss': score, 'status': STATUS_OK}
def create_model(tr_pairs, tr_y, te_pairs, te_y,input_shape):
epochs = 20
dropout1 = {{uniform(0,1)}}
dropout2 = {{uniform(0,1)}}
dense_filter1 = {{choice([64,128,256])}}
dense_filter2 = {{choice([64,128,256])}}
dense_filter3 = {{choice([64,128,256])}}
# network definition
base_network = create_base_network(input_shape,dense_filter1,dense_filter2,dense_filter3,dropout1,dropout2)
input_a = Input(shape=input_shape)
input_b = Input(shape=input_shape)
processed_a = base_network(input_a)
processed_b = base_network(input_b)
distance = Lambda(euclidean_distance,
output_shape=eucl_dist_output_shape)([processed_a, processed_b])
model = Model([input_a, input_b], distance)
rms = RMSprop()
model.compile(loss=contrastive_loss, optimizer=rms, metrics=[accuracy])
model.fit([tr_pairs[:, 0], tr_pairs[:, 1]], tr_y,
batch_size=128,
epochs=epochs,
verbose=1,
validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y))
y_pred = model.predict([tr_pairs[:, 0], tr_pairs[:, 1]])
tr_acc = compute_accuracy(tr_y, y_pred)
y_pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])
te_acc = compute_accuracy(te_y, y_pred)
print('* Accuracy on training set: %0.2f%%' % (100 * tr_acc))
print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))
return {'loss': -te_acc, 'status': STATUS_OK, 'model': model}
def keras_model():
from keras.models import Sequential, Graph
from keras.layers.embeddings import Embedding
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.layers.core import Dense, Reshape, Activation, Flatten, Dropout
from keras.regularizers import l1, activity_l1, l2, activity_l2
from aiding_funcs.embeddings_handling import get_the_folds, join_folds
import pickle
embeddings = pickle.load( open( "/data/dpappas/personality/emb.p", "rb" ) )
train = pickle.load( open( "/data/dpappas/personality/train.p", "rb" ) )
no_of_folds = 10
folds = get_the_folds(train,no_of_folds)
train_data = join_folds(folds,folds.keys()[:-1])
validation_data = folds[folds.keys()[-1]]
max_input_length = validation_data['features'].shape[1]
CNN_filters = {{choice([5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200])}}
CNN_rows = {{choice([1,2,3,4,5,6,7,8,9,10])}}
Dense_size = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
Dense_size2 = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
Dense_size3 = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
opt = {{choice([ 'adadelta','sgd', 'adam'])}}
is_trainable = {{choice([ True, False ])}}
D = embeddings.shape[-1]
cols = D
out_dim = train_data['labels'].shape[-1]
graph = Graph()
graph.add_input(name='txt_data', input_shape=[train_data['features'].shape[-1]], dtype='int')
graph.add_node(Embedding( input_dim = embeddings.shape[0], output_dim=D, weights=[embeddings], trainable=is_trainable, input_length = max_input_length), name='Emb', input='txt_data')
graph.add_node(Reshape((1, max_input_length, D)), name = "Reshape", input='Emb')
graph.add_node( Convolution2D(CNN_filters, CNN_rows, cols, activation='sigmoid' ) , name='Conv', input='Reshape')
sh = graph.nodes['Conv'].output_shape
graph.add_node( MaxPooling2D(pool_size=(sh[-2], sh[-1])) , name='MaxPool', input='Conv')
graph.add_node( Flatten() , name='Flat', input='MaxPool')
graph.add_node( Dense(Dense_size, activation='sigmoid',W_regularizer=l2({{uniform(0, 1)}}),activity_regularizer=activity_l2({{uniform(0, 1)}})) , name='Dtxt', input='Flat')
graph.add_node( Dropout({{uniform(0, 1)}}) , name='Dropout1', input='Dtxt')
graph.add_input(name='av_data', input_shape=[train_data['AV'].shape[-1]])
graph.add_node( Dense(Dense_size2, activation='sigmoid',W_regularizer=l2({{uniform(0, 1)}}),activity_regularizer=activity_l2({{uniform(0, 1)}})) , name='Dav', input='av_data')
graph.add_node( Dropout({{uniform(0, 1)}}) , name='Dropout2', input='Dav')
graph.add_node( Dense(Dense_size3, activation='sigmoid',W_regularizer=l2({{uniform(0, 1)}}),activity_regularizer=activity_l2({{uniform(0, 1)}})), name='Dense1', inputs=['Dropout2', 'Dropout1'], merge_mode='concat')
graph.add_node( Dropout({{uniform(0, 1)}}) , name='Dropout3', input='Dense1')
graph.add_node( Dense(out_dim, activation='linear') , name='Dense2', input='Dropout3')
graph.add_output(name='output', input = 'Dense2')
graph.compile(optimizer=opt, loss={'output':'rmse'})
graph.fit(
{
'txt_data':train_data['features'],
'av_data':train_data['AV'],
'output':train_data['labels']
},
nb_epoch=500,
batch_size=64
)
scores = graph.evaluate({'txt_data':validation_data['features'], 'av_data':validation_data['AV'], 'output':validation_data['labels']})
print(scores)
return {'loss': scores, 'status': STATUS_OK}
def model(X_train, X_test, y_train, y_test, maxlen, max_features):
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM
from keras.layers.convolutional import Convolution1D, MaxPooling1D
# Embedding
embedding_size = 300
# Convolution
filter_length = 6
nb_filter = 64
pool_length = 4
# LSTM
lstm_output_size = 100
# Training
batch_size = 60
nb_epoch = 2
model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
class_mode='binary')
print('Train...')
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_data=(X_test, y_test), show_accuracy=True)
score, acc = model.evaluate(X_test, y_test, batch_size=batch_size,
show_accuracy=True)
print('Test score:', score)
print('Test accuracy:', acc)
return {'loss': -score[1], 'status': STATUS_OK}
def model(q_train, q_dev, q_val, a_train, a_dev, a_val, qdict, adict):
from keras.models import Sequential
from keras.layers.embeddings import Embedding
from keras.layers.core import Lambda, Dense, Activation
from keras.callbacks import EarlyStopping, ModelCheckpoint
import keras.backend as K
vocab_size = len(qdict)
nb_ans = len(adict)
nb_epoch = 1000
quest_model = Sequential()
quest_model.add(Embedding(input_dim=vocab_size, output_dim={{choice([100, 200, 300, 500])}},
init = {{choice(['uniform', 'normal', 'glorot_uniform', 'glorot_normal', 'he_normal', 'he_uniform'])}},
mask_zero=False, dropout={{uniform(0,1)}}
)
)
quest_model.add(Lambda(function=lambda x: K.sum(x, axis=1), output_shape=lambda shape: (shape[0], ) + shape[2:]))
quest_model.add(Dense(nb_ans))
quest_model.add(Activation('softmax'))
quest_model.compile(loss='categorical_crossentropy',
optimizer={{choice(['sgd', 'adam', 'rmsprop', 'adagrad', 'adadelta', 'adamax'])}},
metrics=['accuracy'])
print('##################################')
print('Train...')
early_stopping = EarlyStopping(monitor='val_loss', patience=10)
checkpointer = ModelCheckpoint(filepath='keras_weights.hdf5', verbose=1, save_best_only=True)
quest_model.fit(q_train, a_train, batch_size={{choice([32, 64, 100])}}, nb_epoch=nb_epoch,
validation_data=(q_dev, a_dev),
callbacks=[early_stopping, checkpointer])
score, acc = quest_model.evaluate(q_val, a_val, verbose=1)
print('##################################')
print('Test accuracy:%.4f' % acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': quest_model}
def keras_model():
from keras.models import Sequential
from keras.layers.core import Dense
from keras.regularizers import l2, activity_l2
from aiding_funcs.embeddings_handling import get_the_folds, join_folds
from aiding_funcs.label_handling import MaxMin, MaxMinFit
import pickle
print('loading test.p')
test = pickle.load( open( "/data/dpappas/Common_Crawl_840B_tokkens_pickles/test.p", "rb" ) )
print('loading train.p')
train = pickle.load( open( "/data/dpappas/Common_Crawl_840B_tokkens_pickles/train.p", "rb" ) )
no_of_folds = 10
folds = get_the_folds(train,no_of_folds)
train_data = join_folds(folds,folds.keys()[:-1])
validation_data = folds[folds.keys()[-1]]
mins, maxs = MaxMin(train_data['labels'])
T_l = MaxMinFit(train_data['labels'], mins, maxs)
t_l = MaxMinFit(validation_data['labels'], mins, maxs)
Dense_size = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
Dense_size2 = {{choice([50, 100, 150, 200, 250, 300, 350, 400, 450, 500])}}
opt = {{choice([ 'adadelta','sgd','rmsprop', 'adagrad', 'adadelta', 'adam'])}}
out_dim = 5
activity_l2_0 = {{uniform(0, 1)}}
activity_l2_1 = {{uniform(0, 1)}}
activity_l2_2 = {{uniform(0, 1)}}
l2_0 = {{uniform(0, 1)}}
l2_1 = {{uniform(0, 1)}}
l2_2 = {{uniform(0, 1)}}
model = Sequential()
model.add(Dense(Dense_size, activation='sigmoid',W_regularizer=l2(l2_0),activity_regularizer=activity_l2(activity_l2_0),input_dim = train_data['skipthoughts'].shape[-1] ))
model.add(Dense(Dense_size2, activation='sigmoid',W_regularizer=l2(l2_1),activity_regularizer=activity_l2(activity_l2_1)))
model.add(Dense(out_dim, activation='linear',W_regularizer=l2(l2_2),activity_regularizer=activity_l2(activity_l2_2)))
model.compile(loss='rmse', optimizer=opt)
#model.fit(train_data['skipthoughts'], train_data['labels'], nb_epoch=500, show_accuracy=False, verbose=2)
#score = model.evaluate( train_data['skipthoughts'], train_data['labels'])
model.fit(train_data['skipthoughts'], T_l, nb_epoch=500, show_accuracy=False, verbose=2)
score = model.evaluate( train_data['skipthoughts'], T_l)
print("score : " +str(score))
return {'loss': score, 'status': STATUS_OK}
def model(X_train, X_test, y_train, y_test, max_features, maxlen):
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM
from keras.callbacks import EarlyStopping, ModelCheckpoint
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128, input_length=maxlen))
model.add(LSTM(128))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
class_mode="binary")
early_stopping = EarlyStopping(monitor='val_loss', patience=4)
checkpointer = ModelCheckpoint(filepath='keras_weights.hdf5',
verbose=1,
save_best_only=True)
hist = model.fit(X_train, y_train,
batch_size={{choice([32, 64, 128])}},
# batch_size=128,
nb_epoch=1,
validation_split=0.08,
show_accuracy=True,
callbacks=[early_stopping, checkpointer])
score, acc = model.evaluate(X_test, y_test, show_accuracy=True, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, Y_train, X_test, Y_test):
img_rows, img_cols = 32, 32
img_channels = 3
nb_dim = 50
nb_epoch=15#35#30
dense_layer_size = {{choice([256, 512, 1024])}}
objective = 'mse'
#optimizer = {{choice(['rmsprop', 'adam', 'sgd'])}}
optimizer = {{choice(['rmsprop', 'sgd'])}}
batch_size = {{choice([32, 64, 128])}}
num_conv1 = int({{quniform(24, 64, 1)}})
num_conv2 = int({{quniform(32, 96, 1)}})
size_conv1 = int({{quniform(2, 5, 1)}})
size_conv2 = int({{quniform(2, 5, 1)}})
early_dropout = {{uniform(0,.75)}}
late_dropout = {{uniform(0,.75)}}
data_augmentation = {{choice(['True','False'])}}
final_activation = {{choice(['none','linear'])}}
params = {'dense_layer_size':dense_layer_size,
'optimizer':optimizer,
'batch_size':batch_size,
'num_conv1':num_conv1,
'num_conv2':num_conv2,
'size_conv1':size_conv1,
'size_conv2':size_conv2,
'final_activation':final_activation,
'early_dropout':early_dropout,
'late_dropout':late_dropout
}
if optimizer == 'sgd':
learning_rate = {{loguniform(np.log(0.001),np.log(0.999))}}
params['learning_rate'] = learning_rate
if data_augmentation:
more_augmentation = {{choice(['True','False'])}}
params['more_augmentation'] = more_augmentation
model = Sequential()
model.add(Convolution2D(num_conv1, size_conv1, size_conv1, border_mode='same',
input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(num_conv1, size_conv1, size_conv1))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(early_dropout))
model.add(Convolution2D(num_conv2, size_conv2, size_conv2, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(num_conv2, size_conv2, size_conv2))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(early_dropout))
model.add(Flatten())
model.add(Dense(dense_layer_size))
model.add(Activation('relu'))
model.add(Dropout(late_dropout))
model.add(Dense(nb_dim))
if final_activation != 'none':
model.add(Activation(final_activation))
if optimizer == 'sgd':
# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=learning_rate, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss=objective, optimizer=sgd)
elif optimizer == 'rmsprop':
model.compile(loss=objective, optimizer='rmsprop')
else:
model.compile(loss=objective, optimizer=optimizer)
print(params)
if not data_augmentation:
print('Not using data augmentation.')
history = model.fit(X_train, Y_train, batch_size=batch_size,
nb_epoch=nb_epoch, show_accuracy=True,
validation_data=(X_test, Y_test), shuffle=True)
else:
print('Using real-time data augmentation.')
if more_augmentation:
# this will do preprocessing and realtime data augmentation
datagen = ImageDataGenerator(
featurewise_center=True, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=True, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images
else:
# this will do preprocessing and realtime data augmentation
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(X_train)
# fit the model on the batches generated by datagen.flow()
history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
samples_per_epoch=X_train.shape[0],
nb_epoch=nb_epoch, show_accuracy=True,
validation_data=(X_test, Y_test),
nb_worker=1)
#score, acc = model.evaluate(X_test, Y_test, verbose=0)
loss = model.evaluate(X_test, Y_test, verbose=0)
print('Test loss:', loss)
return {'loss': loss, 'status': STATUS_OK, 'params':params}
def model(q_train, q_dev, q_val, a_train, a_dev, a_val, qdict, adict):
from keras.models import Sequential
from keras.layers.embeddings import Embedding
from keras.layers.core import Dense, Activation
from keras.layers.recurrent import LSTM
from keras.callbacks import EarlyStopping, ModelCheckpoint
vocab_size = len(qdict)
nb_ans = len(adict)
nb_epoch = 1000
quest_model = Sequential()
quest_model.add(Embedding(input_dim=vocab_size, output_dim={{choice([100, 200, 300, 500])}},
init = {{choice(['uniform', 'lecun_uniform', 'normal',
'identity', 'glorot_uniform', 'glorot_normal',
'he_normal', 'he_uniform'])}},
mask_zero=True, dropout={{uniform(0, 1)}}
)
)
nb_ltsmlayer = {{choice([1, 2])}}
if nb_ltsmlayer == 1:
quest_model.add(LSTM(output_dim={{choice([100, 200, 300, 500])}},
init={{choice(['uniform', 'lecun_uniform', 'normal',
'identity', 'glorot_uniform', 'glorot_normal',
'orthogonal', 'he_normal', 'he_uniform'])}},
inner_init={{choice(['uniform', 'lecun_uniform', 'normal',
'identity', 'glorot_uniform', 'glorot_normal',
'orthogonal', 'he_normal', 'he_uniform'])}},
activation={{choice(['relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'])}},
inner_activation={{choice(['relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'])}},
W_regularizer={{choice([None, 'l1', 'l2', 'l1l2'])}},
U_regularizer={{choice([None, 'l1', 'l2', 'l1l2'])}},
b_regularizer={{choice([None, 'l1', 'l2', 'l1l2'])}},
dropout_W={{uniform(0, 1)}},
dropout_U={{uniform(0, 1)}},
return_sequences=False))
else:
for i in range(nb_ltsmlayer-1):
quest_model.add(LSTM(output_dim={{choice([100, 200, 300, 500])}},
init={{choice(['uniform', 'lecun_uniform', 'normal',
'identity', 'glorot_uniform', 'glorot_normal',
'orthogonal', 'he_normal', 'he_uniform'])}},
inner_init={{choice(['uniform', 'lecun_uniform', 'normal',
'identity', 'glorot_uniform', 'glorot_normal',
'orthogonal', 'he_normal', 'he_uniform'])}},
activation={{choice(['relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'])}},
inner_activation={{choice(['relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'])}},
W_regularizer={{choice([None, 'l1', 'l2', 'l1l2'])}},
U_regularizer={{choice([None, 'l1', 'l2', 'l1l2'])}},
b_regularizer={{choice([None, 'l1', 'l2', 'l1l2'])}},
dropout_W={{uniform(0, 1)}},
dropout_U={{uniform(0, 1)}},
return_sequences=True))
quest_model.add(LSTM(output_dim={{choice([100, 200, 300, 500])}},
init={{choice(['uniform', 'lecun_uniform', 'normal',
'identity', 'glorot_uniform', 'glorot_normal',
'orthogonal', 'he_normal', 'he_uniform'])}},
inner_init={{choice(['uniform', 'lecun_uniform', 'normal',
'identity', 'glorot_uniform', 'glorot_normal',
'orthogonal', 'he_normal', 'he_uniform'])}},
activation={{choice(['relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'])}},
inner_activation={{choice(['relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'])}},
W_regularizer={{choice([None, 'l1', 'l2', 'l1l2'])}},
U_regularizer={{choice([None, 'l1', 'l2', 'l1l2'])}},
b_regularizer={{choice([None, 'l1', 'l2', 'l1l2'])}},
dropout_W={{uniform(0, 1)}},
dropout_U={{uniform(0, 1)}},
return_sequences=False))
quest_model.add(Dense(nb_ans))
quest_model.add(Activation('softmax'))
quest_model.compile(loss='categorical_crossentropy',
optimizer={{choice(['adam', 'rmsprop', 'adagrad', 'adadelta', 'adamax'])}},
metrics=['accuracy'])
print('##################################')
print('Train...')
early_stopping = EarlyStopping(monitor='val_loss', patience=10)
checkpointer = ModelCheckpoint(filepath='lstm_keras_weights.hdf5', verbose=1, save_best_only=True)
quest_model.fit(q_train, a_train, batch_size={{choice([32, 64, 100])}}, nb_epoch=nb_epoch,
validation_data=(q_dev, a_dev),
callbacks=[early_stopping, checkpointer])
quest_model.load_weights('lstm_keras_weights.hdf5')
score, acc = quest_model.evaluate(q_val, a_val, verbose=1)
print('##################################')
print('Test accuracy:%.4f' % acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': quest_model}
Pool_Valid_Acc = np.zeros(shape=(nb_epoch, 1))
Pool_Train_Acc = np.zeros(shape=(nb_epoch, 1))
x_pool_All = np.zeros(shape=(1))
Y_train = np_utils.to_categorical(y_train, nb_classes)
print('Training Model Without Acquisitions in Experiment', e)
model = Sequential()
model.add(Convolution2D(nb_filters, nb_conv, nb_conv, border_mode='valid', input_shape=(1, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Convolution2D(nb_filters*2, nb_conv, nb_conv, border_mode='valid', input_shape=(1, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(nb_filters*2, nb_conv, nb_conv))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout({{uniform(0, 1)}}))
c = 0.25
Weight_Decay = c / float(X_train.shape[0])
model.add(Flatten())
model.add(Dense(128, W_regularizer=l2(Weight_Decay)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(nb_classes))
