def cnn_model(x_train, y_train, x_val, y_val, params):
embedding_layer = Embedding(MAX_NUM_WORDS,
EMBEDDING_DIM,
embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
print('Training model.')
# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu', padding = 'same')(x)
x = MaxPooling1D(5)(x)
# x = BatchNormalization()(x)
x = Conv1D(128, 5, activation='relu', padding = 'same')(x)
x = GlobalMaxPooling1D()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(params['dropout'])(x)
preds = Dense(len(labels_index), activation='softmax')(x)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer=params['optimizer'](lr=lr_normalizer(params['lr'],params['optimizer'])),
metrics=['acc'])
history = model.fit(x_train, y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
validation_data=(x_val, y_val))
return history, model
def higgs_nn(X_train, Y_train, X_valid, Y_valid, params):
model = Sequential()
model.add(
Dense(75,
input_dim=X_train.shape[1],
activation=params['activation'],
kernel_initializer='normal'))
model.add(Dropout(params['dropout']))
hidden_layers(model, params, 2)
model.add(
Dense(2,
activation=params['last_activation'],
kernel_initializer='normal'))
model.compile(loss=params['losses'],
optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
metrics=['acc'])
history = model.fit(X_train,
Y_train,
validation_data=[X_valid, Y_valid],
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=2)
# finally we have to make sure that history object and model are returned
return history, model
def iris_model(x_train, y_train, x_val, y_val, params):
model = Sequential()
model.add(
Dense(params['first_neuron'],
input_dim=x_train.shape[1],
activation='relu'))
model.add(Dropout(params['dropout']))
model.add(Dense(y_train.shape[1], activation=params['last_activation']))
model.compile(optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
loss=params['loss'],
metrics=['acc'])
out = model.fit(x_train,
y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=0,
validation_data=[x_val, y_val],
callbacks=early_stopper(params['epochs'], mode='strict'))
return out, model
def fake_news_model(x_train, y_train, x_val, y_val, params):
t = Tokenizer(num_words=params['max_words'])
t.fit_on_texts(x_train)
train_sequence = t.texts_to_sequences(x_train)
train_padded = pad_sequences(train_sequence, maxlen=params['max_length'])
model = Sequential()
model.add(
Embedding(params['max_words'], 64, input_length=params['max_length']))
model.add(Dropout(params['dropout']))
model.add(LSTM(64))
model.add(Dense(256, activation=params['activation']))
model.add(Dropout(params['dropout']))
model.add(Dense(1, activation=params['last_activation']))
model.compile(loss=params['losses'],
optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
metrics=['accuracy'])
model.summary()
history = model.fit(
train_padded,
y_train,
batch_size=params['batch_size'],
epochs=150,
validation_split=0.1,
callbacks=[EarlyStopping(monitor='val_loss', min_delta=0.00001)],
verbose=2)
return history, model
def build_model(X_train, Y_train, X_val, Y_val, params):
model = Sequential()
model.add(
Embedding(params['vocab_size'],
params['e_size'],
input_length=params['seq_len']))
model.add(Conv1D(32, 7, activation='relu'))
model.add(MaxPooling1D(5))
model.add(Conv1D(32, 7, activation='relu'))
model.add(GlobalAveragePooling1D())
model.add(Dropout(params['dropout']))
hidden_layers(model, params, 1)
model.add(Dense(1, activation=params['last_activation']))
## COMPILE
model.compile(optimizer=params['optimizer'](lr_normalizer(
params['lr'], params['optimizer'])),
loss='binary_crossentropy',
metrics=['acc'])
out = model.fit(X_train,
Y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
validation_data=[X_val, Y_val],
verbose=2)
return out, model
def dl_model(x_train, y_train, x_val, y_val, params):
model = Sequential()
model.add(Conv2D(params['filters_first_conv'], kernel_size=(12,7), strides=(7,4),activation='relu', input_shape=(num_frames,num_feats,1)))
model.add(Dropout(params['dropout']))
model.add(Conv2D(params['filters_second_conv'], kernel_size=5,strides=(2,2),activation='relu', input_shape=(num_frames,num_feats,1)))
model.add(Dropout(params['dropout']))
model.add(Flatten())
model.add(Dense(num_classes, activation='softmax'))
model.compile(optimizer=params['optimizer'](lr=lr_normalizer(params['lr'], params['optimizer'])),
loss=params['loss'],
metrics=['acc'])
#Class weights to weigh up for imbalanced validation set
class_weight = {0: 2,
1: 1.5,
2: 2,
3: 1.0}
out = model.fit(x_train, y_train,
validation_data=[x_val, y_val],
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=0,
class_weight=class_weight)
return out, model
def get_model(self, x_train, y_train, x_val, y_val, params):
model = models.Sequential()
# Input layer with dropout
model.add(
layers.Dense(params['first_neuron'],
activation=params['activation'],
input_shape=(self.n_features_all, )))
model.add(layers.Dropout(params['dropout']))
# Hidden layers with dropout
for i in range(params['hidden_layers']):
model.add(
layers.Dense(params['hidden_neuron'],
activation=params['activation']))
model.add(layers.Dropout(params['dropout']))
# Output layer
model.add(layers.Dense(5, activation=params['last_activation']))
# Build model
model.compile(params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
loss='categorical_crossentropy',
metrics=['accuracy', self.dr, self.far])
history = model.fit(x_train,
y_train,
validation_data=(x_val, y_val),
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=0)
return history, model
def hbb_model(train_data,train_y,valid_data,valid_y,param):
model=Sequential()
model.add(Dense(param['first_neuron'],input_dim=train_data.shape[1],activation=param['activation'],kernel_initializer='normal'))
model.add(Dropout(param['dropout']))
hidden_layers(model,param,1)
model.add(Dense(train_y.shape[1],activation=param['last_activation'],kernel_initializer='normal'))
model.compile(optimizer=param['optimizer'](lr=lr_normalizer(param['lr'],param['optimizer'])),loss=param['losses'],metrics=['acc'])
out=model.fit(train_data,train_y,batch_size=param['batch_size'],epochs=param['epochs'],verbose=0,validation_data=[valid_data,valid_y])
return out,model
def test_lr_normalizer():
'''Test learning rate normalizer to confirm an invalid type is
recognized and throws TalosModelError.'''
from talos.model.normalizers import lr_normalizer
from talos.utils.exceptions import TalosModelError
print('Testing lr_normalizer() and invalid optimizer type...')
# Using string as proxy for any invalid class
# (ex., tensorflow-sourced optimizer)
bad_optimizer = 'test'
try:
lr_normalizer(1, bad_optimizer)
except TalosModelError:
print('Invalid model optimizer caught successfully!')
else:
print('Invalid (string) model optimizer type not caught.')
def make_opt_instance(opt, lr_scale):
# Turn the optimizer into a class if necessary
opt_trans = {
'sgd': SGD,
'rmsprop': RMSprop,
'adagrad': Adagrad,
'adadelta': Adadelta,
'adam': Adam,
'adamax': Adamax,
'nadam': Nadam,
}
_opt = opt_trans[opt.lower()] if isinstance(opt, str) else opt
return _opt(lr=lr_normalizer(float(lr_scale), _opt))
def cnn_model2(x_train, y_trainHot, x_val, y_val, hyperP):
INPUT_SHAPE = (80, 80, 3)
num_kernels = hyperP['num_kernels']
batch_size = hyperP['batch_size']
cnn_layers = hyperP['cnn_layers']
drop_out = hyperP['drop_out']
optimizer = hyperP['optimizer']
learning_rate = hyperP['learning_rate']
OPTIMIZER = optimizer(lr=lr_normalizer(learning_rate, optimizer))
kernel_initializer = hyperP['kernel_initializer']
activation = hyperP['activation']
model = Sequential()
model.add(
Conv2D(num_kernels,
kernel_size=(3, 3),
kernel_initializer=kernel_initializer,
input_shape=INPUT_SHAPE,
data_format='channels_last'))
model.add(Activation(activation))
model.add(MaxPool2D(pool_size=(2, 2)))
#model.add(Conv2D(num_kernels, kernel_size=(3,3), kernel_initializer= kernel_initializer, input_shape=INPUT_SHAPE, data_format='channels_last'))
#model.add(Activation(activation))
for i in range(0, cnn_layers):
model.add(
Conv2D(num_kernels,
kernel_size=(3, 3),
kernel_initializer=kernel_initializer))
model.add(Activation(activation))
model.add(BatchNormalization())
model.add(Dropout(drop_out))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(drop_out))
model.add(Dense(5, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=OPTIMIZER,
metrics=['accuracy'])
early_stopping_monitor = EarlyStopping(monitor='val_loss', patience=5)
out = model.fit(x_train,
y_trainHot,
batch_size=batch_size,
epochs=10,
verbose=0,
validation_split=0.2,
callbacks=[early_stopping_monitor])
return out, model
def telos_baseline_model(x_train, y_train, x_val, y_val, params):
from keras.models import Sequential
from keras.layers import Dense, Dropout, GaussianNoise, BatchNormalization
from keras.constraints import max_norm
model = Sequential()
model.add(
Dropout(params['first_dropout'],
input_shape=(Options.InputFeatureSize, )))
model.add(
Dense(params['first_neuron'],
kernel_initializer='normal',
activation=params['activation']))
model.add(Dropout(params['second_dropout']))
#kernel_constraint=max_norm(3)
#model.add(BatchNormalization())
#rms = RMSprop(lr = 0.00050)
model.add(
Dense(1,
kernel_initializer='normal',
activation=params['last_activation']))
model.compile(
loss=params['losses'],
# here we add a regulizer normalization function from Talos
optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
metrics=['mae', 'mse'])
callbacks = []
if (Options.KerasEarlyStopping):
early_stop = keras.callbacks.EarlyStopping(
monitor='loss',
min_delta=0,
patience=Options.KerasEarlyStoppingPatience,
verbose=Options.KerasVerbose,
mode='auto')
callbacks.append(early_stop)
out = model.fit(x_train,
y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=Options.KerasVerbose,
callbacks=callbacks
#,validation_data=[x_val, y_val]
)
return out, model
def dae_model_hl(x_train, y_train, x_val, y_val, params):
#print(params['x_train_noise'].shape)
print(x_train.shape)
print("masking training")
x_train_noise = mask_function(dataframe=x_train,
noise=float(params['noise']),
batch_sizes=300) #masking training
print("masking validation")
x_val_noise = mask_function(dataframe=x_val,
noise=float(params['noise']),
batch_sizes=300) #masking validation
print("building autoencoder network")
model = Sequential()
model.add(
Dense(params['first_neuron'],
activation=params['activation'],
input_shape=(x_train.shape[1], )))
model.add(Dropout(params['dropout']))
#m.add(Dense(128, activation='elu'))
hidden_layers(model, params, 1)
model.add(
Dense(params['embedding_size'],
activation=params['activation'],
name="bottleneck"))
hidden_layers(model, params, 1)
model.add(Dense(params['first_neuron'], activation=params['activation']))
#m.add(Dense(512, activation='elu'))
model.add(Dropout(params['dropout']))
model.add(Dense(x_train.shape[1], activation=params['last_activation']))
#m.compile(loss='mean_squared_error', optimizer = params['optmizer'])
model.compile(optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
loss=params['loss'],
metrics=['accuracy'])
print("training neural network")
out = model.fit(
x_train,
x_train_noise, #x_train_noise,
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=0,
validation_data=[x_val, x_val_noise], #x_val_noise],
callbacks=early_stopper(params['epochs'], mode='moderate'))
#callbacks=early_stopper(params['epochs'], mode='strict'))#noisy_train, train, batch_size=128, epochs=params['epochs'], verbose=1,
return out, model
def iris_model(x_train, y_train, x_val, y_val, params):
model = Sequential()
model.add(Dense(32, input_dim=4, activation=params['activation']))
model.add(Dense(3, activation='softmax'))
model.compile(optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
loss=params['losses'])
# metrics=['acc'])
out = model.fit(x_train,
y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
validation_data=[x_val, y_val],
verbose=0)
return out, model
def deletion_model_aug(x_train, y_train, x_val, y_val, params):
if model_type == "VGG16":
base_model = VGG16(weights='imagenet', include_top=False)
elif model_type == "Res50":
base_model = ResNet50(weights='imagenet', include_top=False)
else:
base_model = InceptionV3(weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation='sigmoid')(x)
x = Dense(1, activation='sigmoid')(x)
model = Model(inputs=base_model.input, outputs=x)
# train the top added layers first, keep the VGG16 layers frozen
# we can relax this restriction after the last layer is trained
for layer in base_model.layers:
layer.trainable = False
model.compile(optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
loss=binary_crossentropy,
metrics=['accuracy', f1score, precision, recall])
# Perform Data Augmentation while training
batch_size = params['batch_size']
out = model.fit_generator(train_generator.flow(x_train,
y_train,
batch_size=batch_size,
shuffle=False),
steps_per_epoch=x_train.shape[0] // batch_size,
epochs=params['epochs'],
validation_data=val_generator.flow(
x_val,
y_val,
batch_size=batch_size,
shuffle=False),
validation_steps=x_val.shape[0] // batch_size,
class_weight=get_class_weights(y_train)
if params['class_weights'] else None,
callbacks=[
EarlyStopping(monitor="val_f1score",
patience=7,
min_delta=0.001,
mode='max')
])
return out, model
def import_model(X_train, y_train, X_val, y_val, params):
#Load model architecture
model = get_model(X_train,
y_train,
params['dense_one'],
params['dense_two'],
params['dense_three'],
params['activation'],
params['last_activation'],
initialize=True)
#Add multi-gpu support if available
if get_available_gpus() > 1:
model = multi_gpu(model)
#Compile model
model.compile(loss=params['losses'],
metrics=[
'mean_squared_error', 'mean_absolute_error',
'mean_absolute_percentage_error'
],
optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])))
#Set early stopping
earlystop = EarlyStopping(monitor='categorical_accuracy',
min_delta=0.1,
patience=9,
mode='auto')
#Train model
out = model.fit(X_train,
y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
validation_data=[X_val, y_val],
verbose=0,
callbacks=[earlystop])
return out, model
def emotions_model(dummyXtrain, dummyYtrain, dummyXval, dummyYval, params):
model = Sequential()
model.add(
Conv2D(32, (3, 3),
padding='same',
kernel_initializer=params['kernel_initializer']))
model.add(Activation(params['activation_1']))
model.add(Conv2D(params['neurons_layer_2'], (3, 3)))
model.add(Activation(params['activation_2']))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(params['dropout']))
model.add(Conv2D(params['neurons_layer_3'], (3, 3), padding='same'))
model.add(Activation(params['activation_3']))
model.add(Conv2D(params['neurons_layer_4'], (3, 3)))
model.add(Activation(params['activation_4']))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(params['dropout']))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation(params['activation_5']))
model.add(Dropout(params['dropout']))
model.add(Dense(7, activation=params['last_activation']))
model.compile(optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
loss=params['loss'],
metrics=['accuracy'])
history = model.fit(dummyXtrain,
dummyYtrain,
batch_size=params['batch_size'],
epochs=params['epochs'],
validation_data=(dummyXval, dummyYval),
callbacks=[
ModelCheckpoint("conv2d_mwilchek.hdf5",
monitor="val_loss",
save_best_only=True),
early_stopper(params['epochs'], mode='strict')
])
return history, model
def hyperparameter_Scan(self,normed_train_dataset, train_labels, test_data, test_labels, param):
#patience_model, nOutput, nNodes=16, actFn='relu', EPOCHS=100,learnRate=0.001,momentumN=0.0,lastActFn='linear',regCons=0.0,doRate=0.0):
input_layer = Input(shape=(len(normed_train_dataset.keys()),))
dense_1 = Dense(param['nNodes'], activation=param["activation"],activity_regularizer=regularizers.l2(param["regCons"]))(input_layer)
# dense_1 = Dropout(param["dropRate"])(dense_1)
dense_2 = Dense(param['nNodes'], activation=param["activation"],activity_regularizer=regularizers.l2(param["regCons"]))(dense_1)
dense_2 = Dropout(param["dropRate"])(dense_2)
dense_3 = Dense(param['nNodes'], activation=param["activation"], activity_regularizer=regularizers.l2(param["regCons"]))(dense_2)
# dense_3 = Dropout(param["dropRate"])(dense_3)
dense_4 = Dense(param['nNodes'], activation=param["activation"],activity_regularizer=regularizers.l2(param["regCons"]))(dense_3)
dense_4 = Dropout(param["dropRate"])(dense_4)
# separating outputs for density and GSD for custom loss function
output_1 = Dense(1,activation=param["last_activation"],name='Density', activity_regularizer=regularizers.l2(param["regCons"]))(dense_4)
# output_2 = Dense(8,activation='sigmoid',name='out2',activity_regularizer=regularizers.l1(regCons))(dense_4)
output_2 = Dense(8,activation=param["last_activation"],name='GSD', activity_regularizer=regularizers.l2(param["regCons"]))(dense_4)
train_labels_copy = pd.DataFrame.copy(train_labels)
label1 = train_labels['Granule_density']
labels = ['Bin1','Bin2','Bin3','Bin4','Bin5','Bin6','Bin7','Coarse']
# label2 = ['Bin1','Bin2','Bin3','Bin4','Bin5','Bin6','Bin7','Coarse']
label2 = pd.DataFrame([train_labels_copy.pop(i) for i in labels]).T
model = Model(inputs=[input_layer], outputs=[output_1,output_2])
model.compile(optimizer=param['optimizer'](lr=lr_normalizer(param['learningRate'],param['optimizer'])),loss=self.lossFunc_DensityStde(output_1), metrics = ['mae','mse'])
# model.compile(optimizer=Adam(learning_rate=0.0001,beta_1=0.1,beta_2=0.2,epsilon=0.001,amsgrad=True),loss=self.lossFunc_DensityStde(output_1), metrics = ['mae','mse'])
w1 = np.full(len(self.normed_train_dataset),1)
w2 = np.full(len(self.normed_train_dataset),1)
# print(model.summary())
test_label_copy = pd.DataFrame.copy(test_labels)
tlabel1 = test_labels['Granule_density']
tlabel2 = pd.DataFrame([test_label_copy.pop(i) for i in labels]).T
out = model.fit(normed_train_dataset, [label1, label2], epochs=param['epochs'],
verbose=0, validation_data=[test_data,[tlabel1,tlabel2]],sample_weight={'Density': w1, 'GSD':w2},use_multiprocessing=True)
# for layer in model.layers: print(layer.get_config(), layer.get_weights())
return out, model
def autoFNN(X_train, Y_train, X_val, Y_val, params):
model = Sequential()
model.add(
Embedding(params['vocab_size'],
params['e_size'],
input_length=params['seq_len']))
model.add(Flatten())
hidden_layers(model, params, 1)
model.add(Dense(1, activation=params['last_activation']))
model.compile(optimizer=params['optimizer'](lr_normalizer(
params['lr'], params['optimizer'])),
loss='binary_crossentropy',
metrics=['acc'])
out = model.fit(X_train,
Y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
validation_data=[X_val, Y_val],
verbose=2)
return out, model
def build_model(x_train, y_train, x_val, y_val, params):
model = keras.Sequential()
model.add(keras.layers.Dense(10, activation=params['activation'],
input_dim=x_train.shape[1],
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=keras.regularizers.l1_l2(l1=params['l1'], l2=params['l2']),
bias_regularizer=None))
model.add(keras.layers.Dropout(params['dropout']))
# If we want to also test for number of layers and shapes, that's possible
hidden_layers(model, params, 1)
# Then we finish again with completely standard Keras way
model.add(keras.layers.Dense(1, activation=params['activation'], use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=keras.regularizers.l1_l2(l1=params['l1'], l2=params['l2']),
bias_regularizer=None))
model.compile(optimizer=params['optimizer'](lr=lr_normalizer(params['lr'], params['optimizer'])),
loss=params['losses'],
metrics=['mse'])
history = model.fit(x_train, y_train,
validation_data=[x_val, y_val],
batch_size=params['batch_size'],
epochs=params['epochs'],
callbacks=[early_stopper(epochs=params['epochs'], mode='moderate')],
#callbacks=[early_stopper(epochs=params['epochs'], mode='strict')],
verbose=0)
# Finally we have to make sure that history object and model are returned
return history, model
def smallRNN(x_train, y_train, x_test, y_test, params):
#HIDDEN_UNITS = 5
model = Sequential()
model.add(
Bidirectional(LSTM(params["hidden_unit"],
activation='tanh',
inner_activation='sigmoid',
kernel_constraint=maxnorm(2),
kernel_initializer=KERNEL_INITIAL,
return_sequences=True,
dropout=0.3),
input_shape=x_train.shape[1:],
merge_mode='concat'))
#model.add(AveragePooling1D(pool_size=400,strides=400))
#model.add(Bidirectional(LSTM(HIDDEN_UNITS,activation='tanh',inner_activation='sigmoid',kernel_constraint = maxnorm(2),kernel_initializer=KERNEL_INITIAL,return_sequences=True),input_shape=x_train.shape[1:],merge_mode='concat'))
#model.add(AveragePooling1D(pool_size=400,strides=400))
model.add(Flatten())
model.add(Dense(1))
#a soft max classifier
model.add(Activation("sigmoid"))
#model.compile(loss=LOSS, optimizer = params["optimizer"](lr_normalizer(params["lr"], params['optimizer'])), metrics =METRICS)
filepath = "SmallRNN" + str(params["hidden_unit"]) + "best_smallRNN.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True)
early_stopping_monitor = EarlyStopping(monitor='val_loss', patience=8)
model.compile(loss=LOSS,
optimizer=params["optimizer"](lr_normalizer(
params["lr"], params['optimizer'])),
metrics=METRICS)
print(model.summary())
out = model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
verbose=VERBOSE,
validation_split=0.2,
callbacks=[checkpoint, early_stopping_monitor]
) #,roc_callback(training_data=(x_train, y_train),validation_data=(x_test, y_test))]) #,callbacks=callbacks_list)
## if you want early stopping
#Tuning = model.fit(Train_Predictors,Train_class,batch_size=BATCH_SIZE, epochs = NB_EPOCH, verbose = VERBOSE,
#validation_split = VALIDATION_SPLIT,callbacks=[early_stopping_monitor,checkpoint])
#finalModel = load_model(filepath)
#deepPredict(finalModel,Test_Predictors,Test_class)
return out, model #,roc_train,roc_test,acc_train,acc_test
def miniCNN_RNN(x_train, y_train, x_test, y_test, params):
model = Sequential()
model.add(
Conv1D(params["num_kernel"],
kernel_size=params["kernel_size"],
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2),
input_shape=x_train.shape[1:3]))
model.add(Activation("relu"))
model.add(MaxPooling1D(pool_size=10, strides=10))
#model.add(GlobalMaxPooling1D())
#model.add(Flatten())
#model.add(Reshape((50,1))) # shape becomes (batch_size,200,1)
model.add(
Bidirectional(
LSTM(params["hidden_unit"],
activation='tanh',
inner_activation='sigmoid',
kernel_constraint=maxnorm(2),
kernel_initializer=KERNEL_INITIAL,
return_sequences=True,
dropout=0.3)))
#model.add(GlobalMaxPooling1D())
model.add(Flatten())
#model.add(Dense(25))
#model.add(Activation("relu"))
#model.add(GlobalMaxPooling1D())
model.add(Dense(1))
#a soft max classifier
model.add(Activation("sigmoid"))
model.compile(loss=LOSS, optimizer=OPTIMIZER, metrics=METRICS)
#filepath="largeRNN_dropout_"+str(DROP_OUT)+"_{epoch:02d}_{val_acc:.2f}.hdf5"
filepath = "best_smallCNN_RNN_dropout_" + str(
params["hidden_unit"]) + ".hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='max')
early_stopping_monitor = EarlyStopping(monitor='val_loss', patience=8)
#callbacks_list = [checkpoint]
model.compile(loss=LOSS,
optimizer=params["optimizer"](lr_normalizer(
params["lr"], params['optimizer'])),
metrics=METRICS)
print(model.summary())
#Tuning = model.fit(x_train,y_train,batch_size=BATCH_SIZE, epochs = NB_EPOCH, verbose = VERBOSE,
# validation_split = 0.2) #(x_test,y_test),callbacks=[checkpoint,early_stopping_monitor,roc_callback(training_data=(x_train, y_train),validation_data=(x_test, y_test))]) #,callbacks=callbacks_list)
out = model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
verbose=VERBOSE,
validation_split=VALIDATION_SPLIT,
callbacks=[checkpoint, early_stopping_monitor]
) #,roc_callback(training_data=(x_train, y_train),validation_data=(x_test, y_test))]) #,callbacks=callbacks_list)
return out, model
def _create_input_model(self, x_train, y_train, x_val, y_val, params):
import wrangle as wr
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dropout, Flatten
from tensorflow.keras.layers import LSTM, Conv1D, SimpleRNN, Dense, Bidirectional
model = Sequential()
if params['network'] != 'dense':
x_train = wr.array_reshape_conv1d(x_train)
x_val = wr.array_reshape_conv1d(x_val)
if params['network'] == 'conv1d':
model.add(Conv1D(params['first_neuron'], x_train.shape[1]))
model.add(Flatten())
elif params['network'] == 'lstm':
model.add(LSTM(params['first_neuron']))
if params['network'] == 'bidirectional_lstm':
model.add(Bidirectional(LSTM(params['first_neuron'])))
elif params['network'] == 'simplernn':
model.add(SimpleRNN(params['first_neuron']))
elif params['network'] == 'dense':
model.add(
Dense(params['first_neuron'],
input_dim=x_train.shape[1],
activation='relu',
kernel_initializer=params['kernel_initializer']))
model.add(Dropout(params['dropout']))
# add hidden layers to the model
from talos.model.hidden_layers import hidden_layers
hidden_layers(model, params, 1)
# get the right activation and last_neuron based on task
from talos.model.output_layer import output_layer
activation, last_neuron = output_layer(self.task,
params['last_activation'],
y_train, y_val)
model.add(
Dense(last_neuron,
activation=activation,
kernel_initializer=params['kernel_initializer']))
# bundle the optimizer with learning rate changes
from talos.model.normalizers import lr_normalizer
optimizer = params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer']))
# compile the model
model.compile(optimizer=optimizer,
loss=params['losses'],
metrics=self.metrics)
# fit the model
out = model.fit(
x_train,
y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=0,
callbacks=[self.callback(self.experiment_name, params)],
validation_data=[x_val, y_val])
# pass the output to Talos
return out, model
def largeRNN(x_train, y_train, x_test, y_test, params):
HIDDEN_UNITS = 20
model = Sequential()
model.add(
LSTM(params["hidden_unit"],
activation='tanh',
inner_activation='sigmoid',
kernel_constraint=maxnorm(2),
kernel_initializer=KERNEL_INITIAL,
return_sequences=True,
dropout=0.3,
input_shape=x_train.shape[1:]))
model.add(MaxPooling1D(pool_size=10, strides=5))
model.add(
LSTM(params["hidden_unit"],
activation='tanh',
inner_activation='sigmoid',
kernel_constraint=maxnorm(2),
kernel_initializer=KERNEL_INITIAL,
return_sequences=True,
dropout=0.3))
model.add(
LSTM(params["hidden_unit"],
activation='tanh',
inner_activation='sigmoid',
kernel_constraint=maxnorm(2),
kernel_initializer=KERNEL_INITIAL,
return_sequences=True,
input_shape=x_train.shape[1:]))
model.add(MaxPooling1D(pool_size=10, strides=5))
model.add(
LSTM(params["hidden_unit"],
activation='tanh',
inner_activation='sigmoid',
kernel_constraint=maxnorm(2),
kernel_initializer=KERNEL_INITIAL,
return_sequences=True))
model.add(Flatten())
model.add(Dense(1))
#a soft max classifier
model.add(Activation("sigmoid"))
model.compile(loss=LOSS,
optimizer=params["optimizer"](lr_normalizer(
params["lr"], params['optimizer'])),
metrics=METRICS)
#filepath="largeRNN_dropout_"+str(DROP_OUT)+"_{epoch:02d}_{val_acc:.2f}.hdf5"
filepath = "largeRNN" + "best_largeRNN_dropout_" + str(
params["hidden_unit"]) + ".hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='max')
early_stopping_monitor = EarlyStopping(monitor='val_loss', patience=8)
#callbacks_list = [checkpoint]
#model.compile(loss=LOSS, optimizer = 'Adam', metrics =METRICS)
print(model.summary())
#Tuning = model.fit(x_train,y_train,batch_size=BATCH_SIZE, epochs = NB_EPOCH, verbose = VERBOSE,
# validation_split = 0.2) #(x_test,y_test),callbacks=[checkpoint,early_stopping_monitor,roc_callback(training_data=(x_train, y_train),validation_data=(x_test, y_test))]) #,callbacks=callbacks_list)
out = model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
verbose=VERBOSE,
validation_data=(x_test, y_test),
callbacks=[checkpoint, early_stopping_monitor]
) #,roc_callback(training_data=(x_train, y_train),validation_data=(x_test, y_test))]) #,callbacks=callbacks_list)
print(model.summary())
return out, model
def verylargeCNN(x_train, y_train, x_test, y_test, params):
model = Sequential()
model.add(
Conv1D(params["num_kernel"],
kernel_size=params["kernel_size"],
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2),
input_shape=x_train.shape[1:3]))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.3))
model.add(MaxPooling1D())
model.add(
Conv1D(params["num_kernel"],
kernel_size=params["kernel_size"],
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(MaxPooling1D())
model.add(
Conv1D(params["num_kernel"],
kernel_size=params["kernel_size"],
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(MaxPooling1D())
model.add(
Conv1D(params["num_kernel"],
kernel_size=params["kernel_size"],
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Conv1D(params["num_kernel"],
kernel_size=params["kernel_size"],
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Conv1D(params["num_kernel"],
kernel_size=params["kernel_size"],
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Conv1D(params["num_kernel"],
kernel_size=params["kernel_size"],
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Conv1D(params["num_kernel"],
kernel_size=params["kernel_size"],
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(GlobalMaxPooling1D())
model.add(Dense(1))
model.add(Activation("sigmoid"))
filepath = "VerylargeCNN" + "best_veryLargeCNN_dropout_" + str(
params["kernel_size"]) + ".hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto')
early_stopping_monitor = EarlyStopping(monitor='val_loss', patience=8)
model.compile(loss=LOSS,
optimizer=params["optimizer"](lr_normalizer(
params["lr"], params['optimizer'])),
metrics=METRICS)
print(model.summary())
out = model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
verbose=VERBOSE,
validation_split=0.2,
callbacks=[checkpoint, early_stopping_monitor]
) #,roc_callback(training_data=(x_train, y_train),validation_data=(x_test, y_test))]) #,callbacks=callbacks_list)
#finalmodel = load_model(filepath)
return out, model #,roc_train,roc_test,acc_train,acc_test
def _create_input_model(self, x_train, y_train, x_val, y_val, params):
model = Sequential()
if params['network'] != 'dense':
x_train = array_reshape_conv1d(x_train)
x_val = array_reshape_conv1d(x_val)
if params['network'] == 'conv1d':
model.add(Conv1D(params['first_neuron'], x_train.shape[1]))
model.add(Flatten())
elif params['network'] == 'lstm':
model.add(LSTM(params['first_neuron']))
if params['network'] == 'bidirectional_lstm':
model.add(Bidirectional(LSTM(params['first_neuron'])))
elif params['network'] == 'simplernn':
model.add(SimpleRNN(params['first_neuron']))
elif params['network'] == 'dense':
model.add(Dense(params['first_neuron'],
input_dim=x_train.shape[1],
activation='relu'))
model.add(Dropout(params['dropout']))
# add hidden layers to the model
hidden_layers(model, params, 1)
# output layer (this is scetchy)
try:
last_neuron = y_train.shape[1]
except IndexError:
if len(np.unique(y_train)) == 2:
last_neuron = 1
else:
last_neuron = len(np.unique(y_train))
model.add(Dense(last_neuron,
activation=params['last_activation']))
# bundle the optimizer with learning rate changes
optimizer = params['optimizer'](lr=lr_normalizer(params['lr'],
params['optimizer']))
# compile the model
model.compile(optimizer=optimizer,
loss=params['losses'],
metrics=['acc'])
# fit the model
out = model.fit(x_train, y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=0,
validation_data=[x_val, y_val])
# pass the output to Talos
return out, model
