def create_model(input_size, output_size, n_layers, n_neurons,
activation_function, learning_rate, dropout_rate, optimizer):
model = models.Sequential()
model.add(
layers.Dense(n_neurons,
input_shape=(input_size, ),
name='new_androdet_dense_1'))
for _ in range(n_layers):
if dropout_rate != 0.0:
model.add(layers.Dropout(dropout_rate, noise_shape=None,
seed=None))
model.add(layers.Dense(n_neurons, activation=activation_function))
model.add(layers.Dense(output_size, activation="sigmoid"))
#model.summary()
if optimizer == 'rmsprop':
opt = optimizers.rmsprop(lr=learning_rate)
elif optimizer == 'adam':
opt = optimizers.adam(lr=learning_rate)
elif optimizer == 'sgd':
opt = optimizers.sgd(lr=learning_rate)
elif optimizer == 'adagrad':
opt = optimizers.adagrad(lr=learning_rate)
elif optimizer == 'adadelta':
opt = optimizers.adadelta(lr=learning_rate)
elif optimizer == 'adamax':
opt = optimizers.adamax(lr=learning_rate)
elif optimizer == 'nadam':
opt = optimizers.nadam(lr=learning_rate)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=["mean_squared_error"])
return model
def MS_build_multivar_cnnlstm_modelB(config, X, y, val_X, val_y):
# define the input cnn model
model = Sequential()
model.add(
TimeDistributed(Conv1D(64, 7, activation='relu'),
input_shape=(None, int((22 / 2)), 8)))
model.add(TimeDistributed(Dropout(0.2)))
model.add(TimeDistributed(Conv1D(128, 4, activation='relu')))
model.add(TimeDistributed(MaxPooling1D()))
model.add(TimeDistributed(Conv1D(32, 1, activation='relu')))
model.add(TimeDistributed(Conv1D(64, 1, activation='relu')))
model.add(TimeDistributed(MaxPooling1D((1))))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(100, activation='sigmoid'))
model.add(Dense(100, activation='relu'))
model.add(Dense(37))
custom_adamax = opt.adamax(lr=4e-1)
custom_sgd = SGD(lr=4e-1, momentum=0.9)
clr = CyclicLR(mode='triangular2')
model.compile(loss='mae', optimizer='adamax', metrics=['mae'])
# fit network
model.fit(X,
y,
epochs=35,
batch_size=7,
verbose=2,
validation_data=(val_X, val_y),
callbacks=[clr])
return model
def __init__(self, n_features, anomaly_class):
#input
input_layer = Input(shape=(n_features, ))
#encode
encoder = Dense(8, kernel_initializer='he_uniform',
activation='relu')(input_layer)
drop = Dropout(rate=0.2)(encoder)
#latent
latent = Dense(2, kernel_initializer='he_uniform',
activation='relu')(drop)
drop = Dropout(rate=0.2)(latent)
#decode
decoder = Dense(8, kernel_initializer='he_uniform',
activation='relu')(drop)
drop = Dropout(rate=0.2)(decoder)
#output
output_layer = Dense(n_features, activation="sigmoid")(drop)
self.model = Model(inputs=input_layer, outputs=output_layer)
#compile
self.model.compile(loss='mean_squared_error', optimizer=adamax(lr=0.1))
#training
self.history = None
#name
self.model_name = "A2"
#anomaly class
self.anomaly_class = anomaly_class
def getOptimizer(name, rate):
if name is 'adamax':
return adamax(lr=rate)
elif name is 'adam':
return adam(lr=rate)
elif name is 'nadam':
return Nadam(lr=rate)
else:
return RMSprop(lr=rate)
def getOptimizer(name,rate):
if name == 'adamax':
return adamax(lr=rate)
elif name == 'adam':
return adam(lr=rate)
elif name == 'nadam':
return Nadam(lr=rate)
elif name == 'sgd':
return SGD(lr=rate, momentum=0.9, nesterov=True)
else:
return RMSprop(lr=rate)
def _main(model_name):
# data_sets path
train_dir = '/home/lst/datasets/cifar-10-images_train/'
val_dir = '/home/lst/datasets/cifar-10-images_test/'
# model
model = GoogLeNet(input_shape=(224, 224, 3))
# parallel model
parallel_model = multi_gpu_model(model, gpus=2)
# optimizers setting
from keras import optimizers
optimizer = optimizers.adamax(lr=0.002, decay=1e-06)
parallel_model.compile(loss=[
'categorical_crossentropy', 'categorical_crossentropy',
'categorical_crossentropy'
],
loss_weights=[1, 0.3, 0.3],
optimizer=optimizer,
metrics=["accuracy"])
# load data by batch
train_generator, validation_generator, num_train, num_val = load_batch_data(
train_dir, val_dir)
# Callbacks settings
from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard
early_stop = EarlyStopping(monitor='val_loss',
min_delta=0.001,
patience=30,
mode='min',
verbose=1)
checkpoint = ModelCheckpoint(f'{model_name}.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
tensorboard = TensorBoard(log_dir=f'./logs/{model_name}',
histogram_freq=0,
write_graph=True,
write_images=False)
# fit
parallel_model.fit_generator(
train_generator,
validation_data=validation_generator,
steps_per_epoch=math.ceil(num_train / batch_size),
validation_steps=math.ceil(num_val / batch_size),
epochs=epochs,
callbacks=[tensorboard, early_stop, checkpoint],
)
def adammax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0):
"""
Adam optimizer
:param lr: >=0, learning rate
:param beta_1: 0 < beta_1 < 1, generally close to 1
:param beta_2: 0 < beta_2 < 1, generally close to 1
:param epsilon: >=0, fuzz factor. If None, defaults to K.epsilon()
:param decay: >=0, learning rate decay over each update
"""
return optimizers.adamax(lr=lr,
beta_1=beta_1,
beta_2=beta_2,
epsilon=epsilon,
decay=decay)
def DropoutAdamax(self):
# Build the model
print("Building model...")
model = Sequential()
# Single 500-neuron hidden layer with sigmoid activation
model.add(Dropout(rate=0.4, input_shape=(self.nb_features,)))
model.add(Dense(input_dim=self.nb_features, units=int(self.nb_features / 0.4), activation='relu'))
model.add(Dropout(rate=0.4, input_shape=(int(self.nb_features / 0.4),)))
# Output layer with softmax activation
model.add(Dense(units=self.nb_classes, activation='softmax'))
# Specify optimizer, loss and validation metric
opt = adamax(lr=0.0015)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
self.model = model
def create_model(layers_and_filters,
kernels,
activation,
input_shape,
dropout_rate,
optimizer,
learning_rate,
output_size=1):
model = models.Sequential()
i = 0
for filters in layers_and_filters:
model.add(
layers.Conv2D(filters,
kernel_size=kernels[i],
strides=kernels[i],
activation=activation,
input_shape=input_shape))
i += 1
if i < len(layers_and_filters):
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
model.add(layers.BatchNormalization())
if dropout_rate != 0:
model.add(layers.Dropout(dropout_rate))
model.add(layers.Flatten())
model.add(layers.Dense(output_size, activation='sigmoid'))
if optimizer == 'rmsprop':
opt = optimizers.rmsprop(lr=learning_rate)
elif optimizer == 'adam':
opt = optimizers.adam(lr=learning_rate)
elif optimizer == 'sgd':
opt = optimizers.sgd(lr=learning_rate)
elif optimizer == 'adagrad':
opt = optimizers.adagrad(lr=learning_rate)
elif optimizer == 'adadelta':
opt = optimizers.adadelta(lr=learning_rate)
elif optimizer == 'adamax':
opt = optimizers.adamax(lr=learning_rate)
elif optimizer == 'nadam':
opt = optimizers.nadam(lr=learning_rate)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=["mean_squared_error"])
#model.summary()
return model
def __init__(self, n_features):
# Input
input_layer = Input(shape=(n_features, ))
# Hidden layers
layer = Dense(32, kernel_initializer='uniform',
activation="relu")(input_layer)
drop = Dropout(rate=0.2)(layer)
layer = Dense(64, kernel_initializer='he_uniform',
activation="relu")(input_layer)
drop = Dropout(rate=0.2)(layer)
layer = Dense(128, kernel_initializer='he_uniform',
activation="relu")(input_layer)
drop = Dropout(rate=0.2)(layer)
layer = Dense(256, kernel_initializer='he_uniform',
activation="relu")(input_layer)
drop = Dropout(rate=0.2)(layer)
layer = Dense(128, kernel_initializer='he_uniform',
activation="relu")(input_layer)
drop = Dropout(rate=0.2)(layer)
layer = Dense(64, kernel_initializer='he_uniform',
activation="relu")(input_layer)
drop = Dropout(rate=0.2)(layer)
layer = Dense(32, kernel_initializer='he_uniform',
activation="relu")(input_layer)
drop = Dropout(rate=0.2)(layer)
# Output
output_layer = Dense(1,
kernel_initializer="he_uniform",
activation='sigmoid')(drop)
# Model
self.model = Model(inputs=input_layer, outputs=output_layer)
# Compile
self.model.compile(optimizer=adamax(0.05), loss='binary_crossentropy')
# Training
self.history = None
# Name
self.model_name = "BC4"
def model():
model = Sequential()
model.add(Conv2D(16, (3, 3), input_shape=(32, 32, 3), padding="same"))
model.add(LeakyReLU(0.1))
model.add(Conv2D(32, (3, 3), padding="same"))
model.add(LeakyReLU(0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(32, (3, 3), padding="same"))
model.add(LeakyReLU(0.1))
model.add(Conv2D(64, (3, 3), padding="same"))
model.add(LeakyReLU(0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(256))
model.add(LeakyReLU(0.1))
model.add(Dropout(0.5))
model.add(Dense(NUM_CLASSES))
model.add(Activation("softmax"))
model.compile(
loss='categorical_crossentropy', # we train 10-way classification
optimizer=adamax(lr=INIT_LR), # for SGD
metrics=['accuracy'] # report accuracy during training
)
return model
def initialize_optimizer(optimizer_name: str, learning_rate: float, beta1: float, beta2: float,
lr_decay: float, rho: float, fuzz: float, momentum: float) \
-> Union[adam, rmsprop, sgd, adagrad, adadelta, adamax]:
"""
Initializes an optimizer based on the user's choices.
:param optimizer_name: the optimizer's name.
Can be one of 'adam', 'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adamax'.
:param learning_rate: the optimizer's learning_rate
:param beta1: the optimizer's beta1
:param beta2: the optimizer's beta2
:param lr_decay: the optimizer's lr_decay
:param rho: the optimizer's rho
:param fuzz: the optimizer's fuzz
:param momentum: the optimizer's momentum
:return: the optimizer.
"""
if optimizer_name == 'adam':
return adam(lr=learning_rate,
beta_1=beta1,
beta_2=beta2,
decay=lr_decay)
elif optimizer_name == 'rmsprop':
return rmsprop(lr=learning_rate, rho=rho, epsilon=fuzz)
elif optimizer_name == 'sgd':
return sgd(lr=learning_rate, momentum=momentum, decay=lr_decay)
elif optimizer_name == 'adagrad':
return adagrad(lr=learning_rate, decay=lr_decay)
elif optimizer_name == 'adadelta':
return adadelta(lr=learning_rate, rho=rho, decay=lr_decay)
elif optimizer_name == 'adamax':
return adamax(lr=learning_rate,
beta_1=beta1,
beta_2=beta2,
decay=lr_decay)
else:
raise ValueError('An unexpected optimizer name has been encountered.')
def create_optimizer_instance(self, **d):
return optimizers.adamax(**d)
def cnn(self,
dic={
'act': 'tanh',
'init': 'lecun_normal'
},
number={
'kernel': n,
'pool': 100
},
learate=0.01,
name='Default',
ndim=256):
gc.collect()
act = dic['act']
init = dic['init']
kernel = number['kernel']
pool = number['pool']
model = Sequential()
model.add(
Convolution1D(ndim,
kernel_size=(kernel),
input_shape=(self.x_train.shape[1], 1),
padding='same',
kernel_initializer=init,
kernel_regularizer=regularizers.l1_l2(l1=0.01,
l2=0.01)))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(MaxPooling1D(pool_size=(pool), padding='same'))
model.add(Activation(act))
model.add(BatchNormalization())
model.add(Flatten())
model.add(GaussianNoise(0.1))
model.add(Dense(int(ndim / 2)))
model.add(Activation(act))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Dense(int(ndim / 2)))
model.add(Activation(act))
model.add(BatchNormalization())
model.add(Dropout(0.2))
model.add(Dense(int(ndim / 4)))
model.add(Activation(act))
model.add(BatchNormalization())
model.add(Dropout(0.2))
model.add(Dense(int(ndim / 4)))
model.add(Activation(act))
model.add(BatchNormalization())
model.add(Dense(1, activation='sigmoid'))
auc_roc = self.as_keras_metric(tf.metrics.auc)
op = optimizers.adamax(lr=learate)
pr = self.as_keras_metric(tf.metrics.precision)
rec = self.as_keras_metric(tf.metrics.recall)
model.compile(loss='binary_crossentropy',
optimizer=op,
metrics=[auc_roc, rec, pr])
# es = EarlyStopping(monitor='auc',patience=9, verbose=1,mode='max')
# lr = ReduceLROnPlateau(monitor='auc', factor=0.1, patience=3, verbose=1, mode='max', min_delta=0.0001,cooldown=0, min_lr=0)
# history = model.fit(self.x_train, self.y_train, epochs=30, batch_size=250, callbacks=[es, lr], validation_split=0.1,shuffle=True)
# y_pre = model.predict_classes(self.x_test,batch_size=250)
# score = model.evaluate(self.x_test,self.y_test,batch_size=250)
# f1 = (2*score[3]*score[2])/(score[3]+score[2])
# plt.figure(figsize=(12, 6))
# plt.subplot(121)
# plt.plot(history.history['auc'])
# plt.plot(history.history['val_auc'])
# plt.title('model auc')
# plt.ylabel('accuracy')
# plt.xlabel('epoch')
# plt.suptitle('CNN:The Over Sampling is %s'%name)
# plt.legend(['train', 'validation'], loc='upper left')
# plt.subplot(122)
# plt.plot(history.history['precision'])
# plt.plot(history.history['val_precision'])
# plt.title('model precision')
# plt.ylabel('precision')
# plt.xlabel('epoch')
# plt.legend(['train', 'validation'], loc='upper left')
# plt.savefig('E:\\OneDrive - Georgia State University\\Kaggle Home Equity\\Ping\\Reports\\DeepLearning\\CNN\\%s.png'%name)
# plt.show()
# print('The test AUC is ',score[1],'The recall is',score[2],'The precision is',score[3])
# print('F1 Score is',f1)
return model
def bpmodel(self,
dic={
'act': 'sigmoid',
'init': 'lecun_normal'
},
nl={'node': 360},
learate=0.1,
name='Default'):
gc.collect()
act = dic['act']
init = dic['init']
number = nl['node']
model = Sequential()
model.add(
Dense(number,
input_dim=self.dim,
kernel_initializer=init,
kernel_regularizer=regularizers.l1_l2(l1=0.005, l2=0.005)))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dense(number * 10, kernel_initializer=init))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dense(number * 8, kernel_initializer=init))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
model.add(Dense(number * 6, kernel_initializer=init))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dense(number * 4, kernel_initializer=init))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
model.add(Dense(number * 2, kernel_initializer=init))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dense(number, kernel_initializer=init))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dense(int(number / 2), kernel_initializer=init))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
model.add(Dense(int(number / 4), kernel_initializer=init))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dense(int(number / 6), kernel_initializer=init))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dense(int(number / 12), kernel_initializer=init))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dense(1, activation='sigmoid', kernel_initializer=init))
auc_roc = self.as_keras_metric(tf.metrics.auc)
op = optimizers.adamax(lr=learate)
pr = self.as_keras_metric(tf.metrics.precision)
rec = self.as_keras_metric(tf.metrics.recall)
model.compile(loss='binary_crossentropy',
optimizer=op,
metrics=[auc_roc, rec, pr])
model.save(
'E:\OneDrive - Georgia State University\Kaggle Home Equity\Ping\Code\\bpmodel.h5'
)
# es = EarlyStopping(monitor='auc',patience=9, verbose=1,mode='max')
# lr = ReduceLROnPlateau(monitor='auc', factor=0.1, patience=3, verbose=1, mode='max', min_delta=0.0001,cooldown=0, min_lr=0)
# history = model.fit(self.x_train, self.y_train,epochs=50,batch_size=1000,callbacks=[es,lr],validation_split=0.1)
# y_pre = model.predict(self.x_test,batch_size=1000)
# y_pre1 = model.predict_classes(self.x_test,batch_size=1000)
# score = model.evaluate(self.x_test,self.y_test,batch_size=1000)
# f1 = (2*score[3]*score[2])/(score[3]+score[2])
# plt.figure(figsize=(12,6))
# plt.subplot(121)
# plt.plot(history.history['auc'])
# plt.plot(history.history['val_auc'])
# plt.title('model auc')
# plt.ylabel('auc')
# plt.xlabel('epoch')
# plt.suptitle('BP: The Over Sampling is %s'%name)
# plt.legend(['train', 'validation'], loc='upper left')
# plt.subplot(122)
# plt.plot(history.history['precision'])
# plt.plot(history.history['val_precision'])
# plt.title('model precision')
# plt.ylabel('precision')
# plt.xlabel('epoch')
# plt.legend(['train', 'validation'], loc='upper left')
# plt.savefig('E:\\OneDrive - Georgia State University\\Kaggle Home Equity\\Ping\\Reports\\DeepLearning\\BP\\%s.png'%name)
# plt.show()
# print('The test AUC is ',score[1],'The recall is',score[2],'The precision is',score[3])
# print('F1 Score is',f1)
return model
def trainTestLSTM(xTraining, xTesting, yTraining, yTesting, numCls, trainEpoch,
batchSz, labelLst, losFnc, optim, learnRate, featMode,
clsVer):
gc.collect()
clearGPU()
utils2.myPrint('---LSTM Classifier---')
trainShape = np.shape(xTraining)
testShape = np.shape(xTesting)
utils2.myPrint('Train Batch:', trainShape)
utils2.myPrint('Test Batch:', testShape)
models = list()
for version in clsVer:
# create the model
model = Sequential()
# todo - ai : possible variations
# try lstm decay
# try clipnorm and clipvalue
# try convlstm2d and/or concatanate two models
# try sgd instead of adam optimizer
if 'Specto' != featMode: # do not convolve for spectogram, since it is not as long as other modes.
utils2.myPrint('Classifier Version:', version)
if 0 == version:
# inputs 1 # 300 epoch .3924.
model.add(Conv1D(8, 48, strides=48,
input_shape=trainShape[1:]))
model.add(Activation('relu'))
model.add(Conv1D(16, 24, strides=24))
model.add(Activation('sigmoid'))
model.add(LSTM(24, return_sequences=True))
model.add(LSTM(12, return_sequences=False))
model.add(Dense(numCls, activation='softmax'))
elif 1 == version:
model.add(
Conv1D(8,
48,
strides=12,
activation='relu',
input_shape=trainShape[1:]))
model.add(Conv1D(16, 36, strides=6, activation='relu'))
model.add(Conv1D(32, 24, strides=2, activation='relu'))
model.add(Conv1D(64, 24, strides=2, activation='relu'))
model.add(LSTM(64, return_sequences=True))
model.add(LSTM(32, activation='relu', return_sequences=False))
model.add(Dense(numCls, activation='softmax'))
elif 2 == version:
# resulted better than LSTM variant(1 == clsVer) with following configuration
# {'inputFolder': 'D:/atili/MMIExt/Audacity/METU Recordings/Dataset/4spkr5post/', 'featureMode': 'Mags',
# 'channelMode': '0', 'classificationMode': 'Speaker', 'trainingEpoch': 200, 'stepSize': 0,
# 'sampRate': 48, 'batchSize': 32, 'lengthCut': 600, 'learningRate': 0.001,
# 'lossFunction': 'CatCrosEnt', 'optimizer': 'Adam', 'clsModel': 'LSTM', 'clsVersion': 2}
# overfitted with 1.000 accuracy(started around 30th epoch) with following configuration
# {'inputFolder': 'D:/atili/MMIExt/Audacity/METU Recordings/Dataset/allSmall/', 'featureMode': 'Mags',
# 'channelMode': '0', 'classificationMode': 'Speaker', 'trainingEpoch': 400, 'stepSize': 0,
# 'sampRate': 48, 'batchSize': 32, 'lengthCut': 600, 'learningRate': 0.001,
# 'lossFunction': 'CatCrosEnt', 'optimizer': 'Adam', 'clsModel': 'LSTM', 'clsVersion': 2}
model.add(
Conv1D(8,
48,
strides=12,
activation='relu',
input_shape=trainShape[1:]))
model.add(Conv1D(16, 36, strides=6, activation='relu'))
model.add(Conv1D(32, 24, strides=2, activation='relu'))
model.add(Conv1D(64, 24, strides=2, activation='relu'))
model.add(GRU(64, return_sequences=True))
model.add(GRU(32, activation='relu', return_sequences=False))
model.add(Dense(numCls, activation='softmax'))
elif 3 == version: # tezde -1 burdan sonra
model.add(
Conv1D(8,
48,
strides=12,
activation='relu',
input_shape=trainShape[1:]))
model.add(Dropout(0.5))
model.add(Conv1D(16, 36, strides=6, activation='relu'))
model.add(Dropout(0.5))
model.add(Conv1D(32, 24, strides=2, activation='relu'))
model.add(Dropout(0.5))
model.add(Conv1D(64, 24, strides=2, activation='relu'))
model.add(LSTM(64, return_sequences=True))
model.add(LSTM(32, activation='relu', return_sequences=False))
model.add(Dense(numCls, activation='softmax'))
elif 4 == version:
model.add(
Conv1D(16,
96,
strides=12,
activation='relu',
input_shape=trainShape[1:]))
model.add(Dropout(0.5))
model.add(Conv1D(32, 48, strides=6, activation='relu'))
model.add(Dropout(0.5))
model.add(Conv1D(64, 24, strides=2, activation='relu'))
model.add(Dropout(0.5))
model.add(CuDNNGRU(64, return_sequences=True))
model.add(Dropout(0.5))
model.add(CuDNNGRU(64, return_sequences=True))
model.add(Dropout(0.5))
model.add(CuDNNGRU(32, return_sequences=False))
model.add(Dense(numCls, activation='softmax'))
elif 5 == version:
model.add(
Conv1D(16,
96,
strides=12,
activation='relu',
input_shape=trainShape[1:]))
model.add(Dropout(0.5))
model.add(Conv1D(32, 48, strides=6, activation='relu'))
model.add(Dropout(0.5))
model.add(Conv1D(64, 24, strides=2, activation='relu'))
model.add(Dropout(0.5))
model.add(CuDNNGRU(64, return_sequences=True))
model.add(Dropout(0.5))
model.add(CuDNNGRU(32, return_sequences=False))
model.add(Dense(numCls, activation='softmax'))
elif 6 == version:
# todo - ai : this is temp clsVer. give a static version number to successful model structures
model.add(
Conv1D(16,
96,
strides=12,
activation='relu',
input_shape=trainShape[1:]))
model.add(Dropout(0.2))
model.add(Conv1D(32, 48, strides=6, activation='relu'))
model.add(Dropout(0.2))
model.add(Conv1D(64, 24, strides=2, activation='relu'))
model.add(Dropout(0.2))
model.add(CuDNNGRU(64, return_sequences=True))
model.add(Dropout(0.2))
model.add(CuDNNGRU(32, return_sequences=False))
model.add(Dense(numCls, activation='softmax'))
elif 7 == version:
# todo - ai : this is temp clsVer. give a static version number to successful model structures
model.add(
Conv1D(32,
96,
strides=16,
activation='relu',
input_shape=trainShape[1:]))
model.add(Conv1D(64, 48, strides=8, activation='relu'))
model.add(CuDNNGRU(64, return_sequences=True))
model.add(CuDNNGRU(32, return_sequences=False))
model.add(Dense(numCls, activation='softmax'))
else:
utils2.myPrint('ERROR: Unknown Classifier Version')
sys.exit()
else: # for Spectograms
utils2.myPrint('Classifier Version: Spectogram')
model.add(
LSTM(24,
activation='relu',
return_sequences=True,
input_shape=trainShape[1:]))
model.add(LSTM(32, activation='relu', return_sequences=False))
model.add(Dense(numCls, activation='softmax'))
utils2.printModelConfig(model.get_config())
##
# Optimizer selection (Paper Refs: https://keras.io/optimizers/ and
# https://www.dlology.com/blog/quick-notes-on-how-to-choose-optimizer-in-keras/)
##
if 'Adam' == optim:
opt = optimizers.adam(lr=learnRate)
elif 'Sgd' == optim:
opt = optimizers.sgd(
lr=learnRate,
nesterov=False) # works well with shallow networks
elif 'SgdNest' == optim:
opt = optimizers.sgd(
lr=learnRate,
nesterov=True) # works well with shallow networks
elif 'Adamax' == optim:
opt = optimizers.adamax(lr=learnRate)
elif 'Nadam' == optim:
opt = optimizers.nadam(lr=learnRate)
elif 'Rms' == optim:
opt = optimizers.rmsprop(lr=learnRate)
else:
utils2.myPrint('ERROR: Invalid Optimizer Parameter Value:', optim)
sys.exit()
##
# Loss function selection (Paper Refs: https://keras.io/losses/ and
# https://machinelearningmastery.com/how-to-choose-loss-functions-when-training-deep-learning-neural-networks/)
##
if 'SparCatCrosEnt' == losFnc:
# sparse_categorical_crossentropy uses integers for labels instead of one-hot vectors.
# Saves memory when numCls is big. Other than that identical to categorical_crossentropy, use that instead.
# Do not use this before modifying labelList structure.
los = losses.sparse_categorical_crossentropy
elif 'CatCrosEnt' == losFnc:
los = losses.categorical_crossentropy
elif 'KLDiv' == losFnc:
los = losses.kullback_leibler_divergence
else:
utils2.myPrint('ERROR: Invalid Loss Function Parameter Value:',
losFnc)
sys.exit()
model.compile(loss=los, optimizer=opt, metrics=['accuracy'])
utils2.myPrint('Optimizer:', opt)
utils2.myPrint('Learning Rate:', backend.eval(model.optimizer.lr))
utils2.myPrint('Loss func:', los)
model.summary(print_fn=utils2.myPrint)
# input('Press ENTER to continue with training:')
utils2.myPrint('')
utils2.myPrint('Training:', flush=True)
# prepare callbacks
earlyStopping = EarlyStopping(monitor='val_loss',
mode='min',
patience=45,
min_delta=1e-4,
restore_best_weights=True,
verbose=1)
# save best model for later use
modelName = 'model_' + utils2.scriptStartDateTime + '_numCls-'+str(numCls) + '_loss-'+losFnc + '_opt-'+optim + \
'_lr-'+str(learnRate) + '_featMode-'+featMode + '_clsVer-'+str(version) + '.clsmdl'
modelPath = './models/' + modelName
modelSaving = ModelCheckpoint(modelPath,
monitor='val_loss',
mode='min',
save_best_only=True,
verbose=1)
reduceLrLoss = ReduceLROnPlateau(monitor='val_loss',
mode='min',
factor=0.5,
cooldown=10,
patience=10,
min_delta=1e-4,
min_lr=learnRate / 32,
verbose=1)
# Train
trainingResults = model.fit(
xTraining,
yTraining,
epochs=trainEpoch,
batch_size=batchSz,
validation_data=(xTesting, yTesting),
callbacks=[earlyStopping, modelSaving, reduceLrLoss])
# model.fit() function prints to console but we can not grab it as it is.
# So myPrint it only to file with given info.
for i in range(len(trainingResults.history['loss'])):
utils2.myPrint(
'Epoch #%d: Loss:%.4f, Accuracy:%.4f, Validation Loss:%.4f, Validation Accuracy:%.4f'
% (i + 1, trainingResults.history['loss'][i],
trainingResults.history['acc'][i],
trainingResults.history['val_loss'][i],
trainingResults.history['val_acc'][i]),
mode='file')
utils2.myPrint(trainingResults.history, mode='code')
utils2.myPrint('')
# Restore best Model
utils2.myPrint('Restoring best model...')
model = load_model(modelPath)
models.append(modelPath)
# Final evaluation of the model
utils2.myPrint('Test:')
scores = model.evaluate(xTesting, yTesting, batch_size=testShape[0])
utils2.myPrint('Test Loss:%.8f, Accuracy:%.4f' %
(scores[0], scores[1]))
# write results to a seperate text file (part 2/3)
fResult = open('./Results.txt', 'a+')
fResult.write(', ' + modelName + ', Test Loss:%.8f, Accuracy:%.4f' %
(scores[0], scores[1]))
fResult.close()
# todo - ai : kfold cross validation can be inserted here
# Stats by class
yTesting1Hot = np.argmax(yTesting, axis=1) # Convert one-hot to index
yTesting1Hot = [labelLst[i] for i in yTesting1Hot]
yPredict = model.predict_classes(xTesting)
yPredict = [labelLst[i] for i in yPredict]
utils2.myPrint('Labels:', labelLst)
utils2.myPrint('Confusion Matrix:')
utils2.myPrint(
pd.DataFrame(confusion_matrix(yTesting1Hot,
yPredict,
labels=labelLst),
index=['t:{:}'.format(x) for x in labelLst],
columns=['{:}'.format(x) for x in labelLst]))
utils2.myPrint('Classification Report:')
utils2.myPrint(
classification_report(yTesting1Hot, yPredict, labels=labelLst))
clearGPU()
del model
gc.collect()
if 1 < len(models):
# Test models, ensembled
modelId = utils2.scriptStartDateTime
modelName = 'model_' + modelId + '_ensembled.clsmdl'
modelPath = './models/' + modelName
# write results to a seperate text file (part 3/3)
for mdlIdx in range(len(models)):
models[mdlIdx] = load_model(models[mdlIdx])
models[mdlIdx].name = modelId + '_' + str(
mdlIdx) # change name to be unique
model_input = Input(shape=models[0].input_shape[1:]) # c*h*w
modelEns = ensembleModels(models, model_input)
modelEns.compile(optimizer=optimizers.adam(lr=learnRate),
loss=losses.categorical_crossentropy,
metrics=['accuracy'])
modelEns.summary(print_fn=utils2.myPrint)
modelEns.save(modelPath)
utils2.myPrint('Ensemble Test:')
scores = modelEns.evaluate(xTesting, yTesting, batch_size=testShape[0])
utils2.myPrint('Test Loss:%.8f, Accuracy:%.4f' %
(scores[0], scores[1]))
fResult = open('./Results.txt', 'a+')
fResult.write(', ' + modelName + ', Test Loss:%.8f, Accuracy:%.4f' %
(scores[0], scores[1]))
fResult.close()
# Stats by class
yTesting1Hot = np.argmax(yTesting, axis=1) # Convert one-hot to index
yTesting1Hot = [labelLst[i] for i in yTesting1Hot]
yPredict = modelEns.predict(xTesting)
yPredict = np.argmax(yPredict, axis=1)
yPredict = [labelLst[i] for i in yPredict]
utils2.myPrint('Labels:', labelLst)
utils2.myPrint('Confusion Matrix:')
utils2.myPrint(
pd.DataFrame(confusion_matrix(yTesting1Hot,
yPredict,
labels=labelLst),
index=['t:{:}'.format(x) for x in labelLst],
columns=['{:}'.format(x) for x in labelLst]))
utils2.myPrint('Classification Report:')
utils2.myPrint(
classification_report(yTesting1Hot, yPredict, labels=labelLst))
del modelEns
clearGPU()
gc.collect()
del xTraining
del xTesting
del yTraining
del yTesting
del labelLst
del models
gc.collect()
clearGPU()
def model_builder(network, constants):
model = network(constants)
model.compile(loss='categorical_crossentropy',
optimizer=adamax(lr=constants['learning_rate']),
metrics=['accuracy'])
def adamax_optimizer(self, learning_rate):
optimizer = optimizers.adamax(lr=learning_rate)
return optimizer
def create_model(activation, optimizer, learning_rate, output_size,
merged_layers):
original_new_androdet_model = models.load_model(
"../new_androdet/model_trained.k")
original_cnn_model = models.load_model("../cnn/model_trained.k")
original_dnn_model = models.load_model("../bow/model_trained.k")
new_androdet_model = models.Sequential()
cnn_model = models.Sequential()
dnn_model = models.Sequential()
for layer in original_new_androdet_model.layers[:-1]:
layer.name = 'new_androdet_' + layer.name
layer.trainable = False
new_androdet_model.add(layer)
for layer in original_cnn_model.layers[:-1]:
layer.name = 'cnn_' + layer.name
layer.trainable = False
cnn_model.add(layer)
for layer in original_dnn_model.layers[:-1]:
layer.name = 'dnn_' + layer.name
layer.trainable = False
dnn_model.add(layer)
entropy_input_layer = layers.Input(shape=(1, ), name='entropy_input')
merge_layer = layers.concatenate([
cnn_model.layers[-1].get_output_at(-1),
dnn_model.layers[-1].get_output_at(-1), entropy_input_layer
])
for (i, n_neurons) in enumerate(merged_layers):
merge_layer = layers.Dense(n_neurons,
activation=activation,
name='dense{}'.format(i))(merge_layer)
output_trivial = layers.concatenate(
[merge_layer, new_androdet_model.layers[-1].get_output_at(-1)])
output_trivial = layers.Dense(1, activation='sigmoid')(output_trivial)
output_rest = layers.Dense(output_size - 1,
activation='sigmoid')(merge_layer)
output_all = layers.concatenate([output_trivial, output_rest])
model = models.Model(inputs=[
new_androdet_model.layers[0].get_input_at(-1),
cnn_model.layers[0].get_input_at(-1),
dnn_model.layers[0].get_input_at(-1), entropy_input_layer
],
outputs=output_all)
if optimizer == 'rmsprop':
opt = optimizers.rmsprop(lr=learning_rate)
elif optimizer == 'adam':
opt = optimizers.adam(lr=learning_rate)
elif optimizer == 'sgd':
opt = optimizers.sgd(lr=learning_rate)
elif optimizer == 'adagrad':
opt = optimizers.adagrad(lr=learning_rate)
elif optimizer == 'adadelta':
opt = optimizers.adadelta(lr=learning_rate)
elif optimizer == 'adamax':
opt = optimizers.adamax(lr=learning_rate)
elif optimizer == 'nadam':
opt = optimizers.nadam(lr=learning_rate)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=["mean_squared_error"])
model.summary()
return model
epsilon=1e-08,
decay=0.0
)
opt.adam(
lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0
)
opt.adamax(
lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0
)
opt.nadam(
lr=0.0002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004
)
#vae = make_parallel(vae, 4)
# train the VAE
# fit and evaluate a model
batch_size_arr = [1024, 32, 128, 512, 1024]
epochs_arr = [300, 200, 500]
aprf = np.zeros((30, 4))
for i in range(1):
n_timesteps, n_features, n_outputs = 1, trainX.shape[1], 1
epochs, batch_size, n_neurons, dropout = 300, 512, n_features, 0.5
n_timesteps, n_features, n_outputs = 1, trainX.shape[1], 1
model = Sequential()
model.add(Dense(n_neurons, input_dim=n_features, activation='relu'))
#model.add(Dropout(dropout))
model.add(Dense(int(n_neurons / 2), activation='relu'))
model.add(Dropout(dropout))
model.add(Dense(5, activation='relu'))
model.add(Dense(n_outputs, activation='sigmoid'))
opt = adamax(lr=0.0002)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.save("model_NN.h5")
# fit network
checkpoint = ModelCheckpoint('weights_NN.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
class_weights = {0: 1, 1: 2}
history = model.fit(trainX,
trainy,
epochs=epochs,
random_state=42)
#%%
#CREATE AUTOENCODER
os.chdir(curr_path)
from keras.layers import Activation
from keras import optimizers
def cust(x):
return tf.keras.backend.sigmoid(x) - 0.5
opt = optimizers.adamax(learning_rate=0.001)
model = Sequential()
model.add(
Conv2D(128, (3, 3),
padding='same',
input_shape=(258, 540, 1),
data_format="channels_last"))
model.add(MaxPooling2D((2, 2), padding='same'))
model.add(UpSampling2D(
(2, 2))) #SIGMOID TO EASILY GENERATE IMAGES IN WIDE RANGE
model.add(Conv2D(1, (3, 3), activation=cust, padding='same'))
model.compile(loss="mean_squared_error", optimizer=opt)
print(model.summary())
model.fit(x_train,
y_train,
