def create_model(input_size, output_size, n_layers, n_neurons, activation_function, learning_rate, dropout_rate, optimizer): model = models.Sequential() model.add( layers.Dense(n_neurons, input_shape=(input_size, ), name='new_androdet_dense_1')) for _ in range(n_layers): if dropout_rate != 0.0: model.add(layers.Dropout(dropout_rate, noise_shape=None, seed=None)) model.add(layers.Dense(n_neurons, activation=activation_function)) model.add(layers.Dense(output_size, activation="sigmoid")) #model.summary() if optimizer == 'rmsprop': opt = optimizers.rmsprop(lr=learning_rate) elif optimizer == 'adam': opt = optimizers.adam(lr=learning_rate) elif optimizer == 'sgd': opt = optimizers.sgd(lr=learning_rate) elif optimizer == 'adagrad': opt = optimizers.adagrad(lr=learning_rate) elif optimizer == 'adadelta': opt = optimizers.adadelta(lr=learning_rate) elif optimizer == 'adamax': opt = optimizers.adamax(lr=learning_rate) elif optimizer == 'nadam': opt = optimizers.nadam(lr=learning_rate) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=["mean_squared_error"]) return model
def MS_build_multivar_cnnlstm_modelB(config, X, y, val_X, val_y): # define the input cnn model model = Sequential() model.add( TimeDistributed(Conv1D(64, 7, activation='relu'), input_shape=(None, int((22 / 2)), 8))) model.add(TimeDistributed(Dropout(0.2))) model.add(TimeDistributed(Conv1D(128, 4, activation='relu'))) model.add(TimeDistributed(MaxPooling1D())) model.add(TimeDistributed(Conv1D(32, 1, activation='relu'))) model.add(TimeDistributed(Conv1D(64, 1, activation='relu'))) model.add(TimeDistributed(MaxPooling1D((1)))) model.add(TimeDistributed(Flatten())) model.add(LSTM(100, activation='sigmoid')) model.add(Dense(100, activation='relu')) model.add(Dense(37)) custom_adamax = opt.adamax(lr=4e-1) custom_sgd = SGD(lr=4e-1, momentum=0.9) clr = CyclicLR(mode='triangular2') model.compile(loss='mae', optimizer='adamax', metrics=['mae']) # fit network model.fit(X, y, epochs=35, batch_size=7, verbose=2, validation_data=(val_X, val_y), callbacks=[clr]) return model
def __init__(self, n_features, anomaly_class): #input input_layer = Input(shape=(n_features, )) #encode encoder = Dense(8, kernel_initializer='he_uniform', activation='relu')(input_layer) drop = Dropout(rate=0.2)(encoder) #latent latent = Dense(2, kernel_initializer='he_uniform', activation='relu')(drop) drop = Dropout(rate=0.2)(latent) #decode decoder = Dense(8, kernel_initializer='he_uniform', activation='relu')(drop) drop = Dropout(rate=0.2)(decoder) #output output_layer = Dense(n_features, activation="sigmoid")(drop) self.model = Model(inputs=input_layer, outputs=output_layer) #compile self.model.compile(loss='mean_squared_error', optimizer=adamax(lr=0.1)) #training self.history = None #name self.model_name = "A2" #anomaly class self.anomaly_class = anomaly_class
def getOptimizer(name, rate): if name is 'adamax': return adamax(lr=rate) elif name is 'adam': return adam(lr=rate) elif name is 'nadam': return Nadam(lr=rate) else: return RMSprop(lr=rate)
def getOptimizer(name,rate): if name == 'adamax': return adamax(lr=rate) elif name == 'adam': return adam(lr=rate) elif name == 'nadam': return Nadam(lr=rate) elif name == 'sgd': return SGD(lr=rate, momentum=0.9, nesterov=True) else: return RMSprop(lr=rate)
def _main(model_name): # data_sets path train_dir = '/home/lst/datasets/cifar-10-images_train/' val_dir = '/home/lst/datasets/cifar-10-images_test/' # model model = GoogLeNet(input_shape=(224, 224, 3)) # parallel model parallel_model = multi_gpu_model(model, gpus=2) # optimizers setting from keras import optimizers optimizer = optimizers.adamax(lr=0.002, decay=1e-06) parallel_model.compile(loss=[ 'categorical_crossentropy', 'categorical_crossentropy', 'categorical_crossentropy' ], loss_weights=[1, 0.3, 0.3], optimizer=optimizer, metrics=["accuracy"]) # load data by batch train_generator, validation_generator, num_train, num_val = load_batch_data( train_dir, val_dir) # Callbacks settings from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard early_stop = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=30, mode='min', verbose=1) checkpoint = ModelCheckpoint(f'{model_name}.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1) tensorboard = TensorBoard(log_dir=f'./logs/{model_name}', histogram_freq=0, write_graph=True, write_images=False) # fit parallel_model.fit_generator( train_generator, validation_data=validation_generator, steps_per_epoch=math.ceil(num_train / batch_size), validation_steps=math.ceil(num_val / batch_size), epochs=epochs, callbacks=[tensorboard, early_stop, checkpoint], )
def adammax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0): """ Adam optimizer :param lr: >=0, learning rate :param beta_1: 0 < beta_1 < 1, generally close to 1 :param beta_2: 0 < beta_2 < 1, generally close to 1 :param epsilon: >=0, fuzz factor. If None, defaults to K.epsilon() :param decay: >=0, learning rate decay over each update """ return optimizers.adamax(lr=lr, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, decay=decay)
def DropoutAdamax(self): # Build the model print("Building model...") model = Sequential() # Single 500-neuron hidden layer with sigmoid activation model.add(Dropout(rate=0.4, input_shape=(self.nb_features,))) model.add(Dense(input_dim=self.nb_features, units=int(self.nb_features / 0.4), activation='relu')) model.add(Dropout(rate=0.4, input_shape=(int(self.nb_features / 0.4),))) # Output layer with softmax activation model.add(Dense(units=self.nb_classes, activation='softmax')) # Specify optimizer, loss and validation metric opt = adamax(lr=0.0015) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) self.model = model
def create_model(layers_and_filters, kernels, activation, input_shape, dropout_rate, optimizer, learning_rate, output_size=1): model = models.Sequential() i = 0 for filters in layers_and_filters: model.add( layers.Conv2D(filters, kernel_size=kernels[i], strides=kernels[i], activation=activation, input_shape=input_shape)) i += 1 if i < len(layers_and_filters): model.add(layers.MaxPooling2D(pool_size=(2, 2))) model.add(layers.BatchNormalization()) if dropout_rate != 0: model.add(layers.Dropout(dropout_rate)) model.add(layers.Flatten()) model.add(layers.Dense(output_size, activation='sigmoid')) if optimizer == 'rmsprop': opt = optimizers.rmsprop(lr=learning_rate) elif optimizer == 'adam': opt = optimizers.adam(lr=learning_rate) elif optimizer == 'sgd': opt = optimizers.sgd(lr=learning_rate) elif optimizer == 'adagrad': opt = optimizers.adagrad(lr=learning_rate) elif optimizer == 'adadelta': opt = optimizers.adadelta(lr=learning_rate) elif optimizer == 'adamax': opt = optimizers.adamax(lr=learning_rate) elif optimizer == 'nadam': opt = optimizers.nadam(lr=learning_rate) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=["mean_squared_error"]) #model.summary() return model
def __init__(self, n_features): # Input input_layer = Input(shape=(n_features, )) # Hidden layers layer = Dense(32, kernel_initializer='uniform', activation="relu")(input_layer) drop = Dropout(rate=0.2)(layer) layer = Dense(64, kernel_initializer='he_uniform', activation="relu")(input_layer) drop = Dropout(rate=0.2)(layer) layer = Dense(128, kernel_initializer='he_uniform', activation="relu")(input_layer) drop = Dropout(rate=0.2)(layer) layer = Dense(256, kernel_initializer='he_uniform', activation="relu")(input_layer) drop = Dropout(rate=0.2)(layer) layer = Dense(128, kernel_initializer='he_uniform', activation="relu")(input_layer) drop = Dropout(rate=0.2)(layer) layer = Dense(64, kernel_initializer='he_uniform', activation="relu")(input_layer) drop = Dropout(rate=0.2)(layer) layer = Dense(32, kernel_initializer='he_uniform', activation="relu")(input_layer) drop = Dropout(rate=0.2)(layer) # Output output_layer = Dense(1, kernel_initializer="he_uniform", activation='sigmoid')(drop) # Model self.model = Model(inputs=input_layer, outputs=output_layer) # Compile self.model.compile(optimizer=adamax(0.05), loss='binary_crossentropy') # Training self.history = None # Name self.model_name = "BC4"
def model(): model = Sequential() model.add(Conv2D(16, (3, 3), input_shape=(32, 32, 3), padding="same")) model.add(LeakyReLU(0.1)) model.add(Conv2D(32, (3, 3), padding="same")) model.add(LeakyReLU(0.1)) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Conv2D(32, (3, 3), padding="same")) model.add(LeakyReLU(0.1)) model.add(Conv2D(64, (3, 3), padding="same")) model.add(LeakyReLU(0.1)) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(256)) model.add(LeakyReLU(0.1)) model.add(Dropout(0.5)) model.add(Dense(NUM_CLASSES)) model.add(Activation("softmax")) model.compile( loss='categorical_crossentropy', # we train 10-way classification optimizer=adamax(lr=INIT_LR), # for SGD metrics=['accuracy'] # report accuracy during training ) return model
def initialize_optimizer(optimizer_name: str, learning_rate: float, beta1: float, beta2: float, lr_decay: float, rho: float, fuzz: float, momentum: float) \ -> Union[adam, rmsprop, sgd, adagrad, adadelta, adamax]: """ Initializes an optimizer based on the user's choices. :param optimizer_name: the optimizer's name. Can be one of 'adam', 'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adamax'. :param learning_rate: the optimizer's learning_rate :param beta1: the optimizer's beta1 :param beta2: the optimizer's beta2 :param lr_decay: the optimizer's lr_decay :param rho: the optimizer's rho :param fuzz: the optimizer's fuzz :param momentum: the optimizer's momentum :return: the optimizer. """ if optimizer_name == 'adam': return adam(lr=learning_rate, beta_1=beta1, beta_2=beta2, decay=lr_decay) elif optimizer_name == 'rmsprop': return rmsprop(lr=learning_rate, rho=rho, epsilon=fuzz) elif optimizer_name == 'sgd': return sgd(lr=learning_rate, momentum=momentum, decay=lr_decay) elif optimizer_name == 'adagrad': return adagrad(lr=learning_rate, decay=lr_decay) elif optimizer_name == 'adadelta': return adadelta(lr=learning_rate, rho=rho, decay=lr_decay) elif optimizer_name == 'adamax': return adamax(lr=learning_rate, beta_1=beta1, beta_2=beta2, decay=lr_decay) else: raise ValueError('An unexpected optimizer name has been encountered.')
def create_optimizer_instance(self, **d): return optimizers.adamax(**d)
def cnn(self, dic={ 'act': 'tanh', 'init': 'lecun_normal' }, number={ 'kernel': n, 'pool': 100 }, learate=0.01, name='Default', ndim=256): gc.collect() act = dic['act'] init = dic['init'] kernel = number['kernel'] pool = number['pool'] model = Sequential() model.add( Convolution1D(ndim, kernel_size=(kernel), input_shape=(self.x_train.shape[1], 1), padding='same', kernel_initializer=init, kernel_regularizer=regularizers.l1_l2(l1=0.01, l2=0.01))) model.add(BatchNormalization()) model.add(Activation(act)) model.add(MaxPooling1D(pool_size=(pool), padding='same')) model.add(Activation(act)) model.add(BatchNormalization()) model.add(Flatten()) model.add(GaussianNoise(0.1)) model.add(Dense(int(ndim / 2))) model.add(Activation(act)) model.add(BatchNormalization()) model.add(GaussianNoise(0.1)) model.add(Dense(int(ndim / 2))) model.add(Activation(act)) model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(Dense(int(ndim / 4))) model.add(Activation(act)) model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(Dense(int(ndim / 4))) model.add(Activation(act)) model.add(BatchNormalization()) model.add(Dense(1, activation='sigmoid')) auc_roc = self.as_keras_metric(tf.metrics.auc) op = optimizers.adamax(lr=learate) pr = self.as_keras_metric(tf.metrics.precision) rec = self.as_keras_metric(tf.metrics.recall) model.compile(loss='binary_crossentropy', optimizer=op, metrics=[auc_roc, rec, pr]) # es = EarlyStopping(monitor='auc',patience=9, verbose=1,mode='max') # lr = ReduceLROnPlateau(monitor='auc', factor=0.1, patience=3, verbose=1, mode='max', min_delta=0.0001,cooldown=0, min_lr=0) # history = model.fit(self.x_train, self.y_train, epochs=30, batch_size=250, callbacks=[es, lr], validation_split=0.1,shuffle=True) # y_pre = model.predict_classes(self.x_test,batch_size=250) # score = model.evaluate(self.x_test,self.y_test,batch_size=250) # f1 = (2*score[3]*score[2])/(score[3]+score[2]) # plt.figure(figsize=(12, 6)) # plt.subplot(121) # plt.plot(history.history['auc']) # plt.plot(history.history['val_auc']) # plt.title('model auc') # plt.ylabel('accuracy') # plt.xlabel('epoch') # plt.suptitle('CNN:The Over Sampling is %s'%name) # plt.legend(['train', 'validation'], loc='upper left') # plt.subplot(122) # plt.plot(history.history['precision']) # plt.plot(history.history['val_precision']) # plt.title('model precision') # plt.ylabel('precision') # plt.xlabel('epoch') # plt.legend(['train', 'validation'], loc='upper left') # plt.savefig('E:\\OneDrive - Georgia State University\\Kaggle Home Equity\\Ping\\Reports\\DeepLearning\\CNN\\%s.png'%name) # plt.show() # print('The test AUC is ',score[1],'The recall is',score[2],'The precision is',score[3]) # print('F1 Score is',f1) return model
def bpmodel(self, dic={ 'act': 'sigmoid', 'init': 'lecun_normal' }, nl={'node': 360}, learate=0.1, name='Default'): gc.collect() act = dic['act'] init = dic['init'] number = nl['node'] model = Sequential() model.add( Dense(number, input_dim=self.dim, kernel_initializer=init, kernel_regularizer=regularizers.l1_l2(l1=0.005, l2=0.005))) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dense(number * 10, kernel_initializer=init)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dense(number * 8, kernel_initializer=init)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) model.add(Dense(number * 6, kernel_initializer=init)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dense(number * 4, kernel_initializer=init)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) model.add(Dense(number * 2, kernel_initializer=init)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dense(number, kernel_initializer=init)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dense(int(number / 2), kernel_initializer=init)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) model.add(Dense(int(number / 4), kernel_initializer=init)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dense(int(number / 6), kernel_initializer=init)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dense(int(number / 12), kernel_initializer=init)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dense(1, activation='sigmoid', kernel_initializer=init)) auc_roc = self.as_keras_metric(tf.metrics.auc) op = optimizers.adamax(lr=learate) pr = self.as_keras_metric(tf.metrics.precision) rec = self.as_keras_metric(tf.metrics.recall) model.compile(loss='binary_crossentropy', optimizer=op, metrics=[auc_roc, rec, pr]) model.save( 'E:\OneDrive - Georgia State University\Kaggle Home Equity\Ping\Code\\bpmodel.h5' ) # es = EarlyStopping(monitor='auc',patience=9, verbose=1,mode='max') # lr = ReduceLROnPlateau(monitor='auc', factor=0.1, patience=3, verbose=1, mode='max', min_delta=0.0001,cooldown=0, min_lr=0) # history = model.fit(self.x_train, self.y_train,epochs=50,batch_size=1000,callbacks=[es,lr],validation_split=0.1) # y_pre = model.predict(self.x_test,batch_size=1000) # y_pre1 = model.predict_classes(self.x_test,batch_size=1000) # score = model.evaluate(self.x_test,self.y_test,batch_size=1000) # f1 = (2*score[3]*score[2])/(score[3]+score[2]) # plt.figure(figsize=(12,6)) # plt.subplot(121) # plt.plot(history.history['auc']) # plt.plot(history.history['val_auc']) # plt.title('model auc') # plt.ylabel('auc') # plt.xlabel('epoch') # plt.suptitle('BP: The Over Sampling is %s'%name) # plt.legend(['train', 'validation'], loc='upper left') # plt.subplot(122) # plt.plot(history.history['precision']) # plt.plot(history.history['val_precision']) # plt.title('model precision') # plt.ylabel('precision') # plt.xlabel('epoch') # plt.legend(['train', 'validation'], loc='upper left') # plt.savefig('E:\\OneDrive - Georgia State University\\Kaggle Home Equity\\Ping\\Reports\\DeepLearning\\BP\\%s.png'%name) # plt.show() # print('The test AUC is ',score[1],'The recall is',score[2],'The precision is',score[3]) # print('F1 Score is',f1) return model
def trainTestLSTM(xTraining, xTesting, yTraining, yTesting, numCls, trainEpoch, batchSz, labelLst, losFnc, optim, learnRate, featMode, clsVer): gc.collect() clearGPU() utils2.myPrint('---LSTM Classifier---') trainShape = np.shape(xTraining) testShape = np.shape(xTesting) utils2.myPrint('Train Batch:', trainShape) utils2.myPrint('Test Batch:', testShape) models = list() for version in clsVer: # create the model model = Sequential() # todo - ai : possible variations # try lstm decay # try clipnorm and clipvalue # try convlstm2d and/or concatanate two models # try sgd instead of adam optimizer if 'Specto' != featMode: # do not convolve for spectogram, since it is not as long as other modes. utils2.myPrint('Classifier Version:', version) if 0 == version: # inputs 1 # 300 epoch .3924. model.add(Conv1D(8, 48, strides=48, input_shape=trainShape[1:])) model.add(Activation('relu')) model.add(Conv1D(16, 24, strides=24)) model.add(Activation('sigmoid')) model.add(LSTM(24, return_sequences=True)) model.add(LSTM(12, return_sequences=False)) model.add(Dense(numCls, activation='softmax')) elif 1 == version: model.add( Conv1D(8, 48, strides=12, activation='relu', input_shape=trainShape[1:])) model.add(Conv1D(16, 36, strides=6, activation='relu')) model.add(Conv1D(32, 24, strides=2, activation='relu')) model.add(Conv1D(64, 24, strides=2, activation='relu')) model.add(LSTM(64, return_sequences=True)) model.add(LSTM(32, activation='relu', return_sequences=False)) model.add(Dense(numCls, activation='softmax')) elif 2 == version: # resulted better than LSTM variant(1 == clsVer) with following configuration # {'inputFolder': 'D:/atili/MMIExt/Audacity/METU Recordings/Dataset/4spkr5post/', 'featureMode': 'Mags', # 'channelMode': '0', 'classificationMode': 'Speaker', 'trainingEpoch': 200, 'stepSize': 0, # 'sampRate': 48, 'batchSize': 32, 'lengthCut': 600, 'learningRate': 0.001, # 'lossFunction': 'CatCrosEnt', 'optimizer': 'Adam', 'clsModel': 'LSTM', 'clsVersion': 2} # overfitted with 1.000 accuracy(started around 30th epoch) with following configuration # {'inputFolder': 'D:/atili/MMIExt/Audacity/METU Recordings/Dataset/allSmall/', 'featureMode': 'Mags', # 'channelMode': '0', 'classificationMode': 'Speaker', 'trainingEpoch': 400, 'stepSize': 0, # 'sampRate': 48, 'batchSize': 32, 'lengthCut': 600, 'learningRate': 0.001, # 'lossFunction': 'CatCrosEnt', 'optimizer': 'Adam', 'clsModel': 'LSTM', 'clsVersion': 2} model.add( Conv1D(8, 48, strides=12, activation='relu', input_shape=trainShape[1:])) model.add(Conv1D(16, 36, strides=6, activation='relu')) model.add(Conv1D(32, 24, strides=2, activation='relu')) model.add(Conv1D(64, 24, strides=2, activation='relu')) model.add(GRU(64, return_sequences=True)) model.add(GRU(32, activation='relu', return_sequences=False)) model.add(Dense(numCls, activation='softmax')) elif 3 == version: # tezde -1 burdan sonra model.add( Conv1D(8, 48, strides=12, activation='relu', input_shape=trainShape[1:])) model.add(Dropout(0.5)) model.add(Conv1D(16, 36, strides=6, activation='relu')) model.add(Dropout(0.5)) model.add(Conv1D(32, 24, strides=2, activation='relu')) model.add(Dropout(0.5)) model.add(Conv1D(64, 24, strides=2, activation='relu')) model.add(LSTM(64, return_sequences=True)) model.add(LSTM(32, activation='relu', return_sequences=False)) model.add(Dense(numCls, activation='softmax')) elif 4 == version: model.add( Conv1D(16, 96, strides=12, activation='relu', input_shape=trainShape[1:])) model.add(Dropout(0.5)) model.add(Conv1D(32, 48, strides=6, activation='relu')) model.add(Dropout(0.5)) model.add(Conv1D(64, 24, strides=2, activation='relu')) model.add(Dropout(0.5)) model.add(CuDNNGRU(64, return_sequences=True)) model.add(Dropout(0.5)) model.add(CuDNNGRU(64, return_sequences=True)) model.add(Dropout(0.5)) model.add(CuDNNGRU(32, return_sequences=False)) model.add(Dense(numCls, activation='softmax')) elif 5 == version: model.add( Conv1D(16, 96, strides=12, activation='relu', input_shape=trainShape[1:])) model.add(Dropout(0.5)) model.add(Conv1D(32, 48, strides=6, activation='relu')) model.add(Dropout(0.5)) model.add(Conv1D(64, 24, strides=2, activation='relu')) model.add(Dropout(0.5)) model.add(CuDNNGRU(64, return_sequences=True)) model.add(Dropout(0.5)) model.add(CuDNNGRU(32, return_sequences=False)) model.add(Dense(numCls, activation='softmax')) elif 6 == version: # todo - ai : this is temp clsVer. give a static version number to successful model structures model.add( Conv1D(16, 96, strides=12, activation='relu', input_shape=trainShape[1:])) model.add(Dropout(0.2)) model.add(Conv1D(32, 48, strides=6, activation='relu')) model.add(Dropout(0.2)) model.add(Conv1D(64, 24, strides=2, activation='relu')) model.add(Dropout(0.2)) model.add(CuDNNGRU(64, return_sequences=True)) model.add(Dropout(0.2)) model.add(CuDNNGRU(32, return_sequences=False)) model.add(Dense(numCls, activation='softmax')) elif 7 == version: # todo - ai : this is temp clsVer. give a static version number to successful model structures model.add( Conv1D(32, 96, strides=16, activation='relu', input_shape=trainShape[1:])) model.add(Conv1D(64, 48, strides=8, activation='relu')) model.add(CuDNNGRU(64, return_sequences=True)) model.add(CuDNNGRU(32, return_sequences=False)) model.add(Dense(numCls, activation='softmax')) else: utils2.myPrint('ERROR: Unknown Classifier Version') sys.exit() else: # for Spectograms utils2.myPrint('Classifier Version: Spectogram') model.add( LSTM(24, activation='relu', return_sequences=True, input_shape=trainShape[1:])) model.add(LSTM(32, activation='relu', return_sequences=False)) model.add(Dense(numCls, activation='softmax')) utils2.printModelConfig(model.get_config()) ## # Optimizer selection (Paper Refs: https://keras.io/optimizers/ and # https://www.dlology.com/blog/quick-notes-on-how-to-choose-optimizer-in-keras/) ## if 'Adam' == optim: opt = optimizers.adam(lr=learnRate) elif 'Sgd' == optim: opt = optimizers.sgd( lr=learnRate, nesterov=False) # works well with shallow networks elif 'SgdNest' == optim: opt = optimizers.sgd( lr=learnRate, nesterov=True) # works well with shallow networks elif 'Adamax' == optim: opt = optimizers.adamax(lr=learnRate) elif 'Nadam' == optim: opt = optimizers.nadam(lr=learnRate) elif 'Rms' == optim: opt = optimizers.rmsprop(lr=learnRate) else: utils2.myPrint('ERROR: Invalid Optimizer Parameter Value:', optim) sys.exit() ## # Loss function selection (Paper Refs: https://keras.io/losses/ and # https://machinelearningmastery.com/how-to-choose-loss-functions-when-training-deep-learning-neural-networks/) ## if 'SparCatCrosEnt' == losFnc: # sparse_categorical_crossentropy uses integers for labels instead of one-hot vectors. # Saves memory when numCls is big. Other than that identical to categorical_crossentropy, use that instead. # Do not use this before modifying labelList structure. los = losses.sparse_categorical_crossentropy elif 'CatCrosEnt' == losFnc: los = losses.categorical_crossentropy elif 'KLDiv' == losFnc: los = losses.kullback_leibler_divergence else: utils2.myPrint('ERROR: Invalid Loss Function Parameter Value:', losFnc) sys.exit() model.compile(loss=los, optimizer=opt, metrics=['accuracy']) utils2.myPrint('Optimizer:', opt) utils2.myPrint('Learning Rate:', backend.eval(model.optimizer.lr)) utils2.myPrint('Loss func:', los) model.summary(print_fn=utils2.myPrint) # input('Press ENTER to continue with training:') utils2.myPrint('') utils2.myPrint('Training:', flush=True) # prepare callbacks earlyStopping = EarlyStopping(monitor='val_loss', mode='min', patience=45, min_delta=1e-4, restore_best_weights=True, verbose=1) # save best model for later use modelName = 'model_' + utils2.scriptStartDateTime + '_numCls-'+str(numCls) + '_loss-'+losFnc + '_opt-'+optim + \ '_lr-'+str(learnRate) + '_featMode-'+featMode + '_clsVer-'+str(version) + '.clsmdl' modelPath = './models/' + modelName modelSaving = ModelCheckpoint(modelPath, monitor='val_loss', mode='min', save_best_only=True, verbose=1) reduceLrLoss = ReduceLROnPlateau(monitor='val_loss', mode='min', factor=0.5, cooldown=10, patience=10, min_delta=1e-4, min_lr=learnRate / 32, verbose=1) # Train trainingResults = model.fit( xTraining, yTraining, epochs=trainEpoch, batch_size=batchSz, validation_data=(xTesting, yTesting), callbacks=[earlyStopping, modelSaving, reduceLrLoss]) # model.fit() function prints to console but we can not grab it as it is. # So myPrint it only to file with given info. for i in range(len(trainingResults.history['loss'])): utils2.myPrint( 'Epoch #%d: Loss:%.4f, Accuracy:%.4f, Validation Loss:%.4f, Validation Accuracy:%.4f' % (i + 1, trainingResults.history['loss'][i], trainingResults.history['acc'][i], trainingResults.history['val_loss'][i], trainingResults.history['val_acc'][i]), mode='file') utils2.myPrint(trainingResults.history, mode='code') utils2.myPrint('') # Restore best Model utils2.myPrint('Restoring best model...') model = load_model(modelPath) models.append(modelPath) # Final evaluation of the model utils2.myPrint('Test:') scores = model.evaluate(xTesting, yTesting, batch_size=testShape[0]) utils2.myPrint('Test Loss:%.8f, Accuracy:%.4f' % (scores[0], scores[1])) # write results to a seperate text file (part 2/3) fResult = open('./Results.txt', 'a+') fResult.write(', ' + modelName + ', Test Loss:%.8f, Accuracy:%.4f' % (scores[0], scores[1])) fResult.close() # todo - ai : kfold cross validation can be inserted here # Stats by class yTesting1Hot = np.argmax(yTesting, axis=1) # Convert one-hot to index yTesting1Hot = [labelLst[i] for i in yTesting1Hot] yPredict = model.predict_classes(xTesting) yPredict = [labelLst[i] for i in yPredict] utils2.myPrint('Labels:', labelLst) utils2.myPrint('Confusion Matrix:') utils2.myPrint( pd.DataFrame(confusion_matrix(yTesting1Hot, yPredict, labels=labelLst), index=['t:{:}'.format(x) for x in labelLst], columns=['{:}'.format(x) for x in labelLst])) utils2.myPrint('Classification Report:') utils2.myPrint( classification_report(yTesting1Hot, yPredict, labels=labelLst)) clearGPU() del model gc.collect() if 1 < len(models): # Test models, ensembled modelId = utils2.scriptStartDateTime modelName = 'model_' + modelId + '_ensembled.clsmdl' modelPath = './models/' + modelName # write results to a seperate text file (part 3/3) for mdlIdx in range(len(models)): models[mdlIdx] = load_model(models[mdlIdx]) models[mdlIdx].name = modelId + '_' + str( mdlIdx) # change name to be unique model_input = Input(shape=models[0].input_shape[1:]) # c*h*w modelEns = ensembleModels(models, model_input) modelEns.compile(optimizer=optimizers.adam(lr=learnRate), loss=losses.categorical_crossentropy, metrics=['accuracy']) modelEns.summary(print_fn=utils2.myPrint) modelEns.save(modelPath) utils2.myPrint('Ensemble Test:') scores = modelEns.evaluate(xTesting, yTesting, batch_size=testShape[0]) utils2.myPrint('Test Loss:%.8f, Accuracy:%.4f' % (scores[0], scores[1])) fResult = open('./Results.txt', 'a+') fResult.write(', ' + modelName + ', Test Loss:%.8f, Accuracy:%.4f' % (scores[0], scores[1])) fResult.close() # Stats by class yTesting1Hot = np.argmax(yTesting, axis=1) # Convert one-hot to index yTesting1Hot = [labelLst[i] for i in yTesting1Hot] yPredict = modelEns.predict(xTesting) yPredict = np.argmax(yPredict, axis=1) yPredict = [labelLst[i] for i in yPredict] utils2.myPrint('Labels:', labelLst) utils2.myPrint('Confusion Matrix:') utils2.myPrint( pd.DataFrame(confusion_matrix(yTesting1Hot, yPredict, labels=labelLst), index=['t:{:}'.format(x) for x in labelLst], columns=['{:}'.format(x) for x in labelLst])) utils2.myPrint('Classification Report:') utils2.myPrint( classification_report(yTesting1Hot, yPredict, labels=labelLst)) del modelEns clearGPU() gc.collect() del xTraining del xTesting del yTraining del yTesting del labelLst del models gc.collect() clearGPU()
def model_builder(network, constants): model = network(constants) model.compile(loss='categorical_crossentropy', optimizer=adamax(lr=constants['learning_rate']), metrics=['accuracy'])
def adamax_optimizer(self, learning_rate): optimizer = optimizers.adamax(lr=learning_rate) return optimizer
def create_model(activation, optimizer, learning_rate, output_size, merged_layers): original_new_androdet_model = models.load_model( "../new_androdet/model_trained.k") original_cnn_model = models.load_model("../cnn/model_trained.k") original_dnn_model = models.load_model("../bow/model_trained.k") new_androdet_model = models.Sequential() cnn_model = models.Sequential() dnn_model = models.Sequential() for layer in original_new_androdet_model.layers[:-1]: layer.name = 'new_androdet_' + layer.name layer.trainable = False new_androdet_model.add(layer) for layer in original_cnn_model.layers[:-1]: layer.name = 'cnn_' + layer.name layer.trainable = False cnn_model.add(layer) for layer in original_dnn_model.layers[:-1]: layer.name = 'dnn_' + layer.name layer.trainable = False dnn_model.add(layer) entropy_input_layer = layers.Input(shape=(1, ), name='entropy_input') merge_layer = layers.concatenate([ cnn_model.layers[-1].get_output_at(-1), dnn_model.layers[-1].get_output_at(-1), entropy_input_layer ]) for (i, n_neurons) in enumerate(merged_layers): merge_layer = layers.Dense(n_neurons, activation=activation, name='dense{}'.format(i))(merge_layer) output_trivial = layers.concatenate( [merge_layer, new_androdet_model.layers[-1].get_output_at(-1)]) output_trivial = layers.Dense(1, activation='sigmoid')(output_trivial) output_rest = layers.Dense(output_size - 1, activation='sigmoid')(merge_layer) output_all = layers.concatenate([output_trivial, output_rest]) model = models.Model(inputs=[ new_androdet_model.layers[0].get_input_at(-1), cnn_model.layers[0].get_input_at(-1), dnn_model.layers[0].get_input_at(-1), entropy_input_layer ], outputs=output_all) if optimizer == 'rmsprop': opt = optimizers.rmsprop(lr=learning_rate) elif optimizer == 'adam': opt = optimizers.adam(lr=learning_rate) elif optimizer == 'sgd': opt = optimizers.sgd(lr=learning_rate) elif optimizer == 'adagrad': opt = optimizers.adagrad(lr=learning_rate) elif optimizer == 'adadelta': opt = optimizers.adadelta(lr=learning_rate) elif optimizer == 'adamax': opt = optimizers.adamax(lr=learning_rate) elif optimizer == 'nadam': opt = optimizers.nadam(lr=learning_rate) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=["mean_squared_error"]) model.summary() return model
epsilon=1e-08, decay=0.0 ) opt.adam( lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0 ) opt.adamax( lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0 ) opt.nadam( lr=0.0002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004 ) #vae = make_parallel(vae, 4) # train the VAE
# fit and evaluate a model batch_size_arr = [1024, 32, 128, 512, 1024] epochs_arr = [300, 200, 500] aprf = np.zeros((30, 4)) for i in range(1): n_timesteps, n_features, n_outputs = 1, trainX.shape[1], 1 epochs, batch_size, n_neurons, dropout = 300, 512, n_features, 0.5 n_timesteps, n_features, n_outputs = 1, trainX.shape[1], 1 model = Sequential() model.add(Dense(n_neurons, input_dim=n_features, activation='relu')) #model.add(Dropout(dropout)) model.add(Dense(int(n_neurons / 2), activation='relu')) model.add(Dropout(dropout)) model.add(Dense(5, activation='relu')) model.add(Dense(n_outputs, activation='sigmoid')) opt = adamax(lr=0.0002) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy']) model.save("model_NN.h5") # fit network checkpoint = ModelCheckpoint('weights_NN.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1) class_weights = {0: 1, 1: 2} history = model.fit(trainX, trainy, epochs=epochs,
random_state=42) #%% #CREATE AUTOENCODER os.chdir(curr_path) from keras.layers import Activation from keras import optimizers def cust(x): return tf.keras.backend.sigmoid(x) - 0.5 opt = optimizers.adamax(learning_rate=0.001) model = Sequential() model.add( Conv2D(128, (3, 3), padding='same', input_shape=(258, 540, 1), data_format="channels_last")) model.add(MaxPooling2D((2, 2), padding='same')) model.add(UpSampling2D( (2, 2))) #SIGMOID TO EASILY GENERATE IMAGES IN WIDE RANGE model.add(Conv2D(1, (3, 3), activation=cust, padding='same')) model.compile(loss="mean_squared_error", optimizer=opt) print(model.summary()) model.fit(x_train, y_train,