def _go_cached(self): """ Generate the model needed to use cached batches (i.e. drop the non-trainable steps) """ print("Using cached mode (skipping non-trainable steps)") first_trainable = None if self.model_trainable == None: print("Trainable model not found. Building.") mdl = Sequential() for i in range(len(self.model.layers)): if self.model.layers[i].trainable == True: if first_trainable is None: first_trainable = self.model.layers[i] mdl.add(Dropout(0, input_shape=(14, 14, 512))) mdl.add(self.model.layers[i]) else: mdl.add(self.model.layers[i]) if first_trainable == None: print( "All layers non-trainable. Don't know what to cut. Exiting" ) return self.model_trainable = mdl self.model_trainable.compile(optimizer=Nadam(), loss='categorical_crossentropy', metrics=['accuracy']) self.model_trainable.summary()
def build_model(seq_len: int, word_embedding_dim: int, vocab_size: int, hidden_state_dim: int, learning_rate: float): sequence_input = Input(shape=(seq_len, ), dtype='int32') embeddings = Embedding(vocab_size, word_embedding_dim, input_length=seq_len)(sequence_input) lstm = Bidirectional( LSTM(hidden_state_dim, return_sequences=True, return_state=True, dropout=.5, recurrent_dropout=.4))(embeddings) lstm, forward_h, forward_c, backward_h, backward_c = Bidirectional( LSTM(hidden_state_dim, return_sequences=True, return_state=True, dropout=0.5, recurrent_dropout=.4))(lstm) state_h = Add()([forward_h, backward_h]) attention = Attention(hidden_state_dim) context_vector, attention_weights = attention(lstm, state_h) dense = Dense(100, activation='relu')(context_vector) dropout = Dropout(rate=.3)(dense) output = Dense(1, activation='sigmoid')(dropout) model = Model(inputs=sequence_input, outputs=output, name="TweetsModel") print(model.summary()) model.compile(optimizer=Nadam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy']) return model
def boosting_rank_net(input_shape, hns=[8, 6, 4, 4], classes=2): """ """ res_model = boosting_res_net((input_shape[1], ), hns, out_layer_name='proba') res_model = Model(res_model.input, res_model.get_layer('pre_sigmoid').output) inputs = Input(input_shape) minor_inputs = Lambda(lambda x: x[:, 0], name='minor_input')(inputs) pred_minor = res_model(minor_inputs) minor_out_proba = Lambda(lambda x: x, name='minor_out_proba')(pred_minor) major_inputs = Lambda(lambda x: x[:, 1], name='major_input')(inputs) pred_major = res_model(major_inputs) major_out_proba = Lambda(lambda x: x, name='major_out_proba')(pred_major) sub = Subtract()([major_out_proba, minor_out_proba]) sub = Lambda(lambda x: x * RANK_SCALE, name='rank_scale_layer')(sub) proba = Activation('sigmoid')(sub) model = Model(inputs, proba) model.compile(optimizer=Nadam(lr=0.001), loss=min_pred) return model
def _model(train, test, target): optimizer = Nadam(lr=0.0002) reshape_X_train, reshape_X_test, reshape_y_train, reshape_y_test, reshape_scaled_train, validation = selfprocessing( train, test, target) num = train.shape[1] print(num) model = Sequential() model.add( LSTM(11, activation='linear', return_sequences=True, input_shape=(1, num))) model.add(Dropout(0.4)) model.add( LSTM(22, activation='linear', return_sequences=True, input_shape=(1, num))) model.add(Dropout(0.4)) model.add( LSTM(44, activation='linear', return_sequences=True, input_shape=(1, num))) model.add(Dropout(0.4)) model.add(LSTM(88, activation='linear', input_shape=(1, num))) model.add(Dropout(0.1)) model.add(Dense(1, activation='sigmoid')) model.compile(loss='mae', optimizer=optimizer, metrics=['mse']) result = model.fit(reshape_X_train, reshape_y_train, epochs=30000, shuffle=False, callbacks=[ EarlyStopping(monitor='loss', min_delta=0.001, mode='min', patience=30) ], batch_size=1280) evaluate = model.evaluate(reshape_X_test, reshape_y_test) pred = model.predict(validation) old = model.predict(reshape_scaled_train) print(minmax.inverse_transform(pred)) print(evaluate) plt.figure(1) plt.plot(result.history['loss']) plt.title('loss' + str(round(result.history['loss'][-1], 6))) plt.show() plt.figure(2) plt.plot(minmax.inverse_transform(old)) plt.plot(minmax.inverse_transform(target)) plt.show() plt.figure(3) plt.plot(minmax.inverse_transform(pred)) plt.show()
def compile(self): optimizer = Nadam(lr=self.lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) # optimizer = SGD() self.probabilityNetwork = make_parallel(self.probabilityNetwork, 2) self.probabilityNetwork.compile(optimizer=optimizer, loss='mse', metrics=['acc', 'mse'])
def __init__(self, input_shape=(778, 576, 1), num_categories=5, parallel_mode=False, verbose=False): """ https://keras.io/getting-started/functional-api-guide/#multi-input-and-multi-output-models https://keras.io/getting-started/functional-api-guide/#shared-layers https://blog.keras.io/building-autoencoders-in-keras.html https://github.com/fchollet/keras/blob/master/examples/variational_autoencoder_deconv.py """ self.num_categories = num_categories filters = 128 img_rows, img_cols, img_chns = input_shape input_img = Input(shape=input_shape, name="main_input") if verbose: print("Network input shape is", input_img.get_shape()) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(input_img) x = BatchNormalization()(x) x = LeakyReLU(alpha=0.05)(x) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(x) x = BatchNormalization()(x) x = LeakyReLU(alpha=0.05)(x) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(x) x = BatchNormalization()(x) x = LeakyReLU(alpha=0.05)(x) self.framer = Model(input_img, x) optimizer = Nadam(lr=0.0002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004, clipnorm=0.618) self.framer.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['acc'])
def compile(self): # print("LR: ",self.lr) # self.lr = 10**np.random.uniform(-2.2, -3.8) optimizer = Nadam(lr=self.lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) # optimizer = SGD() # self.probabilityNetwork = make_parallel(self.probabilityNetwork, 2) self.probabilityNetwork.compile( optimizer=optimizer, loss='categorical_crossentropy', metrics=['acc', 'mse', 'categorical_crossentropy'])
def res_net(input_shape, hns=[8, 6, 4, 4], classes=2): """ """ inputs = Input(shape=input_shape) x = BatchNormalization()(inputs) x = identity_block(x, hns[0], name='block0', dropout=False) x = identity_block(x, hns[1], name='block1', dropout=False) x = identity_block(x, hns[2], name='block2', dropout=False) #x = identity_block(x, hns[3], name = 'block3', dropout = True) x = Dense(1, name='pre_sigmoid')(x) x = BatchNormalization()(x) proba = Activation('sigmoid')(x) model = Model(inputs, x) model.compile(optimizer=Nadam(lr=0.001), loss='binary_crossentropy') return model
def boosting_parallel_res_net(input_shape, hns = [8, 6, 4, 7], classes = 2): """ """ boost_input = Input(shape=(1,)) # res_module res_shape = (input_shape[0] - 1,) boost_res_net_model = boosting_res_net(input_shape) res_inputs = Input(shape = res_shape) boost_res_net_out_list = [boost_res_net_model([res_inputs, boost_input]) for i in range(8)] boost_res_net_out = concatenate(boost_res_net_out_list, axis = 1) x = Dense(4, activation = 'sigmoid')(boost_res_net_out) proba = Dense(1, activation = 'sigmoid')(x) model = Model([res_inputs, boost_input], proba) model.compile(optimizer=Nadam(lr = 0.001), loss='binary_crossentropy') return model
def boosting_dnn(input_shape, hns = [8, 6, 4, 7], classes = 2): """ """ inputs = Input(input_shape) boost_input = Lambda(lambda x: x[:, -1])(inputs) # dnn_input = Lambda(lambda x: x[:, :-1])(inputs) dnn_input = inputs #dnn_module # dnn_model = create_dnn((input_shape[0] - 1,), hns) dnn_model = create_dnn((input_shape[0],), hns) dnn_pre_sigmoid = Model(dnn_model.input, dnn_model.get_layer('pre_sigmoid').output)(dnn_input) # boost pre_sigmoid = Add(name = 'pre_sigmoid')([dnn_pre_sigmoid, boost_input]) proba = Activation('sigmoid')(pre_sigmoid) model = Model(inputs, proba) model.compile(optimizer=Nadam(lr = 0.001), loss='binary_crossentropy') return model
def boosting_res_net(input_shape, hns = [128, 64, 16, 4], classes = 2, out_layer_name = None): """ """ inputs = Input(input_shape) boost_input = Lambda(lambda x: x[:, -1])(inputs) # res_module res_inputs = Lambda(lambda x: x[:, :-1])(inputs) res_model = res_net((input_shape[0] - 1, ), hns) #res_inputs = inputs #res_model = res_net(input_shape, hns) res_pre_sigmoid = Model(res_model.input, res_model.get_layer('pre_sigmoid').output)(res_inputs) # boost pre_sigmoid = Add(name = 'pre_sigmoid')([res_pre_sigmoid, boost_input]) proba = Activation('sigmoid', name = out_layer_name)(pre_sigmoid) model = Model(inputs, proba) model.compile(optimizer=Nadam(lr = 0.001), loss='binary_crossentropy') return model
def _generate_precache_model(self, batches): if self.model_nontrainable == None: last_non_trainable = None mdl = Sequential() for i in range(len(self.model.layers)): if self.model.layers[i].trainable == True: break last_non_trainable = self.model.layers[i] mdl.add(self.model.layers[i]) if last_non_trainable == None: print( "All layers trainable. Don't know what to cache. Exiting") return None print("Cached model - using output of", last_non_trainable.name) mdl.compile(optimizer=Nadam(), loss='categorical_crossentropy', metrics=['accuracy']) self.model_nontrainable = mdl cnt = batches.full_step_count() batch_data = batches.iter.next() data = [] label = [] for i in range(cnt): batch_data = batches.iter.next() if (i % 10 == 0): print("Loaded {0} datapoints\r".format(i * batches.batch_size), end='', flush=True) data.append(batch_data[0]) label.append(batch_data[1]) imgs = np.concatenate([data[i] for i in range(cnt)]) labels = np.concatenate([label[i] for i in range(cnt)]) y = self.model_nontrainable.predict(imgs, verbose=1) print(y.shape) return {'data': y, 'labels': labels}
x = GlobalAveragePooling2D()(x) x = Dense(units=53, activation='softmax', name='output_predictions')(x) model = Model(inputs=input_image, outputs=x, name='Classifier') # In[3]: parallel_model = multi_gpu_model(model=model, gpus=4) tb = TensorBoard(log_dir='logs', write_graph=True) mc = ModelCheckpoint(filepath='models/top_weights.h5', monitor='val_acc', save_best_only='True', save_weights_only='True', verbose=1) es = EarlyStopping(monitor='val_loss', patience=15, verbose=1) rlr = ReduceLROnPlateau() callbacks = [tb, mc, es, rlr] nadam = Nadam(lr=1e-3) parallel_model.compile(optimizer=nadam, loss='categorical_crossentropy', metrics=['accuracy']) # In[ ]: parallel_model.fit_generator(train_generator, steps_per_epoch=train_steps, epochs=epochs, validation_data=validation_generator, validation_steps=val_steps, workers=8, callbacks=callbacks)
#!/usr/bin/env python3 import tensorflow as tf from tensorflow.python.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint from tensorflow.python.keras.optimizers import Nadam from neural_network_solver.dataset import dataset from neural_network_solver.model import autoencoder dataset_generator = dataset() model = autoencoder() model.compile(optimizer=Nadam(lr=0.001), loss=tf.keras.losses.categorical_crossentropy, metrics=['accuracy']) history = model.fit(dataset_generator, validation_split=0.2, batch_size=128, epochs=999, verbose=1, callbacks=[ EarlyStopping(patience=8, verbose=1), ReduceLROnPlateau(patience=4, min_lr=1e-05), ModelCheckpoint( './neural_network_solver/models/autoencoder.hdfs', monitor='val_loss', verbose=False, save_best_only=True, save_weights_only=False, mode='auto', )
y_i = [0, 1] except Exception as e: print(e) break X.append(x_i) Y.append(y_i) X = [(np.array(x) - np.mean(x)) / np.std(x) for x in X] # comment it to remove normalization X, Y = np.array(X), np.array(Y) X_train, X_test, Y_train, Y_test = create_Xt_Yt(X, Y) opt = Nadam(lr=0.001) model = Sequential() model.add(Dense(64, input_dim=30, activity_regularizer=regularizers.l2(0.01))) model.add(BatchNormalization()) model.add(LeakyReLU()) model.add(Dropout(0.5)) model.add(Dense(16, activity_regularizer=regularizers.l2(0.01))) model.add(BatchNormalization()) model.add(LeakyReLU()) model.add(Dense(2)) model.add(Activation('softmax'))
import common_function from tensorflow.python.keras.preprocessing.image import ImageDataGenerator from tensorflow.python.keras.optimizers import Nadam import os import asyncio from tensorflow.python.keras.callbacks import ModelCheckpoint from tensorflow.python.keras.layers import Dropout, Flatten, Dense, Activation from tensorflow.python.keras.models import load_model, Model # Just disables the warning, doesn't enable AVX/FMA (no GPU) # os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' epochs = 40 l_rate = 1.0e-4 decay = l_rate / epochs nadam = Nadam(lr=l_rate, beta_1=0.9, beta_2=0.999) batch_size = 32 img_width, img_height = 24, 24 path_data_set = './ytd' input_img, merged = model_3_branch.get_model(img_width, img_height) num_train_images = 424961 # training images: 424961 # total images: 605855 file_path = 'tbe_cnn_ytd_nadam.h5' datagen = ImageDataGenerator( rescale=1. / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, )
def __init__(self, input_shape=(64, 64, 1), num_categories=5, parallel_mode=False, verbose=False): """ https://keras.io/getting-started/functional-api-guide/#multi-input-and-multi-output-models https://keras.io/getting-started/functional-api-guide/#shared-layers https://blog.keras.io/building-autoencoders-in-keras.html https://github.com/fchollet/keras/blob/master/examples/variational_autoencoder_deconv.py """ self.num_categories = num_categories # number of filters # convolution kernel size filters = 128 latent_dim = 64 epsilon_std = 1.0 noise_std = .01 img_rows, img_cols, img_chns = input_shape input_img = Input(shape=input_shape, name="main_input") if verbose: print("Network input shape is", input_img.get_shape()) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(input_img) x = LeakyReLU(alpha=0.05)(x) x = BatchNormalization()(x) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(x) x = LeakyReLU(alpha=0.05)(x) x = BatchNormalization()(x) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(x) x = LeakyReLU(alpha=0.05)(x) x = BatchNormalization()(x) x = MaxPooling2D((2, 2), padding='same', name="encoded")(x) conv_output_shape = K.int_shape(x) intermediate_dim = conv_output_shape[1] * conv_output_shape[ 2] * conv_output_shape[3] if verbose: print("Convolution output shape is", conv_output_shape) x = Flatten()(x) hidden = Dense(intermediate_dim, activation='selu')(x) z_mean = Dense(latent_dim)(hidden) z_log_var = Dense(latent_dim)(hidden) def sampling(args): z_mean, z_log_var = args return K.random_normal( shape=K.shape(z_log_var), mean=0., stddev=noise_std) * K.exp( .5 * z_log_var) + z_mean z = Lambda(sampling)([z_mean, z_log_var]) encoding_shape = K.int_shape(z) if verbose: print("Encoding shape is", encoding_shape, "(", latent_dim, "dimensions )") # this model maps an input to its encoded representation encoder = Model(input_img, z) # next, declare the auto_encoding output side n = 0 n += 1 ae = Dense(intermediate_dim, activation='selu')(z) n += 1 ae = Reshape(conv_output_shape[1:])(ae) n += 1 ae = Conv2DTranspose(filters, (3, 3), padding='same')(ae) n += 1 ae = LeakyReLU(alpha=0.05)(ae) n += 1 ae = BatchNormalization()(ae) n += 1 ae = UpSampling2D((2, 2))(ae) n += 1 ae = Conv2DTranspose(filters, (3, 3), padding='same')(ae) n += 1 ae = LeakyReLU(alpha=0.05)(ae) n += 1 ae = BatchNormalization()(ae) n += 1 ae = UpSampling2D((2, 2))(ae) n += 1 ae = Conv2DTranspose(filters, (3, 3), padding='same')(ae) n += 1 ae = LeakyReLU(alpha=0.05)(ae) n += 1 ae = UpSampling2D((2, 2))(ae) n += 1 decoded = Conv2D(1, (2, 2), padding='same')(ae) # activation='sigmoid', if verbose: print("Decoder output shape is", decoded.get_shape()) # this is a pipe from the input image to the reconstructed output autoencoder = Model(input_img, decoded) # use right side of architecture encoded input to construct an image encoded_input = Input(shape=encoding_shape[1:]) deco = encoded_input for l in range(-n, 0): deco = autoencoder.layers[l](deco) decoder = Model(encoded_input, deco) # and then, the classifier n = 0 n += 1 cl = Dense(filters, activation='selu')(z) n += 1 cl = AlphaDropout(0.1)(cl) n += 1 cl = Dense(filters, activation='selu')(cl) n += 1 cl = AlphaDropout(0.1)(cl) n += 1 cl = Dense(filters, activation='selu')(cl) n += 1 classified = Dense(num_categories, activation='softmax')(cl) if verbose: print("Classifier output shape is", classified.get_shape()) # provide classification on images imageclassifier = Model(input_img, classified) # imageclassifier.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) # and classifications on encoded representations encoded_input = Input(shape=(encoding_shape[1:])) fc = encoded_input for l in range(-n, 0): fc = imageclassifier.layers[l](fc) featureclassifier = Model(encoded_input, fc) # featureclassifier.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) # Some KL loss def vae_objective(x, x_decoded): kl_loss = -0.5 * K.sum( 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) base_loss = tf.reduce_sum(metrics.binary_crossentropy( x, x_decoded), axis=[-1, -2]) return base_loss + kl_loss # Finally, compile the full model (1 input, 2 outputs) classycoder = Model(inputs=[input_img], outputs=[decoded, classified]) # = make_parallel(classycoder, 2) # Todo? optimizer = Nadam(lr=0.0002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004, clipnorm=0.618) # = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=1.0) classycoder.compile(optimizer=optimizer, loss=[vae_objective, 'categorical_crossentropy'], loss_weights=[0.1, 0.9], metrics=['acc']) # API mapping self.encoding_dims = latent_dim self.encoding_shape = encoding_shape # Recieve a image and encode it into its latent space representation if parallel_mode: self.encoder = make_parallel(encoder, config.GPUs) else: self.encoder = encoder # reconstructs an image from its encoded representation if parallel_mode: self.decoder = make_parallel(decoder, config.GPUs) else: self.decoder = decoder # direct pipe from input_image to its classification if parallel_mode: self.imageclassifier = make_parallel(imageclassifier, config.GPUs) else: self.imageclassifier = imageclassifier # direct pipe from encoded representation to classification if parallel_mode: self.featureclassifier = make_parallel(featureclassifier, config.GPUs) else: self.featureclassifier = featureclassifier # multiple output model, train this for the rest to work. # todo: make_parallel self.classycoder = classycoder
def compile(self): self.model.compile(optimizer=Nadam(), loss='categorical_crossentropy', metrics=['accuracy']) print("Model compiled")
x = Flatten()(x) x = Dense(32, activation='relu')(x) x = Dense(2, activation='softmax')(x) # UNSW-NB15 is 2 and 10,CICIDS2017 is 15 model = Model(inputs=input_singal, outputs=x) return model model = build_model() model.summary() time_start = time.time() reduce_lr = keras.callbacks.ReduceLROnPlateau(moniter='val_loss', factor=0.1, patience=10) nadam = Nadam(lr=0.008, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) model.compile(loss="categorical_crossentropy", optimizer="nadam", metrics=["accuracy"]) save_dir = os.path.join(os.getcwd(), 'model_save') if not os.path.isdir(save_dir): os.makedirs(save_dir) filename = "model_{epoch:02d}.hdf5" ckpt_callback = tf.keras.callbacks.ModelCheckpoint( os.path.join(save_dir, filename), monitor='val_acc' , verbose=1, save_best_only=False) # checkpoint = tf.train.Checkpoint(model=model) # checkpoint.restore(tf.train.latest_checkpoint("../dataset/SGM-CNN/classification decision/model_save")) history = model.fit(x_train, y_train, epochs=10, batch_size=batch_size, verbose=2,