def build(self, input_shape): in_dim = input_shape[-1] latent_dim = self.latent_dim initializer = RandomNormal(mean=0., stddev=1.) transform_shape = (self.n_heads, in_dim, latent_dim) # projection weights self.query = self.add_weight("query", transform_shape, initializer=initializer, regularizer=l1(1e-6), dtype=np.float32) self.key = self.add_weight("key", transform_shape, initializer=initializer, regularizer=l1(1e-6), dtype=np.float32) self.value = self.add_weight("value", transform_shape, initializer=initializer, regularizer=l1(1e-6), dtype=np.float32) # attention weights self.W = self.add_weight("W", (self.n_heads, latent_dim, 1), initializer=initializer, dtype=np.float32) self.O = self.add_weight("O", (self.n_heads * latent_dim, in_dim), initializer=initializer, dtype=np.float32) self.P = self.add_weight("P", (input_shape[-2] - 1, in_dim), regularizer=l1(1e-6), initializer=initializer, dtype=np.float32) self.bias = self.add_weight("bias", (self.n_heads, ), initializer=initializer, regularizer=l2(1e-6), dtype=np.float32)
def build_model(n_gpu, vocab): with tf.device('/cpu:0'): # Inputs model = Sequential() model.add( Dense(dense_size_1, input_dim=vocab.vocab_size, activation='relu', kernel_regularizer=regularizers.l1(regular_lambda))) model.add( Dense(dense_size_2, activation='relu', kernel_regularizer=regularizers.l1(regular_lambda))) model.add(Dense(1, activation='sigmoid')) if n_gpu > 1: model = multi_gpu_model(model, gpus=n_gpu) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1_score]) model.summary() return model
def get_model(train_dataset): model = Sequential([ Dense(units=32, activation=relu, input_shape=(train_dataset.shape[1], )), Dropout(rate=.5), Dense(units=32, activation=relu, kernel_regularizer=l2(l2=.001), bias_regularizer=l1(l1=.002)), Dropout(rate=.5), Dense(units=32, activation=relu, kernel_regularizer=l1(l1=.001)), Dropout(rate=.5), Dense(units=32, activation=relu, kernel_regularizer=l1_l2(l1=.001, l2=.002)), Dense(units=32, activation=relu, kernel_regularizer=l1(l1=.001)), Dense(units=32, activation=relu, kernel_regularizer=l2(l2=.005)), Dense(units=1) ]) model.summary() return model
def GEN(input_shape=( 64, 64, 3, )): mod = mod_inp = Input(shape=input_shape) mod = Flatten()(mod) mod = Dense( 2, activation="softmax", kernel_regularizer=regularizers.l1(0.001), )(mod) return keras.models.Model(inputs=mod_inp, outputs=mod)
def create_base_network(input_shape): ''' Base network to be shared. ''' input = Input(shape=input_shape) x = Conv2D(32, (7, 7), activation='relu', input_shape=input_shape, kernel_regularizer=regularizers.l2(0.01), bias_regularizer=regularizers.l1(0.01))(input) x = MaxPooling2D()(x) x = Conv2D(64, (3, 3), activation='relu', kernel_regularizer=regularizers.l2(0.01), bias_regularizer=regularizers.l1(0.01))(x) x = Flatten()(x) x = Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.01), bias_regularizer=regularizers.l1(0.01))(x) return Model(input, x)
def dataclassifier(self): baseline = self.baseline x = baseline.output x = Flatten()(x) x = BatchNormalization()(x) x = Dense(32, activation='swish', kernel_regularizer=l1(self.lr_dense))(x) classification = Dense(self.multi_classes, activation='softmax', kernel_regularizer=l1(self.lr_class))(x) optimizer = Adam(lr=.001, amsgrad=True) classification = Model(baseline.input, classification) classification.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['categorical_accuracy']) return classification
def convolutional_block(input_layer, n_filters, activation=None, filter_size=(3, 3, 3), strides=(1, 1, 1), padding='same', data_format='channels_last', batch_normalization=False, dropout=None, l1=None, l2=None, axis=4, use_bias=True, bn_name=None, conv_name=None): """ Block of one convolutional layer with possible activation and batch normalization. :param input_layer: Input layer to the convolution. Keras layer. :param n_filters: Number of filters in the particular convolutional layer. Positive integer. :param activation: Activation_function after every convolution. String activation name. :param filter_size: Size of the convolution filter (kernel). Tuple of 3 positive integers. :param strides: Strides values. Tuple of 3 positive integers. :param padding: Used padding by convolution. Takes values: 'same' or 'valid'. :param data_format: Ordering of the dimensions in the inputs. Takes values: 'channel_first' or 'channel_last'. :param batch_normalization: If set to True, will use Batch Normalization layers after each convolution layer. :param dropout: percentage of weights to be dropped, float between 0 and 1. :param l1: L1 regularization. :param l2: L2 regularization. :param axis: Axis for batch normalization. :param use_bias: :param bn_name: Name of the batch normalization layer. String. :param conv_name: :return: Keras layer. """ if l1 is not None: if l2 is not None: layer = Convolution3D(filters=n_filters, kernel_size=filter_size, strides=strides, padding=padding, data_format=data_format, kernel_regularizer=regularizers.l1_l2(l1=l1, l2=l2), use_bias=use_bias, name=conv_name)(input_layer) else: layer = Convolution3D(filters=n_filters, kernel_size=filter_size, strides=strides, padding=padding, data_format=data_format, kernel_regularizer=regularizers.l1(l1), use_bias=use_bias, name=conv_name)(input_layer) else: if l2 is not None: layer = Convolution3D(filters=n_filters, kernel_size=filter_size, strides=strides, padding=padding, data_format=data_format, kernel_regularizer=regularizers.l2(l2), use_bias=use_bias, name=conv_name)( input_layer) else: layer = Convolution3D(filters=n_filters, kernel_size=filter_size, strides=strides, padding=padding, data_format=data_format, use_bias=use_bias, name=conv_name)(input_layer) if batch_normalization: layer = BatchNormalization(axis=axis, name=bn_name)( layer) # the axis that should be normalized (typically the features axis), integer. # Layer input shape: (samples, conv_dim1, conv_dim2, conv_dim3, channels)` if data_format='channels_last' if activation is not None: layer = Activation(activation=activation)(layer) if dropout is not None: layer = Dropout(dropout)(layer) return layer
def __init__( self, forecastHorizon=1, lookBack=32, architecture=[32], dropout=0.0, regularization=0.0, loadPath=None, bidirectional=True, convInputLayer=False, modelName="LSTM", kernelSize=1, filters=5, poolSize=2, ): np.random.seed(1) self.modelType = "LSTM" self.modelName = modelName self.forecastHorizon = forecastHorizon self.lookBack = lookBack self.architecture = architecture self.dropout = dropout self.regularization = l1(regularization) self.bidirectional = bidirectional self.convInputLayer = convInputLayer self.kernelSize = kernelSize self.filters = filters self.batchSize = 128 self.trainingLossEvaluation = [] self.poolSize = poolSize if loadPath is not None: self.model = load_model(loadPath) try: self.lookBack = self.model.layers[0].output_shape[0][1] except: self.lookBack = self.model.layers[0].output_shape[1] try: self.noTimeSeries = self.model.layers[0].input_shape[0][2] except: self.noTimeSeries = self.model.layers[0].input_shape[2] for layer in self.model.layers: if hasattr(layer, "rate"): self.dropout = layer.rate if loadPath.endswith(".h5"): self.modelName = loadPath[9:-3] print("Model Loaded")
def get_keras_model(inputDim, hiddenDim=128, encodeDim=8, batchNorm=True, qBatchNorm=False, l1reg=0, input_batchNorm=False, halfcode_layers=4, fan_in_out=64, **kwargs): """ define the keras model the model based on the simple dense auto encoder (128*128*128*128*8*128*128*128*128) """ # Declare encode network inputLayer = Input(shape=(inputDim, )) kwargs = {'kernel_regularizer': l1(l1reg)} for i in range(halfcode_layers): if i == 0: h = Dense(fan_in_out, **kwargs)(inputLayer) else: h = Dense(hiddenDim, **kwargs)(h) if batchNorm: h = BatchNormalization()(h) h = Activation('relu')(h) #Declare latent layer if halfcode_layers == 0: h = Dense(encodeDim, **kwargs)(inputLayer) else: h = Dense(encodeDim, **kwargs)(h) if batchNorm: h = BatchNormalization()(h) h = Activation('relu')(h) # Declare decoder network for i in range(halfcode_layers): if i == halfcode_layers - 1: h = Dense(fan_in_out, **kwargs)(h) else: h = Dense(hiddenDim, **kwargs)(h) if batchNorm: h = BatchNormalization()(h) h = Activation('relu')(h) h = Dense(inputDim, **kwargs)(h) return Model(inputs=inputLayer, outputs=h)
def construct_model(): model = tf.keras.models.Sequential() #model.add(layers.InputLayer(input_shape=(train_set.shape[1],train_set.shape[2],train_set.shape[3]), batch_size=(BATCHSIZE))) model.add( layers.Conv2D(filters=3, kernel_size=(3, 3), padding="same", input_shape=(train_set[0].shape))) model.add(layers.BatchNormalization()) model.add(layers.Activation('relu')) model.add( layers.Conv2D(filters=16, kernel_size=(3, 3), strides=(2, 2), padding='same')) model.add(layers.BatchNormalization()) model.add(layers.Activation('relu')) model.add(layers.MaxPool2D((2, 2))) model.add( layers.Conv2D(filters=32, kernel_size=(3, 3), strides=(2, 2), padding='same')) model.add(layers.BatchNormalization()) model.add(layers.Activation('relu')) model.add(layers.MaxPool2D((2, 2))) model.add( layers.Conv2D(filters=48, kernel_size=(3, 3), padding='same', strides=(2, 2))) model.add(layers.BatchNormalization()) model.add(layers.Activation('relu')) model.add(layers.GlobalAveragePooling2D()) model.add(layers.Flatten()) model.add(layers.Dense(8, kernel_regularizer=(regularizers.l1(0)))) model.add(layers.Activation('relu')) model.add(layers.Dense(3)) model.add(layers.Activation('softmax')) return model
def create_model(ModelInfo): ### Defining reguralization technique # reg = l1(0.001) model = keras.Sequential() model.add( layers.Dense( ModelInfo['Nerouns'][0], input_shape=[7], activation=ModelInfo['Activation_Method'][0], # kernel_initializer =ModelInfo['W_Initialization_Method'][0], # bias_initializer = ModelInfo['W_Initialization_Method'][0], # activity_regularizer=ModelInfo['Reguralization'][0], activity_regularizer=l1(ModelInfo['Reguralization'][0]), # kernel_constraint=ModelInfo['kernel_constraint'][0])), kernel_constraint=max_norm(ModelInfo['kernel_constraint'][0]))), model.add(layers.Dropout(ModelInfo['Dropout_Value'][0])), for c in range(1, ModelInfo['Layers'][0]): print('Index=', c) model.add( layers.Dense( ModelInfo['Nerouns'][c], activation=tf.nn.relu, # kernel_initializer =ModelInfo['W_Initialization_Method'][0], # bias_initializer = ModelInfo['W_Initialization_Method'][0], # activity_regularizer=ModelInfo['Reguralization'][c], activity_regularizer=l1(ModelInfo['Reguralization'][c]), # kernel_constraint=ModelInfo['kernel_constraint'][c])), kernel_constraint=max_norm( ModelInfo['kernel_constraint'][c]))), model.add(layers.Dropout(ModelInfo['Dropout_Value'][c])), # model.add(layers.BatchNormalization()), model.add(layers.Dense(1, activation=ModelInfo['Activation_Method'][0])), return model
def create_net(l1_regularization=None): # The regularization: regularizer = l1(l1_regularization) if l1_regularization is not None else None # The encoder: input_img = Input(shape=(28, 28, 1)) # 784-dimensional data x = Conv2D(16, (3, 3), activation='relu', padding='same')(input_img) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(8, (3, 3), activation='relu', padding='same')(x) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(8, (3, 3), activation='relu', padding='same')(x) x = MaxPooling2D((2, 2), padding='same')(x) encoded = Conv2D(8, (3, 3), activation='relu', padding='same', activity_regularizer=regularizer)(x) encoder = Model(input_img, encoded) # Features' shape is (4,4,8), i.e. 128-dimensional. # The decoder layers: decoder_input = Conv2D(8, (3, 3), activation='relu', padding='same') decoder_up01 = UpSampling2D((2, 2)) decoder_conv01 = Conv2D(8, (3, 3), activation='relu', padding='same') decoder_up02 = UpSampling2D((2, 2)) decoder_conv02 = Conv2D(16, (3, 3), activation='relu') decoder_up03 = UpSampling2D((2, 2)) decoder_conv03 = Conv2D(1, (3, 3), activation='sigmoid', padding='same') def decoder_from(enc_in): x = decoder_input(enc_in) x = decoder_up01(x) x = decoder_conv01(x) x = decoder_up02(x) x = decoder_conv02(x) x = decoder_up03(x) return decoder_conv03(x) # The decoder: encoded_img = Input(shape=(4,4,8)) decoded_test = decoder_from(encoded_img) decoder = Model(encoded_img,decoded_test) # The whole autoencoder: decoded = decoder_from(encoded) autoencoder = Model(input_img, decoded) # The optimization: autoencoder.compile(optimizer='adam', loss='binary_crossentropy') # Returning all: return autoencoder, encoder, decoder
def run(input_dim, encoding_dim): #Input() is used to instantiate a Keras tensor input_layer = Input(shape=(input_dim, )) encoder = Dense(encoding_dim, activation="tanh", activity_regularizer=regularizers.l1(10e-5))(input_layer) decoder = Dense(input_dim, activation='relu')(encoder) #you either use a keras model or build one yourself as we do here autoencoder = Model(inputs=input_layer, outputs=decoder) return autoencoder
def learn(self, Features, Targets=None): num_batches = 5 if Targets is not None: raise Exception("Per definition Target is equal to Features") if self.encoding_dim is np.NaN: self.encoding_dim = Features.shape[1] if self.input_dim is np.NaN: self.input_dim = (1, self.encoding_dim) print(self.encoding_dim, ' ', self.input_dim) # Stupid identity function is to learn np.random.seed(7) input_layer = Input(shape=self.encoding_dim) encoder = Dense( self.encoding_dim, activation="tanh", activity_regularizer=regularizers.l1(10e-5))(input_layer) encoder = Dense(int(self.encoding_dim / 2), activation="relu")(encoder) decoder = Dense(int(self.encoding_dim / 2), activation='relu')(encoder) decoder = Dense(self.encoding_dim, activation='tanh')(decoder) model = Model(inputs=input_layer, outputs=decoder) model.compile(optimizer='rmsprop', loss=['mse', 'categorical_crossentropy'], loss_weights=[1.0, 20.0], metrics=['mean_squared_error']) if os.path.isfile(self.name) and self.forcelearning == False: # Restore the weights model.load_weights(self.name) else: checkpointer = ModelCheckpoint(filepath=self.name, verbose=0, save_best_only=True) # Autoencoders learn the identiy, due to that X = X model.fit(Features, Features, epochs=100, batch_size=num_batches, verbose=1, callbacks=[checkpointer]) # Save the weights model.save_weights(self.name) self.model = model
def __init__(self, X_train_shape): self.input_shape = (X_train_shape[1], X_train_shape[2]) self.model = Sequential() self.model.add( LSTM(units=30, input_shape=self.input_shape, activation='tanh', recurrent_activation='sigmoid', use_bias=True, return_sequences=True, kernel_regularizer=regularizers.l1(1e-05))) self.model.add( LSTM(units=20, input_shape=self.input_shape, activation='tanh', recurrent_activation='sigmoid', use_bias=True, return_sequences=False, kernel_regularizer=regularizers.l1(1e-05))) self.model.add( Dense(units=10, kernel_regularizer=regularizers.l1(1e-05))) self.model.add(ELU(alpha=1)) self.model.add(Dense(units=1, activation='relu'))
def create_model(num_input, hidden_layer, num_classes): input_layer = tf.keras.Input(shape=(num_input, )) hidden = layers.Dense( hidden_layer[0], kernel_regularizer=regularizers.l2(0.1), activity_regularizer=regularizers.l1(0.1))(input_layer) hidden = layers.ELU(alpha=1)(hidden) hidden = layers.Dropout(0.1)(hidden) hidden = layers.Dense(hidden_layer[1], kernel_regularizer=regularizers.l2(0.1), activity_regularizer=regularizers.l1(0.1))(hidden) hidden = layers.ELU(alpha=1)(hidden) hidden = layers.Dropout(0.1)(hidden) hidden = layers.Dense(hidden_layer[2], kernel_regularizer=regularizers.l2(0.1), activity_regularizer=regularizers.l1(0.1))(hidden) hidden = layers.ELU(alpha=1)(hidden) hidden = layers.Dropout(0.1)(hidden) if num_classes > 2: output_layer = layers.Dense(num_classes, activation="softmax")(hidden) else: output_layer = layers.Dense(1, activation="sigmoid")(hidden) model = models.Model(input_layer, output_layer) if num_classes > 2: loss_func = 'sparse_categorical_crossentropy' else: loss_func = 'binary_crossentropy' opt = optimizers.Adam(learning_rate=0.005, beta_1=0.9, beta_2=0.999, amsgrad=False) model.compile(optimizer=opt, loss=loss_func, metrics=['accuracy']) return model
def create_model(): model = Sequential() model.add( Conv2D(8, (4, 4), strides=(1, 1), input_shape=(32, 32, 1), name="conv2d_0_m")) model.add(Activation(activation='relu', name='relu1')) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='max1')) model.add(Conv2D(16, (2, 2), strides=(1, 1), name="conv2d_1_m")) model.add(Activation(activation='relu', name='relu2')) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='max2')) model.add(Flatten()) model.add( Dense(120, name='fc1', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001))) model.add(Activation(activation='relu', name='relu3')) model.add( Dense(84, name='fc2', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001))) model.add(Activation(activation='relu', name='relu4')) model.add( Dense(10, name='output', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001))) model.add(Activation(activation='softmax', name='softmax')) return model
def Relearn3(): # Load pickle model x = pickle.load(open("Data/x.pickle", "rb")) y = pickle.load(open("Data/y.pickle", "rb")) # normalizing data x = np.asarray(x) / 255.0 y = np.asarray(y) # Building the model model = Sequential() # 3 convolutional layersy model.add(Conv2D(32, (3, 3), input_shape=x.shape[1:])) model.add(Activation("relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(64, (3, 3))) model.add(Activation("relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(64, (3, 3))) model.add(Activation("relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) # 1 hidden layers model.add(Flatten()) model.add(Dense(32, activity_regularizer=l1(0.001))) model.add(Activation("relu")) model.add(Dropout(0.5)) #model.add(Dense(128)) #model.add(Activation("relu")) # The output layer with 5 neurons, for 5 classes model.add(Dense(5)) model.add(Activation("softmax")) # Compiling the model using some basic parameters model.compile(loss="sparse_categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) # validation_split corresponds to the percentage of images used for the validation phase compared to all the images history = model.fit(x, y, batch_size=50, epochs=50, validation_split=0.05) # Saving the model as .h5 and .model model.save("Data/modelweight.h5") model.save("Data/CNN.model")
def cnn(l1, l2, depth, gamma, lr, w): model = models.Sequential() model.add( layers.Conv1D(l1, w, activation='relu', kernel_regularizer=regularizers.l1(gamma), input_shape=(40, depth), padding='same')) model.add(layers.MaxPooling1D(2)) model.add(layers.Flatten()) model.add(layers.Dropout(0.5)) model.add( layers.Dense(l2, activation='relu', kernel_regularizer=regularizers.l1(gamma))) model.add( layers.Dense(1, activation='sigmoid', kernel_regularizer=regularizers.l1(gamma))) model.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=lr), metrics=['acc']) return model
def old_unet(input_size, num_classes, num_channels=1, filter_multiplier=10, regularization_rate=0.): input_ = Input((input_size, input_size, num_channels)) skips = [] output = input_ num_layers = int(np.floor(np.log2(input_size))) down_conv_kernel_sizes = np.zeros([num_layers], dtype=int) down_filter_numbers = np.zeros([num_layers], dtype=int) up_conv_kernel_sizes = np.zeros([num_layers], dtype=int) up_filter_numbers = np.zeros([num_layers], dtype=int) for layer_index in range(num_layers): down_conv_kernel_sizes[layer_index] = int(3) down_filter_numbers[layer_index] = int((layer_index + 1) * filter_multiplier + num_classes) up_conv_kernel_sizes[layer_index] = int(4) up_filter_numbers[layer_index] = int((num_layers - layer_index - 1) * filter_multiplier + num_classes) for shape, filters in zip(down_conv_kernel_sizes, down_filter_numbers): skips.append(output) output = Conv2D(filters, (shape, shape), strides=2, padding="same", activation="relu", bias_regularizer=l1(regularization_rate))(output) for shape, filters in zip(up_conv_kernel_sizes, up_filter_numbers): output = UpSampling2D()(output) skip_output = skips.pop() output = concatenate([output, skip_output], axis=3) if filters != num_classes: output = Conv2D(filters, (shape, shape), activation="relu", padding="same", bias_regularizer=l1(regularization_rate))(output) output = BatchNormalization(momentum=.9)(output) else: output = Conv2D(filters, (shape, shape), activation="softmax", padding="same", bias_regularizer=l1(regularization_rate))(output) assert len(skips) == 0 return Model([input_], [output])
def model(self): """Model定义及训练 """ log('[{time}] Building model...'.format(time=get_time())) model = Sequential() model.add( Dense(self._dense_size_1, input_dim=len(self._feature_train[0]), activation='relu', kernel_regularizer=regularizers.l1(self._regular))) model.add( Dense(self._dense_size_2, activation='relu', kernel_regularizer=regularizers.l1(self._regular))) model.add(Dense(1, activation='sigmoid')) # if os.path.exists(self.model_file): # model.load_weights(self.model_file) model.compile( loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', self.precision, self.recall, self.f1_score]) log(model.summary()) # checkpoint checkpoint = ModelCheckpoint(self._model_file + '.{epoch:03d}-{val_f1_score:.4f}.hdf5', monitor='val_f1_score', verbose=1, save_best_only=True, mode='max') callbacks_list = [checkpoint] log('[{time}] Training...'.format(time=get_time())) model.fit(self._feature_train, self._label_train, epochs=self._epoch, callbacks=callbacks_list, validation_data=(self._feature_test, self._label_test))
def get_regularizers(self, l2, l1): if self.regularizer == 'l2': kernel_regularizer = regularizers.l2(l2), print('[INFO] -- regularizers: l2\n') elif self.regularizer == 'l1': kernel_regularizer = regularizers.l1(l1), print('[INFO] -- regularizers: l1\n') elif self.regularizer == 'l2_l1': kernel_regularizer = regularizers.l1_l2(l1=l1, l2=l2), print('[INFO] -- regularizers: l1_l2\n') else: kernel_regularizer = None print('[INFO] -- regularizers: None \n') return kernel_regularizer
def model_2grus(Inputs, Inputs_alt, X_train, Xalt_train, Y_train, NPARTS=20, NSV=5): CLR = 0.001 L1R = 0.00001 print(Inputs) print(Inputs_alt) gru = GRU(100, activation='relu', recurrent_activation='hard_sigmoid', name='gru_base', activity_regularizer=l1(L1R))(Inputs) dense = Dense(100, activation='relu', activity_regularizer=l1(L1R))(gru) norm = BatchNormalization(momentum=0.6, name='dense4_bnorm')(dense) gru_alt = GRU(100, activation='relu', recurrent_activation='hard_sigmoid', name='gru_base_alt', activity_regularizer=l1(L1R))(Inputs_alt) dense_alt = Dense(100, activation='relu', activity_regularizer=l1(L1R))(gru_alt) norm_alt = BatchNormalization(momentum=0.6, name='dense4_bnorm_alt')(dense_alt) added = Add()([norm, norm_alt]) dense = Dense(50, activation='relu', activity_regularizer=l1(L1R))(added) norm = BatchNormalization(momentum=0.6, name='dense5_bnorm')(dense) dense = Dense(20, activation='relu', activity_regularizer=l1(L1R))(norm) dense = Dense(10, activation='relu', activity_regularizer=l1(L1R))(dense) out = Dense(1, activation='sigmoid', activity_regularizer=l1(L1R))(norm) classifier = Model(inputs=[Inputs, Inputs_alt], outputs=[out]) lossfunction = 'binary_crossentropy' classifier.compile(loss=[lossfunction], optimizer=Adam(CLR), metrics=['accuracy']) models = {'classifier': classifier} return models
def create(self) -> tf.keras.models.Model: input_size = self.num_features * self.window_size return tf.keras.models.Sequential([ InputLayer((input_size, ), name='input'), Dense(128, activation='relu', name='enc/dense01'), Dense(64, activation='relu', name='enc/dense02'), Dense(16, activation='relu', name='enc/dense03', activity_regularizer=l1(10e-6)), Dense(64, activation='relu', name='dec/dense01'), Dense(128, activation='relu', name='dec/dense02'), Dense(input_size, activation=None, name='dec/dense03') ], name=self.name)
def conv2D_block(num_filters=32, sample_shape=None, inputA=None, drop_out=0, first_layer=False, conv_kernel_size=(5, 5), conv_stride=(1, 1), GAP=True, max_pool_size=(2, 2), max_pool_stride=(1, 1), act_regularization=0): if first_layer: # inputA = Input(shape=sample_shape) conv = Conv2D( num_filters, kernel_size=conv_kernel_size, strides=conv_stride, kernel_initializer='he_uniform', # kernel_regularizer=l1(act_regularization), # bias_regularizer=l1(act_regularization), # activity_regularizer=None, activation=None, padding='same', use_bias=False, input_shape=sample_shape)(inputA) else: conv = Conv2D( num_filters, kernel_size=conv_kernel_size, strides=conv_stride, kernel_initializer='he_uniform', # kernel_regularizer=l1(act_regularization), # bias_regularizer=l1(act_regularization), # activity_regularizer=None, activation=None, padding='same', use_bias=False)(inputA) bn = BatchNormalization(center=True, scale=True, trainable=False)(conv) act = Activation('relu', activity_regularizer=l1(act_regularization))(bn) if GAP: out = GlobalAveragePooling2D()(act) else: max_pool = MaxPooling2D(pool_size=max_pool_size, strides=max_pool_stride)(act) out = Dropout(drop_out)(max_pool) return out
def convolutional_model(input_shape, class_names): model = tf.keras.models.Sequential([ layers.Conv2D(32, (3, 3), activation=layers.LeakyReLU(alpha=0.01), input_shape=input_shape), layers.Conv2D(32, (3, 3), activation=layers.LeakyReLU(alpha=0.01)), #layers.BatchNormalization(), layers.MaxPooling2D((2, 2)), layers.Dropout(0.1), layers.Conv2D(64, (3, 3), activation=layers.LeakyReLU(alpha=0.01)), layers.Conv2D(64, (3, 3), activation=layers.LeakyReLU(alpha=0.01)), #layers.BatchNormalization(), layers.MaxPooling2D((2, 2)), layers.Dropout(0.1), layers.Conv2D(128, (3, 3), activation=layers.LeakyReLU(alpha=0.01)), layers.Conv2D(128, (3, 3), activation=layers.LeakyReLU(alpha=0.01)), #layers.BatchNormalization(), layers.MaxPooling2D((2, 2)), layers.Dropout(0.1), layers.Conv2D(256, (3, 3), activation=layers.LeakyReLU(alpha=0.01)), layers.Conv2D(256, (3, 3), activation=layers.LeakyReLU(alpha=0.01)), #layers.BatchNormalization(), layers.MaxPooling2D((2, 2)), layers.Dropout(0.1), layers.Flatten(), layers.Dense(1024, activation='linear', activity_regularizer=regularizers.l1(0.0001)), layers.LeakyReLU(alpha=0.01), layers.Dense(128, activation=layers.LeakyReLU(alpha=0.01)), layers.Dense(64, activation=layers.LeakyReLU(alpha=0.01)), layers.Dense(8, activation='softmax') ]) # https://keras.io/optimizers/ #decay_rate = 0.01 / 100. #learning_rate / epochs #sgd = optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=False) #, decay=decay_rate) #rmsprop = optimizers.RMSprop(learning_rate=0.001, rho=0.9) #adagrad = optimizers.Adagrad(learning_rate=0.01) #adadelta = optimizers.Adadelta(learning_rate=1.0, rho=0.95) #adam = optimizers.Adam(learning_rate=0.0001, beta_1=0.9, beta_2=0.999, amsgrad=False) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) return model, "conv_model", 500
def initialize_model(self, regularization_type=None): r, lr, dim, dropout, dp = self.model_parameters # model parameters is a tuple:(r=regularization_parameters,lr=learning rate for ADAM, dim=number and dimension of hidden layers, dropout=boolean if dropout is used in trainig, dp=dropout rate) dim = [int(d) for d in dim] number_of_hidden_layers = len(dim) dropout = bool(dropout) # -------------------------------------------------- NN Architecture -------------------------------------------------# # define input layer inputs = layers.Input(shape=(self.X_train.shape[1], )) # set regularization if regularization_type == 'l2' or regularization_type is None: REG = regularizers.l2(r) logging.debug('l2 regularization') if regularization_type == 'l1': REG = regularizers.l1(r) logging.debug('l1 regularization') if regularization_type == 'l1_l2': logging.debug('l1&l2 regularization') REG = regularizers.l1_l2(r) # first hidden layer x = layers.Dense(dim[0], kernel_regularizer=REG, bias_regularizer=REG, activation='relu')(inputs) if dropout is True: x = layers.Dropout(rate=dp)(x) # remaining hidden layer for k in range(1, number_of_hidden_layers): x = layers.Dense(dim[k], kernel_regularizer=REG, bias_regularizer=REG, activation='relu')(x) if dropout is True: x = layers.Dropout(rate=dp)(x) # final output layer predictions = layers.Dense(1, activation='relu')(x) model = models.Model(inputs=inputs, outputs=predictions) # ADAM = adaptive moment estimation a first-order gradient-based optimization algorithm ADAM = optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, decay=0.0, amsgrad=False) # compile the model and define the loss function model.compile(optimizer=ADAM, loss='mean_absolute_error') # -------------------------------------------------- NN Architecture -------------------------------------------------# self.model = model logging.debug('Neural Net initialized')
def SingleOutputCNN( input_shape, output_shape, cnns_per_maxpool=1, maxpool_layers=1, dense_layers=1, dense_units=64, dropout=0.25, regularization=False, global_maxpool=False, name='', ) -> Model: function_name = cast(types.FrameType, inspect.currentframe()).f_code.co_name model_name = f"{function_name}-{name}" if name else function_name # model_name = seq([ function_name, name ]).filter(lambda x: x).make_string("-") # remove dependency on pyfunctional - not in Kaggle repo without internet inputs = Input(shape=input_shape) x = inputs for cnn1 in range(0,maxpool_layers): for cnn2 in range(1, cnns_per_maxpool+1): x = Conv2D( 32 * cnn2, kernel_size=(3, 3), padding='same', activation='relu')(x) x = MaxPooling2D(pool_size=(2, 2))(x) x = BatchNormalization()(x) x = Dropout(dropout)(x) if global_maxpool: x = GlobalMaxPooling2D()(x) x = Flatten()(x) for nn1 in range(0,dense_layers): if regularization: x = Dense(dense_units, activation='relu', kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))(x) else: x = Dense(dense_units, activation='relu')(x) x = BatchNormalization()(x) x = Dropout(dropout)(x) x = Dense(output_shape, activation='softmax')(x) model = Model(inputs, x, name=model_name) # plot_model(model, to_file=os.path.join(os.path.dirname(__file__), f"{name}.png")) return model
def pixleated_structre(): Input_img = Input(shape=(256, 256, 3)) x1 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l1(10e-10))(Input_img) x2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l1(10e-10))(x1) x3 = MaxPool2D(padding='same')(x2) x4 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l1(10e-10))(x3) x5 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l1(10e-10))(x4) x6 = MaxPool2D(padding='same')(x5) encoded = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l1(10e-10))(x6) x7 = UpSampling2D()(encoded) x8 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l1(10e-10))(x7) x9 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l1(10e-10))(x8) x10 = Add()([x5, x9]) x11 = UpSampling2D()(x10) x12 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l1(10e-10))(x11) x13 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l1(10e-10))(x12) x14 = Add()([x2, x13]) decoded = Conv2D(3, (3, 3), padding='same',activation='relu', kernel_regularizer=regularizers.l1(10e-10))(x14) autoencoder = Model(Input_img, decoded) autoencoder.compile(optimizer='adadelta', loss='mse', metrics=['accuracy']) return autoencoder
def build_model(self): self.h_units = cal_hidden_layer_of_units(self.params['h_layers'], self.params['encode_dim']) self.input = Input(shape=(len(self.features), )) self.hidden = self.input for _unit in self.h_units: self.hidden = Dense(_unit, activation=self.params['activation'], activity_regularizer=regularizers.l1( self.params['l1']))(self.hidden) self.fr_output = Dense(len(self.features), activation='relu')(self.hidden) self.model = Model(inputs=self.input, outputs=self.fr_output) self.model.compile(loss='mae', optimizer=RMSprop(lr=self.params['lr']), metrics=['mae']) print(self.model.summary())