def encoder_model(model): new_input = model.input new_output = model.layers[-2].output img_encoder = Model(new_input, new_output) return img_encoder
def get_model2(input_shape): in_x = Input(input_shape) X = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same', activation='relu')(in_x) X = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X) X = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same', activation='relu')(X) X = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X) X = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same', activation='relu')(X) X = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X) X = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same', activation='relu')(X) X = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X) #################################################################### X1 = Conv2D(32, kernel_size=(5, 5), strides=(1, 1), padding='same', activation='relu')(in_x) X1 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X1) X1 = Conv2D(32, kernel_size=(5, 5), strides=(1, 1), padding='same', activation='relu')(X1) X1 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X1) X1 = Conv2D(32, kernel_size=(5, 5), strides=(1, 1), padding='same', activation='relu')(X1) X1 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X1) X1 = Conv2D(32, kernel_size=(5, 5), strides=(1, 1), padding='same', activation='relu')(X1) X1 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X1) # #################################################################### X2 = Conv2D(32, kernel_size=(2, 2), strides=(1, 1), padding='same', activation='relu')(in_x) X2 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X2) X2 = Conv2D(32, kernel_size=(2, 2), strides=(1, 1), padding='same', activation='relu')(X2) X2 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X2) X2 = Conv2D(32, kernel_size=(2, 2), strides=(1, 1), padding='same', activation='relu')(X2) X2 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X2) X2 = Conv2D(32, kernel_size=(2, 2), strides=(1, 1), padding='same', activation='relu')(X2) X2 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X2) # #################################################################### X3 = Conv2D(32, kernel_size=(4, 4), strides=(1, 1), padding='same', activation='relu')(in_x) X3 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X3) X3 = Conv2D(32, kernel_size=(4, 4), strides=(1, 1), padding='same', activation='relu')(X3) X3 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X3) X3 = Conv2D(32, kernel_size=(4, 4), strides=(1, 1), padding='same', activation='relu')(X3) X3 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X3) X3 = Conv2D(32, kernel_size=(4, 4), strides=(1, 1), padding='same', activation='relu')(X3) X3 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X3) X = concatenate([X, X1, X3], axis=3) print(X) X = Flatten()(X) X = Dense(96, activation='relu')(X) X = Dropout(0.4)(X) X = Dense(64, activation='relu')(X) X = Dropout(0.4)(X) X = Dense(32, activation='relu')(X) X = Dropout(0.4)(X) X = Dense(17, activation='softmax')(X) model = Model(inputs=in_x, outputs=X) return model
l = 100 t = [(i - 1) / (l - 1) for i in range(1, l + 1)] bx = normalize([2 * math.pi * i - math.sin(2 * math.pi * i) for i in t]) by = normalize([1 - math.cos(2 * math.pi * i) for i in t]) train = [[bx[i], by[i]] for i in range(len(bx))] # COMPILATION k = list() for n in [9, 10, 11]: for i in range(5): inp = Input(shape=(1, )) n_units = n out1 = first_layer(units=n_units)(inp) out2 = second_layer(units=n_units - 2)(out1) model = Model(inputs=inp, outputs=out2) opt = tf.keras.optimizers.Adamax( learning_rate=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Adamax', ) model.compile(loss='mae', optimizer=opt) # TRAINING history = model.fit(t, train, batch_size=1, epochs=4000, verbose=0) k.append(history.history['loss'][-1]) # SAVING THE MODEL
# In[5]: from tensorflow.keras.optimizers import RMSprop # Flatten the output layer to 1 dimension x = layers.Flatten()(last_output) # Add a fully connected layer with 1,024 hidden units and ReLU activation x = layers.Dense(1024, activation='relu')(x) # Add a dropout rate of 0.2 x = layers.Dropout(.2)(x) # Add a final sigmoid layer for classification x = layers.Dense(1, activation='sigmoid')(x) model = Model(pre_trained_model.input, x) model.compile(optimizer=RMSprop(lr=0.0001), loss='binary_crossentropy', metrics=['accuracy']) model.summary() # Expected output will be large. Last few lines should be: # mixed7 (Concatenate) (None, 7, 7, 768) 0 activation_248[0][0] # activation_251[0][0] # activation_256[0][0] # activation_257[0][0] # __________________________________________________________________________________________________ # flatten_4 (Flatten) (None, 37632) 0 mixed7[0][0]
def U_Net_3D(image_shape, activation='elu', feature_maps=[32, 64, 128, 256], depth=3, drop_values=[0.1, 0.1, 0.1, 0.1], spatial_dropout=False, batch_norm=False, k_init='he_normal', z_down=2, loss_type="bce", optimizer="sgd", lr=0.001, n_classes=1): """Create 3D U-Net. Parameters ---------- image_shape : 3D tuple Dimensions of the input image. activation : str, optional Keras available activation type. feature_maps : array of ints, optional Feature maps to use on each level. Must have the same length as the ``depth+1``. depth : int, optional Depth of the network. drop_values : float, optional Dropout value to be fixed. spatial_dropout : bool, optional Use spatial dropout instead of the `normal` dropout. batch_norm : bool, optional Make batch normalization. k_init : string, optional Kernel initialization for convolutional layers. z_down : int, optional Downsampling used in z dimension. Set it to 1 if the dataset is not isotropic. loss_type : str, optional Loss type to use, three type available: ``bce`` (Binary Cross Entropy) , ``w_bce`` (Weighted BCE, based on weigth maps) and ``w_bce_dice`` (Weighted loss: ``weight1*BCE + weight2*Dice``). optimizer : str, optional Optimizer used to minimize the loss function. Posible options: ``sgd`` or ``adam``. lr : float, optional Learning rate value. n_classes: int, optional Number of classes. Returns ------- model : Keras model Model containing the U-Net. Calling this function with its default parameters returns the following network: .. image:: img/unet_3d.png :width: 100% :align: center Image created with `PlotNeuralNet <https://github.com/HarisIqbal88/PlotNeuralNet>`_. """ if len(feature_maps) != depth + 1: raise ValueError("feature_maps dimension must be equal depth+1") if len(drop_values) != depth + 1: raise ValueError("'drop_values' dimension must be equal depth+1") dinamic_dim = (None, ) * (len(image_shape) - 1) + (image_shape[-1], ) inputs = Input(dinamic_dim) x = inputs #x = Input(image_shape) #inputs = x if loss_type == "w_bce": weights = Input(image_shape) # List used to access layers easily to make the skip connections of the U-Net l = [] # ENCODER for i in range(depth): x = Conv3D(feature_maps[i], (3, 3, 3), activation=None, kernel_initializer=k_init, padding='same')(x) x = BatchNormalization()(x) if batch_norm else x x = Activation(activation)(x) if spatial_dropout and drop_values[i] > 0: x = SpatialDropout3D(drop_values[i])(x) elif drop_values[i] > 0 and not spatial_dropout: x = Dropout(drop_values[i])(x) x = Conv3D(feature_maps[i], (3, 3, 3), activation=None, kernel_initializer=k_init, padding='same')(x) x = BatchNormalization()(x) if batch_norm else x x = Activation(activation)(x) l.append(x) x = MaxPooling3D((2, 2, z_down))(x) # BOTTLENECK x = Conv3D(feature_maps[depth], (3, 3, 3), activation=None, kernel_initializer=k_init, padding='same')(x) x = BatchNormalization()(x) if batch_norm else x x = Activation(activation)(x) if spatial_dropout and drop_values[depth] > 0: x = SpatialDropout3D(drop_values[depth])(x) elif drop_values[depth] > 0 and not spatial_dropout: x = Dropout(drop_values[depth])(x) x = Conv3D(feature_maps[depth], (3, 3, 3), activation=None, kernel_initializer=k_init, padding='same')(x) x = BatchNormalization()(x) if batch_norm else x x = Activation(activation)(x) # DECODER for i in range(depth - 1, -1, -1): x = Conv3DTranspose(feature_maps[i], (2, 2, 2), strides=(2, 2, z_down), padding='same')(x) x = concatenate([x, l[i]]) x = Conv3D(feature_maps[i], (3, 3, 3), activation=None, kernel_initializer=k_init, padding='same')(x) x = BatchNormalization()(x) if batch_norm else x x = Activation(activation)(x) if spatial_dropout and drop_values[i] > 0: x = SpatialDropout3D(drop_values[i])(x) elif drop_values[i] > 0 and not spatial_dropout: x = Dropout(drop_values[i])(x) x = Conv3D(feature_maps[i], (3, 3, 3), activation=None, kernel_initializer=k_init, padding='same')(x) x = BatchNormalization()(x) if batch_norm else x x = Activation(activation)(x) outputs = Conv3D(n_classes, (1, 1, 1), activation='sigmoid')(x) # Loss type if loss_type == "w_bce": model = Model(inputs=[inputs, weights], outputs=[outputs]) else: model = Model(inputs=[inputs], outputs=[outputs]) # Select the optimizer if optimizer == "sgd": opt = tf.keras.optimizers.SGD(lr=lr, momentum=0.99, decay=0.0, nesterov=False) elif optimizer == "adam": opt = tf.keras.optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) else: raise ValueError("Error: optimizer value must be 'sgd' or 'adam'") # Compile the model if loss_type == "bce": if n_classes > 1: model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=[jaccard_index_softmax]) else: model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[jaccard_index]) elif loss_type == "w_bce": model.compile(optimizer=opt, loss=binary_crossentropy_weighted(weights), metrics=[jaccard_index]) elif loss_type == "w_bce_dice": model.compile(optimizer=opt, loss=weighted_bce_dice_loss(w_dice=0.66, w_bce=0.33), metrics=[jaccard_index]) else: raise ValueError("'loss_type' must be 'bce', 'w_bce' or 'w_bce_dice'") return model
x = bottleneck_block(64, x) # Double the size of filters and reduce feature maps by 75% (strides=2, 2) to fit the next Residual Group x = conv_block(128, x) # Second Residual Block Group of 128 filters for _ in range(3): x = bottleneck_block(128, x) # Double the size of filters and reduce feature maps by 75% (strides=2, 2) to fit the next Residual Group x = conv_block(256, x) # Third Residual Block Group of 256 filters for _ in range(5): x = bottleneck_block(256, x) # Double the size of filters and reduce feature maps by 75% (strides=2, 2) to fit the next Residual Group x = conv_block(512, x) # Fourth Residual Block Group of 512 filters for _ in range(2): x = bottleneck_block(512, x) # Now Pool at the end of all the convolutional residual blocks x = layers.GlobalAveragePooling2D()(x) # Final Dense Outputting Layer for 1000 outputs outputs = layers.Dense(1000, activation='softmax')(x) model = Model(inputs, outputs)
x = layers.BatchNormalization(axis=chanDim)(x) x = layers.Conv2D(filters=250, kernel_size=3, strides=1, activation="relu", padding="same")(x) x = layers.MaxPool2D(2)(x) x = layers.BatchNormalization(axis=chanDim)(x) # flatten the network and then construct our latent vector volumeSize = K.int_shape(x) x = layers.Flatten()(x) latent = layers.Dense(latentDim)(x) # build the encoder model encoder = Model(input, latent, name="encoder") #decoder latentInputs = layers.Input(shape=(latentDim)) x = layers.Dense(np.prod(volumeSize[1:]))(latentInputs) x = layers.Reshape((volumeSize[1], volumeSize[2], volumeSize[3]))(x) # apply a CONV_TRANSPOSE => RELU => BN operation x = layers.Conv2DTranspose(filters=250, kernel_size=3, strides=1, activation="relu", padding="same")(x) x = layers.UpSampling2D(2)(x) x = layers.BatchNormalization(axis=chanDim)(x)
def evaluate_nnca(l1_train, l1_test, l2_train, l2_test, dimensions, evaluation_function=reciprocal_rank, neurons=[100], activation_function="relu", max_epochs=100, dropout=0.2, optimizer="adam", loss="MSE"): callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5) scores = [] for idz, dimension in enumerate(dimensions): if len(neurons) == 1: x = tf.keras.layers.Input(shape=(dimension, )) d1 = tf.keras.layers.Dropout(dropout)(x) emb = tf.keras.layers.Dense(neurons[0], activation=activation_function)(d1) d2 = tf.keras.layers.Dropout(dropout)(emb) y = tf.keras.layers.Dense(dimension, activation=activation_function)(d2) model = Model(x, y) elif len(neurons) == 3: x = tf.keras.layers.Input(shape=(dimension, )) d1 = tf.keras.layers.Dropout(dropout)(x) h1 = tf.keras.layers.Dense(neurons[0], activation=activation_function)(d1) emb = tf.keras.layers.Dense(neurons[1], activation=activation_function)(h1) h2 = tf.keras.layers.Dense(neurons[2], activation=activation_function)(emb) d2 = tf.keras.layers.Dropout(dropout)(h2) y = tf.keras.layers.Dense(dimension, activation=None)(d2) model = Model(x, y) if loss == "cosine_sim": loss = tf.keras.losses.CosineSimilarity() l1_to_l2_clf = Model(x, y) l2_to_l1_clf = Model(x, y) l1_to_l2_clf.compile(optimizer=optimizer, loss=loss, metrics=tf.keras.losses.CosineSimilarity()) l2_to_l1_clf.compile(optimizer=optimizer, loss=loss, metrics=tf.keras.losses.CosineSimilarity()) hist1 = l1_to_l2_clf.fit(l1_train[:, :dimension], l2_train[:, :dimension], epochs=max_epochs, validation_data=(l1_test[:, :dimension], l2_test[:, :dimension]), callbacks=[callback]) hist2 = l2_to_l1_clf.fit(l2_train[:, :dimension], l1_train[:, :dimension], epochs=max_epochs, validation_data=(l2_test[:, :dimension], l1_test[:, :dimension]), callbacks=[callback]) fake_fr = l1_to_l2_clf.predict(l1_test[:, :dimension]) fake_en = l2_to_l1_clf.predict(l2_test[:, :dimension]) merged_trans_vecs = np.concatenate((fake_en, l2_test[:, :dimension]), axis=1) real_vecs = np.concatenate((l1_test[:, :dimension], fake_fr), axis=1) score = evaluation_function(merged_trans_vecs, real_vecs) scores.append(score) return scores, hist1, hist2
def evaluate_nncc(l1_train, l1_test, l2_train, l2_test, dimensions, evaluation_function=reciprocal_rank, neurons=[100], activation_function="relu", max_epochs=100, dropout=0.2, optimizer="adam", loss="MSE"): callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5) l1_train, l1_test, l2_train, l2_test = np.asarray(l1_train), np.asarray( l1_test), np.asarray(l2_train), np.asarray(l2_test) scores = [] for idz, dimension in enumerate(dimensions): en = l1_train[:, :dimension] - np.mean(l1_train[:, :dimension], axis=0) fr = l2_train[:, :dimension] - np.mean(l2_train[:, :dimension], axis=0) zero_matrix = np.zeros((en.shape[0], dimension)) X1 = np.concatenate((en, zero_matrix), axis=1) X2 = np.concatenate((zero_matrix, fr), axis=1) X = np.concatenate((X1, X2), axis=0) Y1 = np.concatenate((en, fr), axis=1) Y2 = np.concatenate((en, fr), axis=1) Y = np.concatenate((Y1, Y2), axis=0) if len(neurons) == 1: x = tf.keras.layers.Input(shape=(dimension * 2, )) d1 = tf.keras.layers.Dropout(dropout)(x) emb = tf.keras.layers.Dense(neurons[0], activation=activation_function)(d1) d2 = tf.keras.layers.Dropout(dropout)(emb) y = tf.keras.layers.Dense(dimension * 2, activation=activation_function)(d2) model = Model(x, y) elif len(neurons) == 3: x = tf.keras.layers.Input(shape=(dimension * 2, )) d1 = tf.keras.layers.Dropout(dropout)(x) h1 = tf.keras.layers.Dense(neurons[0], activation=activation_function)(d1) emb = tf.keras.layers.Dense(neurons[1], activation=activation_function)(h1) h2 = tf.keras.layers.Dense(neurons[2], activation=activation_function)(emb) d2 = tf.keras.layers.Dropout(dropout)(h2) y = tf.keras.layers.Dense(dimension * 2, activation=None)(d2) model = Model(x, y) if loss == "cosine_sim": loss = tf.keras.losses.CosineSimilarity() model.compile( optimizer=optimizer, loss=loss, #"MSE",#tf.keras.losses.CosineSimilarity(), metrics=[tf.keras.losses.CosineSimilarity()]) history = model.fit(X, Y, epochs=max_epochs, validation_split=0.1, callbacks=[callback]) en = l1_test[:, :dimension] - np.mean(l1_train[:, :dimension], axis=0) fr = l2_test[:, :dimension] - np.mean(l1_train[:, :dimension], axis=0) zero_matrix = np.zeros((l1_test.shape[0], dimension)) X1 = np.concatenate((en, zero_matrix), axis=1) X2 = np.concatenate((zero_matrix, fr), axis=1) X = np.concatenate((X1, X2), axis=0) encoder = Model(x, emb) english_encodings_nncc = encoder.predict(X1) french_encodings_nncc = encoder.predict(X2) score = evaluation_function(english_encodings_nncc, french_encodings_nncc) scores.append(score) return scores, history
def MultiOutputCNN( input_shape, output_shape: Union[List, Dict], cnns_per_maxpool=4, maxpool_layers=4, # increasing `maxpool_layers` prefers fewer `cnns_per_maxpool` (ideal total CNNs = 15 / 16) cnn_units=32, cnn_kernel=3, cnn_strides=1, dense_layers=1, # `dense_layers=1` is preferred over `2` or `3` dense_units=256, activation='relu', # 'relu' | 'crelu' | 'leaky_relu' | 'relu6' | 'softmax' | 'tanh' | 'hard_sigmoid' | 'sigmoid' dropout=0.25, regularization=False, # `regularization=True` prefers `global_maxpool=False` and fewer dense_units - but worse results global_maxpool=False, # `global_maxpool=True` prefers double the number of `dense_units` and +1 `cnns_per_maxpool` name='', ) -> Model: function_name = cast(types.FrameType, inspect.currentframe()).f_code.co_name model_name = f"{function_name}-{name}" if name else function_name # model_name = seq([ function_name, name ]).filter(lambda x: x).make_string("-") # remove dependency on pyfunctional - not in Kaggle repo without internet inputs = Input(shape=input_shape) x = inputs for cnn1 in range(1, maxpool_layers + 1): for cnn2 in range(1, cnns_per_maxpool + 1): x = Conv2D(cnn_units * cnn1, kernel_size=cnn_kernel, strides=cnn_strides, padding='same', activation=activation)(x) x = MaxPooling2D(pool_size=(2, 2))(x) x = BatchNormalization()(x) x = Dropout(dropout)(x) if global_maxpool: x = GlobalMaxPooling2D()(x) x = Flatten()(x) for nn1 in range(0, dense_layers): if regularization: x = Dense(dense_units, activation=activation, kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))(x) else: x = Dense(dense_units, activation=activation)(x) x = BatchNormalization()(x) x = Dropout(dropout)(x) x = Flatten(name='output')(x) if isinstance(output_shape, dict): outputs = [ Dense(output_shape, activation='softmax', name=key)(x) for key, output_shape in output_shape.items() ] else: outputs = [ Dense(output_shape, activation='softmax', name=f'output_{index}')(x) for index, output_shape in enumerate(output_shape) ] model = Model(inputs, outputs, name=model_name) # plot_model(model, to_file=os.path.join(os.path.dirname(__file__), f"{name}.png")) return model
(X_train, y_train), (X_test, y_test) = datasets.cifar10.load_data() Y_train = to_categorical(y_train, num_classes) Y_test = to_categorical(y_test, num_classes) base_model = VGG16(weights='imagenet', include_top=False, input_shape=(32, 32, 3)) # Extract the last layer from third block of vgg16 model last = base_model.get_layer('block3_pool').output # Add classification layers on top of it x = Flatten()(last) x = Dense(256, activation='relu')(x) x = Dropout(0.5)(x) pred = Dense(10, activation='sigmoid')(x) model = Model(base_model.input, pred) # set the base model's layers to non-trainable # uncomment next two lines if you don't want to # train the base model # for layer in base_model.layers: # layer.trainable = False # compile the model with a SGD/momentum optimizer # and a very slow learning rate. model.compile(loss='binary_crossentropy', optimizer=tf.optimizers.SGD(lr=args.learning_rate, momentum=0.9), metrics=['accuracy']) # prepare data augmentation configuration train_datagen = ImageDataGenerator(rescale=1. / 255,
def U_Net( input_size=(256, 256, 1), #Correspond à la taille des images utilisées initial=64 #Le nombre de features maps utilisé au départ ): """ Crée un réseau de type U-net pour la segmentation Parameters ---------- - input_size : la taille des images à utiliser et le nombre de channels couleur (1 pour les DICOM) - initial : nombre de feature map à utiliser au déaprt (allourdit le réseau) Returns ------- - model : le réseau prêt à l'entrainement Notes ----- Ne pas hésiter à modifier les hyperparamètres : - le learning rate - l'optimizer """ inputs = Input(input_size) conv1 = Conv2D(initial, 3, activation='relu', padding='same', kernel_initializer='he_normal')(inputs) conv1 = Conv2D(initial, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv1) pool1 = MaxPooling2D(pool_size=(2, 2))(conv1) conv2 = Conv2D(initial * 2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool1) conv2 = Conv2D(initial * 2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv2) pool2 = MaxPooling2D(pool_size=(2, 2))(conv2) conv3 = Conv2D(initial * 4, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool2) conv3 = Conv2D(initial * 4, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv3) pool3 = MaxPooling2D(pool_size=(2, 2))(conv3) conv4 = Conv2D(initial * 8, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool3) conv4 = Conv2D(initial * 8, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv4) drop4 = Dropout(0.5)(conv4) pool4 = MaxPooling2D(pool_size=(2, 2))(drop4) conv5 = Conv2D(initial * 16, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool4) conv5 = Conv2D(initial * 16, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv5) drop5 = Dropout(0.5)(conv5) up6 = Conv2D(initial * 8, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(drop5)) merge6 = concatenate([drop4, up6], axis=3) conv6 = Conv2D(initial * 8, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge6) conv6 = Conv2D(initial * 8, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv6) up7 = Conv2D(initial * 4, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv6)) merge7 = concatenate([conv3, up7], axis=3) conv7 = Conv2D(initial * 4, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge7) conv7 = Conv2D(initial * 4, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv7) up8 = Conv2D(initial * 2, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv7)) merge8 = concatenate([conv2, up8], axis=3) conv8 = Conv2D(initial * 2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge8) conv8 = Conv2D(initial * 2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv8) up9 = Conv2D(initial, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv8)) merge9 = concatenate([conv1, up9], axis=3) conv9 = Conv2D(initial, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge9) conv9 = Conv2D(initial, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9) conv9 = Conv2D(2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9) conv10 = Conv2D(1, 1, activation='sigmoid')(conv9) model = Model(inputs=inputs, outputs=conv10) model.compile(optimizer=Adam(lr=1e-4), loss='binary_crossentropy', metrics=['accuracy']) return model
def YOLOv3Net(cfgfile, model_size, num_classes): blocks = parse_cfg(cfgfile) outputs = {} output_filters = [] filters = [] out_pred = [] scale = 0 inputs = input_image = Input(shape=model_size) inputs = inputs / 255.0 for i, block in enumerate(blocks[1:]): if block["type"] == "convolutional": activation = block["activation"] filters = int(block["filters"]) kernel_size = int(block["size"]) strides = int(block["stride"]) if strides > 1: #top and left padding with zeros inputs = ZeroPadding2D(((1, 0), (1, 0)))(inputs) inputs = Conv2D(filters, kernel_size, strides=strides, padding='valid' if strides > 1 else 'same', name='conv_' + str(i), use_bias=False if ("batch_normalize" in block) else True)(inputs) if "batch_normalize" in block: inputs = BatchNormalization(name="bnorm_" + str(i))(inputs) inputs = LeakyReLU(alpha=0.1, name='leaky_' + str(i))(inputs) elif block["type"] == "upsample": stride = int(block["stride"]) inputs = UpSampling2D(stride)(inputs) elif block["type"] == "route": block["layers"] = block["layers"].split(',') start = int(block["layers"][0]) if len(block["layers"]) > 1: end = int(block["layers"][1]) - i filters = output_filters[i + start] + output_filters[end] inputs = tf.concat([outputs[i + start], outputs[i + end]], axis=-1) else: filters = output_filters[i + start] inputs = outputs[i + start] elif block["type"] == "shortcut": from_ = int(block["from"]) inputs = outputs[i - 1] + outputs[i + from_] elif block["type"] == "yolo": mask = block["mask"].split(",") mask = [int(x) for x in mask] anchors = block["anchors"].split(",") anchors = [int(a) for a in anchors] anchors = [(anchors[i], anchors[i + 1]) for i in range(0, len(anchors), 2)] anchors = [anchors[i] for i in mask] n_anchors = len(anchors) out_shape = inputs.get_shape().as_list() inputs = tf.reshape( inputs, [-1, n_anchors * out_shape[1] * out_shape[2], 5 + num_classes]) box_centers = inputs[:, :, 0:2] box_shapes = inputs[:, :, 2:4] confidence = inputs[:, :, 4:5] classes = inputs[:, :, 5:num_classes + 5] box_centers = tf.sigmoid(box_centers) confidence = tf.sigmoid(confidence) classes = tf.sigmoid(classes) anchors = tf.tile(anchors, [out_shape[1] * out_shape[2], 1]) box_shapes = tf.exp(box_shapes) * tf.cast(anchors, dtype=tf.float32) x = tf.range(out_shape[1], dtype=tf.float32) y = tf.range(out_shape[2], dtype=tf.float32) cx, cy = tf.meshgrid(x, y) cx = tf.reshape(cx, (-1, 1)) cy = tf.reshape(cy, (-1, 1)) cxy = tf.concat([cx, cy], axis=-1) cxy = tf.tile(cxy, [1, n_anchors]) cxy = tf.reshape(cxy, [1, -1, 2]) strides = (input_image.shape[1] // out_shape[1], input_image.shape[2] // out_shape[2]) box_centers = (box_centers + cxy) * strides prediction = tf.concat( [box_centers, box_shapes, confidence, classes], axis=-1) if scale: out_pred = tf.concat([out_pred, prediction], axis=1) else: out_pred = prediction scale = 1 outputs[i] = inputs output_filters.append(filters) model = Model(input_image, out_pred) model.summary() return model
def get_adm_model(learning_rate, l2_reg, price_min, price_max, price_interval, embedd_size): price_step = int( math.floor((price_max - price_min + K.epsilon()) / price_interval)) total_len = 0 input_tensors = [] # Input Embedding Layers embedding_tensors = [] # composing embedding layers for column_name, count_value, dimension_length in embedding_paras: total_len += embedd_size # or dimension_length input_tensor = Input(name='{}_index'.format(column_name), shape=(1, ), dtype='int64') embedding_tensor = Embedding( count_value, embedd_size, # or dimension_length input_length=1, embeddings_initializer='glorot_normal', name='{}_embedding'.format(column_name))(input_tensor) embedding_tensor = Flatten()(embedding_tensor) embedding_tensors.append(embedding_tensor) input_tensors.append(input_tensor) # Input Numerical Layers numerical_tensors = [] numerical_embedd_tensors = [] for column_name in numerical_paras: total_len += 1 input_tensor = Input(name='{}'.format(column_name), shape=(1, ), dtype='float32') numerical_tensors.append(input_tensor) input_tensors.append(input_tensor) input_embedd_tensor = Lambda(lambda x: tf.tile(x, [1, embedd_size]))( input_tensor) numerical_embedd_tensors.append(input_embedd_tensor) feature_num = len(input_tensors) # 44 # features (embedding + numeric) ## 1-order x_o1_tensor = Concatenate()(embedding_tensors + numerical_tensors) ## 2-order x_o2_tensor = Lambda(lambda x: tf.stack(x, axis=1))( embedding_tensors + numerical_embedd_tensors) # (None, feature_num, dim) x_o2_tensor = InteractingLayer(att_embedding_size=8, head_num=2)(x_o2_tensor) x_o2_tensor = Flatten()(x_o2_tensor) ## high-order x_oh_tensor = Dense( total_len / 2, activation='relu', kernel_regularizer=regularizers.l2(l2_reg))(x_o1_tensor) x_oh_tensor = Dense( total_len / 4, activation='relu', kernel_regularizer=regularizers.l2(l2_reg))(x_oh_tensor) # output layer output_tensor = Concatenate(axis=1)( [x_o1_tensor, x_o2_tensor, x_oh_tensor]) output_tensor = Dense( price_step, kernel_regularizer=regularizers.l2(l2_reg))(output_tensor) output_tensor = Softmax(name='mixture_price_3')(output_tensor) model = Model(inputs=input_tensors, outputs=[output_tensor]) adam = optimizers.Adam(lr=learning_rate) optimizer = hvd.DistributedOptimizer(adam) model.compile(loss=neighborhood_likelihood_loss, optimizer=optimizer, experimental_run_tf_function=False) return model
def mobile_net_v2(input_shape=(224,224,3), num_classes=1000): """ mobile net v2 based on https://arxiv.org/pdf/1801.04381.pdf Args: input_shape (tuple): input shape num_classes (int): number of categories Returns: mobile net v2 model """ input = layers.Input(shape=input_shape) x = layers.Conv2D( 32, 3, 2, padding='same', use_bias=False )(input) x = layers.BatchNormalization()(x) x = layers.ReLU(6.0)(x) x = conv_block(x, 16, 1, 1, expand=False) x = conv_block(x, 24, 2, 6) x = conv_block(x, 24, 1, 6) x = conv_block(x, 32, 2, 6) x = conv_block(x, 32, 1, 6) x = conv_block(x, 32, 1, 6) x = conv_block(x, 64, 2, 6) x = conv_block(x, 64, 1, 6) x = conv_block(x, 64, 1, 6) x = conv_block(x, 64, 1, 6) x = conv_block(x, 96, 1, 6) x = conv_block(x, 96, 1, 6) x = conv_block(x, 96, 1, 6) x = conv_block(x, 160, 2, 6) x = conv_block(x, 160, 1, 6) x = conv_block(x, 160, 1, 6) x = conv_block(x, 320, 1, 6) x = layers.Conv2D( 1280, 1, 1, padding='same', use_bias=False )(x) x = layers.BatchNormalization()(x) x = layers.ReLU(6.0)(x) x = layers.GlobalAveragePooling2D()(x) x = layers.Dense(num_classes)(x) x = layers.Activation('softmax')(x) model = Model(inputs=input, outputs=x) model.summary() return model
filters=8, kernel_size=( 3, 3), padding='same', activation='relu', use_bias=False, connectivity_level=built_in_sparsity.pop(0) or None)(layer) layer = Flatten()(layer) layer = Dropout(0.01)(layer) layer = Sparse(units=10, activation='softmax', use_bias=False, connectivity_level=built_in_sparsity.pop(0) or None)(layer) model = Model(input_layer, layer) model.summary() model.compile( NoisySGD( lr=0.01), 'categorical_crossentropy', ['accuracy']) # Train model with backprop. model.fit(x_train, y_train, batch_size=64, epochs=5, verbose=2, validation_data=(x_test, y_test), callbacks=callback_list) # Store model so SNN Toolbox can find it.
x_train = scaler.transform(x_train) x_test = scaler.transform(x_test) print(np.max(x), np.min(x)) print(np.max(x[0])) #2. model from tensorflow.keras import Model from tensorflow.keras.layers import Input, Dense input1 = Input(shape=(13, )) dense1 = Dense(128, activation='relu')(input1) dense1 = Dense(64, activation='relu')(dense1) dense1 = Dense(64, activation='relu')(dense1) dense1 = Dense(64, activation='relu')(dense1) output1 = Dense(1)(dense1) model = Model(inputs=input1, outputs=output1) #3. compile and fit model.compile(optimizer='adam', loss='mse', metrics=['mae']) model.fit(x_train, y_train, batch_size=4, epochs=150, verbose=1, validation_split=0.2) #4. evalutate and predict mse, mae = model.evaluate(x_test, y_test, batch_size=4) print("mse :", mse, "\nmae :", mae) y_predict = model.predict(x_test)
def hifis_rnn_mlp(cfg, input_dim, metrics, metadata, output_bias=None, hparams=None): ''' Defines a Keras model for HIFIS hybrid recurrent neural network and multilayer perceptron model (i.e. HIFIS-v3) :param cfg: A dictionary of parameters associated with the model architecture :param input_dim: The shape of the model input :param metrics: Metrics to track model's performance :param metadata: Dict containing prediction horizon, time series feature info :param output_bias: initial bias applied to output layer :param hparams: dict of hyperparameters :return: a Keras model object with the architecture defined in this method ''' # Set hyperparameters if hparams is None: nodes_dense0 = cfg['DENSE']['DENSE0'] nodes_dense1 = cfg['DENSE']['DENSE1'] layers = cfg['LAYERS'] dropout = cfg['DROPOUT'] l2_lambda = cfg['L2_LAMBDA'] lr = cfg['LR'] optimizer = Adam(learning_rate=lr) lstm_units = cfg['LSTM']['UNITS'] else: nodes_dense0 = hparams['NODES0'] nodes_dense1 = hparams['NODES1'] layers = hparams['LAYERS'] dropout = hparams['DROPOUT'] lr = 10**hparams['LR'] # Random sampling on logarithmic scale beta_1 = 1 - 10**hparams['BETA_1'] beta_2 = 1 - 10**hparams['BETA_2'] l2_lambda = 10**hparams['L2_LAMBDA'] lstm_units = hparams['LSTM_UNITS'] if hparams['OPTIMIZER'] == 'adam': optimizer = Adam(learning_rate=lr, beta_1=beta_1, beta_2=beta_2) elif hparams['OPTIMIZER'] == 'sgd': optimizer = SGD(learning_rate=lr) if output_bias is not None: output_bias = Constant(output_bias) # Receive input to model and split into 2 tensors containing dynamic and static features respectively X_input = Input(shape=input_dim) split_idx = metadata['NUM_TS_FEATS'] * metadata['T_X'] X_dynamic, X_static = split(X_input, [split_idx, X_input.shape[1] - split_idx], axis=1) # Define RNN component of model using LSTM cells. LSTM input shape is [batch_size, timesteps, features] lstm_input_shape = (metadata['T_X'], metadata['NUM_TS_FEATS']) X_dynamic = Reshape(lstm_input_shape)(X_dynamic) X_dynamic = LSTM(lstm_units, activation='tanh', return_sequences=True)(X_dynamic) X_dynamic = Flatten()(X_dynamic) # Define MLP component of model X = concatenate([X_dynamic, X_static]) # Combine output of LSTM with static features X = Dense(nodes_dense0, input_shape=input_dim, activation='relu', kernel_regularizer=l2(l2_lambda), bias_regularizer=l2(l2_lambda), name="dense0")(X) X = Dropout(dropout, name='dropout0')(X) for i in range(1, layers): X = Dense(nodes_dense1, activation='relu', kernel_regularizer=l2(l2_lambda), bias_regularizer=l2(l2_lambda), name='dense%d' % i)(X) X = Dropout(dropout, name='dropout%d' % i)(X) Y = Dense(1, activation='sigmoid', name="output", bias_initializer=output_bias)(X) # Define model with inputs and outputs model = Model(inputs=X_input, outputs=Y, name='HIFIS-rnn-mlp_' + str(metadata['N_WEEKS']) + '-weeks') # Set model loss function, optimizer, metrics. model.compile(loss=f1_loss(4.5), optimizer=optimizer, metrics=metrics) # Print summary of model and return model object if hparams is None: model.summary() return model
x_test = x_test / 255 x_train = x_train.reshape(x_train.shape[0], 28 * 28) x_test = x_test.reshape(x_test.shape[0], 28 * 28) # x_train=x_train.reshape(x_train.shape[0], x_train.shape[1], x_train.shape[2], 1) # x_test=x_test.reshape(x_test.shape[0], x_test.shape[1], x_test.shape[2], 1) y_train = keras.utils.to_categorical(y_train, 10) y_test = keras.utils.to_categorical(y_test, 10) input_layer = Input(28 * 28) d1 = Dense(units=128, activation='sigmoid')(input_layer) # c1=Conv2D(32, kernel_size=(3, 3), activation='relu')(input_layer) # p1=MaxPool2D(pool_size=(2, 2))(c1) # f1=Flatten()(p1) d2 = Dense(units=10, activation='softmax')(d1) model = Model(inputs=input_layer, outputs=d2) model.compile( # optimizer=keras.optimizers.SGD(learning_rate=LEARNINT_RATE,), optimizer=keras.optimizers.SGD(learning_rate=LEARNINT_RATE), loss='categorical_crossentropy', metrics=['acc'], ) train_jistory = model.fit( x=x_train, y=y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, verbose=1, validation_split=0.2, )
# Training constants n_weights = 80 lr = 3e-4 alpha = 0.1 train_dataset_size = 1000 # Get subset of data due to slowness val_dataset_size = 100 n_epochs = 100 # Tensorflow Keras model Y_input = Input(shape=(solver.n_obs, ), name='Y_input') layer1 = Dense(n_weights, input_shape=(solver.dofs, ), activation='relu') layer1out = layer1(Y_input) layer2 = Dense(n_weights, activation='relu') layer2out = layer2(layer1out) U_output = Dense(solver.dofs, name='U_output')(layer2out) model = Model(inputs=Y_input, outputs=U_output) L = tf.reduce_mean(tf.square(Y_input - G(tf.math.exp(U_output)))) grad_bias_1 = gradients(L, model.trainable_weights[1])[0] dummy_inp = np.random.randn(batch_size, solver.n_obs) grad_bias_1_eval = get_session().run(grad_bias_1, feed_dict={Y_input: dummy_inp}) bias_1_vals = layer1.get_weights()[1] kernel_vals = layer1.get_weights()[0] L_0 = get_session().run(L, feed_dict={Y_input: dummy_inp}) eps_bias = np.random.randn( n_weights) #Random direction to perturn bias weights eps_bias = eps_bias / np.linalg.norm(eps_bias) dir_grad = np.dot(grad_bias_1_eval, eps_bias) n_eps = 32
def YOLOv3Net(cfgfile, model_size, num_classes): #Konstruisanje YOLO v3 modela. #Parametri: # cfgfile: Ime i putanja do konfiguracione datoteke # model_size: Veličina ulaznih podataka u model (visina, širina, dubina) # num_classes: broj klasa #Povratna vrednost: # model: YOLO v3 model # učitavanje informacija iz konfiguracionog mrežnog fajla blocks = parse_cfg(cfgfile) outputs = {} output_filters = [] filters = [] out_pred = [] scale = 0 inputs = input_image = Input(shape=model_size) inputs = inputs / 255.0 # prolazak kroz slojeve i parametre slojeva i konstruisanje TensorFlow modela for i, block in enumerate(blocks[1:]): # Ako je u pitanju konvolucioni blok if (block["type"] == "convolutional"): activation = block["activation"] filters = int(block["filters"]) kernel_size = int(block["size"]) strides = int(block["stride"]) if strides > 1: inputs = ZeroPadding2D(((1, 0), (1, 0)))(inputs) inputs = Conv2D(filters, kernel_size, strides=strides, padding='valid' if strides > 1 else 'same', name='conv_' + str(i), use_bias=False if ("batch_normalize" in block) else True)(inputs) if "batch_normalize" in block: inputs = BatchNormalization(name='bnorm_' + str(i))(inputs) inputs = LeakyReLU(alpha=0.1, name='leaky_' + str(i))(inputs) # Ako je u pitanju blok povećanja dimenzionalnosti elif (block["type"] == "upsample"): stride = int(block["stride"]) inputs = UpSampling2D(stride)(inputs) # Ako je u pitanju [route] blok elif (block["type"] == "route"): block["layers"] = block["layers"].split(',') start = int(block["layers"][0]) if len(block["layers"]) > 1: end = int(block["layers"][1]) - i filters = output_filters[i + start] + output_filters[end] inputs = tf.concat([outputs[i + start], outputs[i + end]], axis=-1) else: filters = output_filters[i + start] inputs = outputs[i + start] # Ako je u pitanju [shortcut] blok elif block["type"] == "shortcut": from_ = int(block["from"]) inputs = outputs[i - 1] + outputs[i + from_] # Yolo detekcioni sloj (nakon konvolucionih obrada) elif block["type"] == "yolo": mask = block["mask"].split(",") mask = [int(x) for x in mask] anchors = block["anchors"].split(",") anchors = [int(a) for a in anchors] anchors = [(anchors[i], anchors[i + 1]) for i in range(0, len(anchors), 2)] anchors = [anchors[i] for i in mask] n_anchors = len(anchors) out_shape = inputs.get_shape().as_list() inputs = tf.reshape(inputs, [-1, n_anchors * out_shape[1] * out_shape[2], \ 5 + num_classes]) box_centers = inputs[:, :, 0:2] box_shapes = inputs[:, :, 2:4] confidence = inputs[:, :, 4:5] classes = inputs[:, :, 5:num_classes + 5] box_centers = tf.sigmoid(box_centers) confidence = tf.sigmoid(confidence) classes = tf.sigmoid(classes) anchors = tf.tile(anchors, [out_shape[1] * out_shape[2], 1]) box_shapes = tf.exp(box_shapes) * tf.cast(anchors, dtype=tf.float32) x = tf.range(out_shape[1], dtype=tf.float32) y = tf.range(out_shape[2], dtype=tf.float32) cx, cy = tf.meshgrid(x, y) cx = tf.reshape(cx, (-1, 1)) cy = tf.reshape(cy, (-1, 1)) cxy = tf.concat([cx, cy], axis=-1) cxy = tf.tile(cxy, [1, n_anchors]) cxy = tf.reshape(cxy, [1, -1, 2]) strides = (input_image.shape[1] // out_shape[1], \ input_image.shape[2] // out_shape[2]) box_centers = (box_centers + cxy) * strides prediction = tf.concat( [box_centers, box_shapes, confidence, classes], axis=-1) if scale: out_pred = tf.concat([out_pred, prediction], axis=1) else: out_pred = prediction scale = 1 outputs[i] = inputs output_filters.append(filters) # Ulazna slika predstavlja početnu tačku # Predikcije iz YOLO sloja predstavljaju krajnju tačku modela model = Model(input_image, out_pred) # Ispis svih slojeva i parametara modela u konzolu # model.summary() return model
def build_model(days: int, posts: int, embedding_len: int, words: int, target_len: int, threshold: float = None) -> Model: """ Build Keras model using Functional API. Model consists of three sub modules: Post Sub Model, Day Sub Model and Fortnight Sub Model. :param days: Number of days in each row. Determines the number of Day Sub Models :param posts: Number of posts in each day. Determines the number of Post Sub Models :param embedding_len: Length of embedding of a single post :param words: Number of words per one post, important to specify the input shape :param target_len: Length of the target vector. Determines the number of units in the last, dense layer. :return: Returns a ready Keras model with empty weights """ # Post sub model parameters # Convolution post_highlights_filters = 80 post_highlights_kernel = 5 post_highlights_stride = 1 # Maxpool post_maxpool_size = 10 post_maxpool_stride = 5 # Day sub model parameters # Convolution day_highlights_filters = 80 day_highlights_kernel = 3 day_highlights_stride = 1 # Maxpool day_maxpool_size = 5 day_maxpool_stride = 3 # Fortnight sub model parameters lstm_units = 150 input_shape = Input(shape=(days, posts, words, embedding_len)) ''' Choose 'relu' as activation function, as it supposedly performs fine. Will try other functions as necessary, if the network does not learn anything. Resources: https://towardsdatascience.com/visualizing-intermediate-activation-in-convolutional-neural-networks-with-keras-260b36d60d0 and: https://ai.stackexchange.com/questions/7088/how-to-choose-an-activation-function and: https://missinglink.ai/guides/neural-network-concepts/7-types-neural-network-activation-functions-right/ ''' post_highlights_convolution = Conv1D(filters=post_highlights_filters, kernel_size=post_highlights_kernel, strides=post_highlights_stride, activation=LeakyReLU(), name='post_highlights_convolution') post_permute = Permute(dims=(2, 1)) shared_post_maxpool = MaxPooling1D(pool_size=post_maxpool_size, strides=post_maxpool_stride, data_format='channels_first', name='shared_post_maxpool') day_highlights_convolution = Conv1D(filters=day_highlights_filters, kernel_size=day_highlights_kernel, strides=day_highlights_stride, activation=LeakyReLU(), name='day_highlights_convolution') shared_day_maxpool = MaxPooling1D(pool_size=day_maxpool_size, strides=day_maxpool_stride, data_format='channels_first', name='shared_day_maxpool') day_outputs = [] for i in range(days): # Slice the input into different days. Take the second dimension, because first one on the left is 'batch_size' day = Lambda(lambda x: x[:, i, :, :, :])(input_shape) post_outputs = [] for j in range(posts): # Slice the input into different posts. Take the second dimension, # because first one on the left is 'batch_size'. What is left is [words:embeddings] out = Lambda(lambda x: x[:, j, :, :])(day) post_x = (post_highlights_convolution)(out) post_x = (post_permute)(post_x) post_x = (shared_post_maxpool)(post_x) post_outputs.append(post_x) day_x = Concatenate()(post_outputs) day_x = (day_highlights_convolution)(day_x) day_x = (shared_day_maxpool)(day_x) day_outputs.append(day_x) week_overview = Concatenate()(day_outputs) output_x = LSTM(units=lstm_units)(week_overview) ''' activation='sigmoid' is important here because of the multi-label classification problem. ''' output_x = Dense(units=target_len, activation='sigmoid')(output_x) model = Model(inputs=input_shape, outputs=output_x) ''' optimizer=Adam is sort of arbitrary choice, but was recommended as well-performing. Check here: https://www.dlology.com/blog/quick-notes-on-how-to-choose-optimizer-in-keras/ and here: https://datascience.stackexchange.com/questions/10523/guidelines-for-selecting-an-optimizer-for-training-neural-networks loss='binary_crossentropy' is important here because of the multi-label classification problem. Check here: https://stackoverflow.com/questions/44164749/how-does-keras-handle-multilabel-classification and here: https://towardsdatascience.com/multi-label-image-classification-with-neural-network-keras-ddc1ab1afede ''' if threshold: recall_metric = tf.keras.metrics.Recall(thresholds=threshold) accuracy_metric = tf.keras.metrics.BinaryAccuracy(threshold=threshold) precision_metric = tf.keras.metrics.Precision(thresholds=threshold) else: recall_metric = tf.keras.metrics.Recall() accuracy_metric = tf.keras.metrics.BinaryAccuracy() precision_metric = tf.keras.metrics.Precision() roc_curve_metric = tf.keras.metrics.AUC(num_thresholds=50, multi_label=False) model.compile(optimizer=Adam(learning_rate=0.01), loss='binary_crossentropy', metrics=[ roc_curve_metric, recall_metric, precision_metric, accuracy_metric ]) print(model.summary()) return model
x = layers.MaxPooling2D(2)(x) # Flatten feature map to a 1-dim tensor x = layers.Flatten()(x) # Create a fully connected layer with ReLU activation and 512 hidden units x = layers.Dense(512, activation='relu')(x) # Add a dropout rate of 0.5 x = layers.Dropout(0.5)(x) # Create output layer with a single node and sigmoid activation output = layers.Dense(1, activation='sigmoid')(x) # Configure and compile the model model = Model(img_input, output) model.compile(loss='binary_crossentropy', optimizer=RMSprop(lr=0.001), metrics=['acc']) # In[11]: history = model.fit_generator(train_generator, steps_per_epoch=100, epochs=30, validation_data=validation_generator, validation_steps=50, verbose=2) # In[ ]:
main_branch = TimeDistributed(utter_embedding)(inputs) main_branch = Bidirectional(LSTM(128, return_sequences=True, activation="relu"))(main_branch) main_branch = Bidirectional(LSTM(64, return_sequences=True, activation="relu"))(main_branch) da_output = Dense(9, activation="softmax", kernel_regularizer=regularizers.l2(0.01), name="da_output")(main_branch) st_output = Dense(1, activation="sigmoid", kernel_regularizer=regularizers.l2(0.01), name="st_output")(main_branch) model = Model(inputs=inputs, outputs=[da_output, st_output]) model.compile(optimizer=RMSprop(lr=0.2, decay=0.001), loss={ "da_output": "categorical_crossentropy", "st_output": "binary_crossentropy" }, loss_weights={ "da_output": 0.5, "st_output": 1.0 }, metrics=['accuracy']) model.summary() plot_model(model, to_file='model_plot.png',
def compiled_tcn( num_feat, # type: int num_classes, # type: int nb_filters, # type: int kernel_size, # type: int dilations, # type: List[int] nb_stacks, # type: int max_len, # type: int padding='causal', # type: str use_skip_connections=True, # type: bool return_sequences=True, regression=False, # type: bool dropout_rate=0.05, # type: float name='tcn', # type: str, activation='linear', # type:str, opt='adam', lr=0.002, metrics=[]): # type: (...) -> keras.Model """Creates a compiled TCN model for a given task (i.e. regression or classification). Classification uses a sparse categorical loss. Please input class ids and not one-hot encodings. Args: num_feat: The number of features of your input, i.e. the last dimension of: (batch_size, timesteps, input_dim). num_classes: The size of the final dense layer, how many classes we are predicting. nb_filters: The number of filters to use in the convolutional layers. kernel_size: The size of the kernel to use in each convolutional layer. dilations: The list of the dilations. Example is: [1, 2, 4, 8, 16, 32, 64]. nb_stacks : The number of stacks of residual blocks to use. max_len: The maximum sequence length, use None if the sequence length is dynamic. padding: The padding to use in the convolutional layers. use_skip_connections: Boolean. If we want to add skip connections from input to each residual block. return_sequences: Boolean. Whether to return the last output in the output sequence, or the full sequence. regression: Whether the output should be continuous or discrete. dropout_rate: Float between 0 and 1. Fraction of the input units to drop. name: Name of the model. Useful when having multiple TCN. activation: The activation used in the residual blocks o = Activation(x + F(x)). opt: Optimizer name. lr: Learning rate. Returns: A compiled keras TCN. """ dilations = process_dilations(dilations) # input_layer = Input(shape=(max_len, num_feat)) # --- Vincent Added to deal with reshaping input --- total_input = max_len * num_feat # max_len~wind size and num_feat~selected columns input_layer = Input(shape=(total_input, )) reshape_layer = Reshape((max_len, num_feat), input_shape=(total_input, ))(input_layer) # --- Vincent Stopped breaking stuff here --- x = TCN(nb_filters, kernel_size, nb_stacks, dilations, padding, use_skip_connections, dropout_rate, return_sequences, activation, name)(reshape_layer) print('x.shape=', x.shape) def get_opt(): if opt == 'adam': return optimizers.Adam(lr=lr, clipnorm=1.) elif opt == 'rmsprop': return optimizers.RMSprop(lr=lr, clipnorm=1.) else: raise Exception('Only Adam and RMSProp are available here') if not regression: # classification x = Dense(num_classes)(x) x = Activation('softmax')(x) output_layer = x model = Model(input_layer, output_layer) # https://github.com/keras-team/keras/pull/11373 # It's now in Keras@master but still not available with pip. # TODO remove later. def accuracy(y_true, y_pred): # reshape in case it's in shape (num_samples, 1) instead of (num_samples,) if K.ndim(y_true) == K.ndim(y_pred): y_true = K.squeeze(y_true, -1) # convert dense predictions to labels y_pred_labels = K.argmax(y_pred, axis=-1) y_pred_labels = K.cast(y_pred_labels, K.floatx()) return K.cast(K.equal(K.cast(y_true, K.floatx()), y_pred_labels), K.floatx()) model.compile(get_opt(), loss='sparse_categorical_crossentropy', metrics=[accuracy] + metrics) else: # regression x = Dense(1)(x) x = Activation('linear')(x) output_layer = x model = Model(input_layer, output_layer) model.compile(get_opt(), loss='mean_squared_error') print(f'model.x = {input_layer.shape}') print(f'model.y = {output_layer.shape}') return model
def U_Net_3D_Xiao(image_shape, lr=0.0001, n_classes=2): """Create 3D U-Net. Parameters ---------- image_shape : array of 3 int Dimensions of the input image. activation : str, optional Keras available activation type. lr : float, optional Learning rate value. n_classes : int, optional Number of classes. Returns ------- model : Keras model Xiao 3D U-Net type proposed network model. Here is a picture of the network extracted from the original paper: .. image:: img/xiao_network.jpg :width: 100% :align: center """ dinamic_dim = (None, ) * (len(image_shape) - 1) + (1, ) inputs = Input(dinamic_dim) x = Conv3D(3, (3, 3, 3), activation=None, kernel_initializer='he_normal', padding='same')(inputs) x = BatchNormalization()(x) x = ELU()(x) # Encoder s1 = residual_block(x, 32) x = MaxPooling3D((2, 2, 1), padding='same')(s1) s2 = residual_block(x, 48) x = MaxPooling3D((2, 2, 1), padding='same')(s2) s3 = residual_block(x, 64) x = MaxPooling3D((2, 2, 1), padding='same')(s3) x = residual_block(x, 80) # Decoder x = UpSampling3D((2, 2, 1))(x) x = Conv3D(64, (1, 1, 1), activation=None, kernel_initializer='he_normal', padding='same')(x) x = BatchNormalization()(x) x = ELU()(x) x = Add()([s3, x]) x = residual_block(x, 64) x = UpSampling3D((2, 2, 1))(x) # Auxiliary ouput 1 a1 = UpSampling3D((2, 2, 1))(x) a1 = Conv3D(n_classes, (1, 1, 1), activation=None, kernel_initializer='he_normal', padding='same', kernel_regularizer=l2(0.01), name='aux1')(a1) a1 = Activation('softmax')(a1) x = Conv3D(48, (1, 1, 1), activation=None, kernel_initializer='he_normal', padding='same')(x) x = BatchNormalization()(x) x = ELU()(x) x = Add()([s2, x]) x = residual_block(x, 48) x = UpSampling3D((2, 2, 1))(x) x = Conv3D(32, (1, 1, 1), activation=None, kernel_initializer='he_normal', padding='same')(x) x = BatchNormalization()(x) x = ELU()(x) # Auxiliary ouput 2 a2 = Conv3D(n_classes, (1, 1, 1), activation=None, kernel_initializer='he_normal', padding='same', kernel_regularizer=l2(0.01), name='aux2')(x) a2 = Activation('softmax')(a2) x = Add()([s1, x]) x = residual_block(x, 32) x = Conv3D(3, (3, 3, 3), activation=None, kernel_initializer='he_normal', padding='same', kernel_regularizer=l2(0.01))(x) x = BatchNormalization()(x) x = ELU()(x) # Adapt the output to use softmax pixel-wise x = Conv3D(n_classes, (1, 1, 1), activation=None, kernel_initializer='he_normal', padding='same')(x) outputs = Activation('softmax', name='main_classifier')(x) model = Model(inputs=[inputs], outputs=[a1, a2, outputs]) opt = tf.keras.optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-8, decay=0.0, amsgrad=False) model.compile(optimizer=opt, loss="categorical_crossentropy", loss_weights=[0.15, 0.3, 1], metrics=[jaccard_index_softmax]) return model
for residual in self.res_blocks: x = residual(x) return self.output_conv(x) if __name__ == "__main__": from tensorflow.keras.layers import Input import numpy as np encoder = Encoder(3, 1, [3, 3, 3], [32, 64, 128], [1, 1, 1], initializer='he_normal') decoder = Decoder(1, 3, 1, [3, 3, 3], [128, 64, 32], [1, 1, 1], initializer='he_normal') inputTensor = Input(shape=(28, 28, 1)) x = encoder(inputTensor) x = decoder(x) model = Model(inputs=inputTensor, outputs=x) model.compile(loss='mse', optimizer='sgd') model.summary()
filter_num, activation) filter_num *= 2 previousLayer = layers.MaxPool2D((2, 2))(previousLayer) temp_layer = previousLayer previousLayer, temp_layer = shortcut_block(previousLayer, temp_layer, filter_num, activation) # average pool, flatten, then dense to instatntiate output layer outputLayer = layers.AveragePooling2D((2, 2))(previousLayer) outputLayer = layers.Flatten()(outputLayer) outputLayer = layers.Dense(outputDimension, activation='sigmoid', kernel_regularizer=regularizers.l2(0.001), name='outputLayer')(outputLayer) #compile model ourResnet = Model(inputs=inputLayer, outputs=[outputLayer], name='fashion_mlp') ourResnet.compile(loss='categorical_crossentropy', metrics=['acc'], optimizer='adam') print(ourResnet.summary()) # In[35]: outputResults = {} for x in range(1, 11): # training the resnet model trainingLabelsCategorical = utils.to_categorical( trainingLabels, num_classes=outputDimension) # ourResnet.fit(trainingData, trainingLabelsCategorical, epochs=1, validation_split=0.85) # testing the resnet model
def ResNet18_6_Parted(classes, input_shape, weight_decay=1e-4): input = Input(shape=input_shape) x = input / 255 # x = conv2d_bn_relu(x, filters=64, kernel_size=(7, 7), weight_decay=weight_decay, strides=(2, 2)) # x = MaxPool2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) x = conv2d_bn_relu(x, filters=64, kernel_size=(3, 3), weight_decay=weight_decay, strides=(1, 1)) # # conv 2 x = ResidualBlock(x, filters=64, kernel_size=(3, 3), weight_decay=weight_decay, downsample=False) x = ResidualBlock(x, filters=64, kernel_size=(3, 3), weight_decay=weight_decay, downsample=False) # # conv 3 x = ResidualBlock(x, filters=128, kernel_size=(3, 3), weight_decay=weight_decay, downsample=True) x = ResidualBlock(x, filters=128, kernel_size=(3, 3), weight_decay=weight_decay, downsample=False) # # conv 4 x = ResidualBlock(x, filters=256, kernel_size=(3, 3), weight_decay=weight_decay, downsample=True) x = ResidualBlock(x, filters=256, kernel_size=(3, 3), weight_decay=weight_decay, downsample=False) # # conv 5 x = ResidualBlock(x, filters=512, kernel_size=(3, 3), weight_decay=weight_decay, downsample=True) x = ResidualBlock(x, filters=512, kernel_size=(3, 3), weight_decay=weight_decay, downsample=False) x = AveragePooling2D(pool_size=(5, 15), padding='valid')(x) x = Flatten()(x) x1 = Dense(classes, name="digit1", activation='softmax', kernel_initializer='he_normal')(x) x2 = Dense(classes, name="digit2", activation='softmax', kernel_initializer='he_normal')(x) x3 = Dense(classes, name="digit3", activation='softmax', kernel_initializer='he_normal')(x) x4 = Dense(classes, name="digit4", activation='softmax', kernel_initializer='he_normal')(x) x5 = Dense(classes, name="digit5", activation='softmax', kernel_initializer='he_normal')(x) x6 = Dense(classes, name="digit6", activation='softmax', kernel_initializer='he_normal')(x) model = Model(input, outputs=[x1, x2, x3, x4, x5, x6], name='ResNet18') return model
def retinanet_resnet(inputs, K, A): """Retinanet with resnet backbone. Classification and regression networks share weights across feature pyramid layers""" # --- Define kwargs dictionary kwargs1 = { 'kernel_size': (1, 1, 1), 'padding': 'valid', } kwargs3 = { 'kernel_size': (1, 3, 3), 'padding': 'same', } kwargs7 = { 'kernel_size': (1, 7, 7), 'padding': 'valid', } # --- Define block components conv1 = lambda x, filters, strides: layers.Conv3D( filters=filters, strides=strides, **kwargs1)(x) conv3 = lambda x, filters, strides: layers.Conv3D( filters=filters, strides=strides, **kwargs3)(x) relu = lambda x: layers.LeakyReLU()(x) conv7 = lambda x, filters, strides: layers.Conv3D( filters=filters, strides=strides, **kwargs7)(x) max_pool = lambda x, pool_size, strides: layers.MaxPooling3D( pool_size=pool_size, strides=strides, padding='valid')(x) norm = lambda x: layers.BatchNormalization()(x) add = lambda x, y: layers.Add()([x, y]) zeropad = lambda x, padding: layers.ZeroPadding3D(padding=padding)(x) upsamp2x = lambda x: layers.UpSampling3D(size=(1, 2, 2))(x) # --- Define stride-1, stride-2 blocks # conv1 = lambda filters, x : relu(conv(x, filters, strides=1)) # conv2 = lambda filters, x : relu(conv(x, filters, strides=(2, 2))) # --- Residual blocks # conv blocks conv_1 = lambda filters, x, strides: relu( norm(conv1(x, filters, strides=strides))) conv_2 = lambda filters, x: relu(norm(conv3(x, filters, strides=1))) conv_3 = lambda filters, x: norm(conv1(x, filters, strides=1)) conv_sc = lambda filters, x, strides: norm( conv1(x, filters, strides=strides)) conv_block = lambda filters1, filters2, x, strides: relu( add(conv_3(filters2, conv_2(filters1, conv_1(filters1, x, strides))), conv_sc(filters2, x, strides))) # identity blocks identity_1 = lambda filters, x: relu(norm(conv1(x, filters, strides=1))) identity_2 = lambda filters, x: relu(norm(conv3(x, filters, strides=1))) identity_3 = lambda filters, x: norm(conv1(x, filters, strides=1)) identity_block = lambda filters1, filters2, x: relu( add( identity_3(filters2, identity_2(filters1, identity_1(filters1, x)) ), x)) # --- feature pyramid blocks fp_block = lambda x, y: add(upsamp2x(x), conv1(y, 256, strides=1)) # --- classification head class_subnet = classification_head(K, A) # --- regression head box_subnet = regression_head(A) # --- ResNet-50 backbone # stage 1 c2 1/4 res1 = max_pool(zeropad( relu( norm( conv7(zeropad(inputs['dat'], (0, 3, 3)), 64, strides=(1, 2, 2)))), (0, 1, 1)), (1, 3, 3), strides=(1, 2, 2)) # stage 2 c2 1/4 res2 = identity_block( 64, 256, identity_block(64, 256, conv_block(64, 256, res1, strides=1))) # stage 3 c3 1/8 res3 = identity_block( 128, 512, identity_block( 128, 512, identity_block(128, 512, conv_block(128, 512, res2, strides=(1, 2, 2))))) # stage 4 c4 1/16 res4 = identity_block( 256, 1024, identity_block( 256, 1024, identity_block( 256, 1024, identity_block( 256, 1024, identity_block( 256, 1024, conv_block(256, 1024, res3, strides=(1, 2, 2))))))) # stage 5 c5 1/32 res5 = identity_block( 512, 2048, identity_block(512, 2048, conv_block(512, 2048, res4, strides=(1, 2, 2)))) # --- Feature Pyramid Network architecture # p5 1/32 fp5 = conv1(res5, 256, strides=1) # p4 1/16 fp4 = fp_block(fp5, res4) p4 = conv3(fp4, 256, strides=1) # p3 1/8 fp3 = fp_block(fp4, res3) p3 = conv3(fp3, 256, strides=1) # p6 1/4 # p6 = conv3(fp5, 256, strides=(2, 2)) # p7 1/2 # p7 = conv3(relu(p6), 256, strides=(2, 2)) feature_pyramid = [p3, p4, fp5] # lambda layer that allows multiple outputs from a shared model to have specific names # layers.Lambda(lambda x:x, name=name)() # --- Class subnet class_outputs = [class_subnet(features) for features in feature_pyramid] # --- Box subnet box_outputs = [box_subnet(features) for features in feature_pyramid] # --- put class and box outputs in dictionary logits = { 'cls-c3': layers.Lambda(lambda x: x, name='cls-c3')(class_outputs[0]), 'reg-c3': layers.Lambda(lambda x: x, name='reg-c3')(box_outputs[0]), 'cls-c4': layers.Lambda(lambda x: x, name='cls-c4')(class_outputs[1]), 'reg-c4': layers.Lambda(lambda x: x, name='reg-c4')(box_outputs[1]), 'cls-c5': layers.Lambda(lambda x: x, name='cls-c5')(class_outputs[2]), 'reg-c5': layers.Lambda(lambda x: x, name='reg-c5')(box_outputs[2]) } model = Model(inputs=inputs, outputs=logits) return model