def perceptual_stitched_loss(y_true, y_pred): """ calculates the perceptual loss for the 4rd block of VGG16, which has a shape of 8x8 and is evaluated on pictures in [0,255]. Since this project works with pictures in [-1,1], a empirically evaluated scale factor is introduced. :param y_true: :param y_pred: :return: the weighted perceptual loss of y_true and y_pred """ global SCALE global percentage_perceptual covered_area = y_true[:, :, :, -3:] y_true = y_true[:, :, :, :-3] # create the VGG16 Network and freeze its weights my_vgg16 = VGG16(include_top=False, weights='imagenet', input_shape=[64, 64, 3]) my_vgg16.trainable = False for l in my_vgg16.layers: l.trainable = False model = Model(inputs=my_vgg16.input, outputs=my_vgg16.get_layer('block4_conv3').output) model.trainable = False # preprocessing, since VGG16 works with data in the range of [0,255], so the images have to be reverted # the images are multiplied with the coverage-matrix in order to have all the non-covered pixels black yt_new = preprocess_to_caffe( revert_zero_center(y_true * covered_area) * 255.0) yp_new = preprocess_to_caffe( revert_zero_center(y_pred * covered_area) * 255.0) return SCALE * percentage_perceptual * K.mean( mean_squared_error(model(yt_new), model(yp_new)))
def get_transfer_model(num_classes, batch_size=20): model = VGG16(include_top=True, weights='imagenet') input_shape = model.layers[0].output_shape[1:3] transfer_layer = model.get_layer('block5_pool') conv_model = Model(inputs=model.input, outputs=transfer_layer.output) # Start a new Keras Sequential model. new_model = Sequential() # Add the convolutional part of the VGG16 model from above. new_model.add(conv_model) # Flatten the output of the VGG16 model because it is from a # convolutional layer. new_model.add(Flatten()) # Add a dense (aka. fully-connected) layer. # This is for combining features that the VGG16 model has # recognized in the image. new_model.add(Dense(1024, activation='relu')) # Add a dropout-layer which may prevent overfitting and # improve generalization ability to unseen data e.g. the test-set. new_model.add(Dropout(0.5)) # Add the final layer for the actual classification. new_model.add(Dense(num_classes, activation='softmax')) optimizer = Adam(lr=1e-5) loss = 'categorical_crossentropy' metrics = ['categorical_accuracy'] conv_model.trainable = False for layer in conv_model.layers: layer.trainable = False new_model.compile(optimizer=optimizer, loss=loss, metrics=metrics) return new_model
def vgg_loss_fixed_img(y_true, y_pred): vgg19 = VGG19(include_top=False, weights='imagenet', input_shape=image_shape) vgg19.trainable = False # Make trainable as False for layer in vgg19.layers: layer.trainable = False model = Model(inputs=vgg19.input, outputs=vgg19.get_layer('block5_conv4').output) model.trainable = False return bk.mean(bk.square(model(y_true) - model(y_pred)))
def create_res_dem_bc(self, kernel_initializer = 'he_normal', img_flat_len = 1024, only_emb = False): attr_input = layers.Input(shape = (50,), name = 'attr') word_emb = layers.Input(shape = (self.wv_len,), name = 'wv') imag_classifier = layers.Input(shape = (img_flat_len,), name = 'img') label = layers.Input(shape = (1,), name = 'label') attr_dense = layers.Dense(self.wv_len, use_bias = True, kernel_initializer=kernel_initializer, kernel_regularizer = l2(1e-4), name = 'attr_dense')(attr_input) ini_dem_model = self.create_dem_bc(kernel_initializer = 'he_normal', img_flat_len = img_flat_len, only_emb = True) ini_dem_model.load_weights('./only_emb.h5') ini_dem_model_part = Model(inputs = ini_dem_model.inputs[2], outputs = ini_dem_model.outputs[0]) ini_dem_model_part.trainable = False ini_attr_word_emb_dense = ini_dem_model_part([word_emb]) if only_emb: attr_word_emb = word_emb else: attr_word_emb = layers.Concatenate(name = 'attr_word_emb')([word_emb, attr_dense]) attr_word_emb_dense = self.full_connect_layer(attr_word_emb, hidden_dim = [ int(img_flat_len * 2), int(img_flat_len * 1.5), int(img_flat_len * 1.25), int(img_flat_len) ], \ activation = 'relu', resnet = False, drop_out_ratio = 0.2) attr_word_emb_dense = layers.Lambda(lambda x: x[0] + x[1])([attr_word_emb_dense, ini_attr_word_emb_dense]) attr_x_img = layers.Lambda(lambda x: x[0] * x[1], name = 'attr_x_img')([attr_word_emb_dense, imag_classifier]) # attr_x_img = layers.Concatenate(name = 'attr_x_img')([attr_word_emb_dense, imag_classifier]) attr_img_input = layers.Input(shape = (img_flat_len,), name = 'attr_img_input') # attr_img_input = layers.Input(shape = (img_flat_len * 2,), name = 'attr_img_input') proba = self.full_connect_layer(attr_img_input, hidden_dim = [1], activation = 'sigmoid') attr_img_model = Model(inputs = attr_img_input, outputs = proba, name = 'attr_x_img_model') out = attr_img_model([attr_x_img]) bc_loss = K.mean(binary_crossentropy(label, out)) model = Model([imag_classifier, attr_input, word_emb, label], outputs = [attr_word_emb_dense, out]) model.add_loss(bc_loss) model.compile(optimizer=Adam(lr=1e-4), loss=None) return model
# Add the final layer for the actual classification. new_model.add(Dense(num_classes, activation='softmax')) optimizer = Adam(lr=1e-5) loss = 'categorical_crossentropy' metrics = ['categorical_accuracy'] def print_layer_trainable(): for layer in conv_model.layers: print("{0}:\t{1}".format(layer.trainable, layer.name)) print_layer_trainable() conv_model.trainable = False for layer in conv_model.layers: layer.trainable = False print_layer_trainable() new_model.compile(optimizer=optimizer, loss=loss, metrics=metrics) epochs = 5 steps_per_epoch = 50 history = new_model.fit_generator(generator=generator_train, epochs=epochs, steps_per_epoch=steps_per_epoch, class_weight=class_weight, validation_data=generator_test, validation_steps=steps_test) # plot_training_history(history)
# recognized in the image. new_model.add(Dense(1024, activation='relu')) # Add a dropout-layer which may prevent overfitting and # improve generalization ability to unseen data e.g. the test-set. new_model.add(Dropout(0.5)) # Add the final layer for the actual classification. new_model.add(Dense(num_classes, activation='softmax')) optimizer = Adam(lr=1e-5) loss = 'categorical_crossentropy' metrics = ['categorical_accuracy'] # since we are not using any pretrained weights, we set all layers to 'trainable' conv_model.trainable = True for layer in conv_model.layers: layer.trainable = 1 # compile model new_model.compile(optimizer=optimizer, loss=loss, metrics=metrics) # train the model history = new_model.fit_generator(generator=generator_train, epochs=epochs, steps_per_epoch=steps_per_epoch, class_weight=class_weight, validation_data=generator_test, validation_steps=steps_test) result = new_model.evaluate_generator(generator_test, steps=steps_test)
def create_model(learning_rate, num_dense_layers, num_dense_nodes, activation): """ Hyper-parameters: learning_rate: Learning-rate for the optimizer. num_dense_layers: Number of dense layers. num_dense_nodes: Number of nodes in each dense layer. activation: Activation function for all layers. """ model = VGG16(include_top=True, weights='imagenet') # We have checked separately and found the last convolutional layer of VGG-16 model is called 'block5_pool' # We refer to this layer as the Transfer Layer because its output will be re-routed to our new fully-connected neural network # which will do the classification for our dataset transfer_layer = model.get_layer('block5_pool') # Next is to create a new model using Keras API # First we take the part of the VGG16 model from its input-layer to the output of the transfer-layer conv_model = Model(inputs=model.input, outputs=transfer_layer.output) # Next we will build a new model on top of this # Start a new Keras Sequential model. new_model = Sequential() # Add the convolutional part of the VGG16 model from above. new_model.add(conv_model) # Flatten the 4-rank output of the convolutional layers # to 2-rank that can be input to a fully-connected / dense layer. new_model.add(Flatten()) # Add fully-connected / dense layers. # The number of layers is a hyper-parameter we want to optimize. for i in range(num_dense_layers): # Name of the layer. This is not really necessary # because Keras should give them unique names. name = 'layer_dense_{0}'.format(i+1) # Add the dense / fully-connected layer to the model. # This has two hyper-parameters we want to optimize: # The number of nodes and the activation function. new_model.add(Dense(num_dense_nodes, activation=activation, name=name)) # Last fully-connected / dense layer with softmax-activation # for use in classification. new_model.add(Dense(num_classes, activation='softmax')) # Use the Adam method for training the network. # Our objective is to find the best learning-rate for the Adam method. optimizer = Adam(lr=learning_rate) # In Transfer Learning we intend to reuse the pre-trained VGG16 model as it is, so we will disable training for all its layers conv_model.trainable = False for layer in conv_model.layers: layer.trainable = False new_model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) return new_model
def init_model(backbone_model_name, freeze_backbone_model, input_shape, attribute_name_to_label_encoder_dict, use_batchnormalization, dropout_rate, kernel_regularization_factor, bias_regularization_factor, gamma_regularization_factor, beta_regularization_factor, evaluation_only, pretrained_model_file_path): def _add_regularizers(model, kernel_regularization_factor, bias_regularization_factor, gamma_regularization_factor, beta_regularization_factor): update_model_required = False for layer in model.layers: if not np.isclose(kernel_regularization_factor, 0.0) and hasattr( layer, "kernel_regularizer"): update_model_required = True layer.kernel_regularizer = l2(kernel_regularization_factor) if not np.isclose(bias_regularization_factor, 0.0) and hasattr( layer, "bias_regularizer"): if layer.use_bias: update_model_required = True layer.bias_regularizer = l2(bias_regularization_factor) if not np.isclose(gamma_regularization_factor, 0.0) and hasattr( layer, "gamma_regularizer"): if layer.scale: update_model_required = True layer.gamma_regularizer = l2(gamma_regularization_factor) if not np.isclose(beta_regularization_factor, 0.0) and hasattr( layer, "beta_regularizer"): if layer.center: update_model_required = True layer.beta_regularizer = l2(beta_regularization_factor) if update_model_required: print("Adding regularizers ...") # https://github.com/keras-team/keras/issues/2717#issuecomment-447570737 vanilla_weights = model.get_weights() model = model_from_json(model.to_json(), custom_objects={ "tf": tf, "swish": tf.nn.swish }) model.set_weights(vanilla_weights) return model # Initiate the backbone model query_result = BackboneWrapper().query_by_model_name(backbone_model_name) assert query_result is not None, "Backbone {} is not supported.".format( backbone_model_name) model_instantiation, preprocess_input, _ = query_result backbone_model_weights = None if len( pretrained_model_file_path) > 0 else "imagenet" backbone_model = model_instantiation(input_shape=input_shape, weights=backbone_model_weights, include_top=False) if freeze_backbone_model: for layer in backbone_model.layers: layer.trainable = False # Add GlobalAveragePooling2D global_average_pooling_tensor = GlobalAveragePooling2D()( backbone_model.output) # https://arxiv.org/pdf/1801.07698v1.pdf Section 3.2.2 Output setting # https://arxiv.org/pdf/1807.11042.pdf classification_embedding_tensor = global_average_pooling_tensor if use_batchnormalization: classification_embedding_tensor = BatchNormalization()( classification_embedding_tensor) if dropout_rate > 0: classification_embedding_tensor = Dropout( rate=dropout_rate)(classification_embedding_tensor) # Add categorical crossentropy loss classification_output_tensor_list = [] for attribute_name, label_encoder in attribute_name_to_label_encoder_dict.items( ): classification_output_tensor = Dense( units=len(label_encoder.classes_), activation="softmax", name="{}_classification_output".format(attribute_name))( classification_embedding_tensor) classification_output_tensor_list.append(classification_output_tensor) classification_loss_function_list = [ "categorical_crossentropy" ] * len(classification_output_tensor_list) # Define the model model = Model(inputs=[backbone_model.input], outputs=classification_output_tensor_list) model = _add_regularizers(model, kernel_regularization_factor, bias_regularization_factor, gamma_regularization_factor, beta_regularization_factor) if evaluation_only: print("Freezing the whole model in the evaluation_only mode ...") model.trainable = False # Compile the model extra_attributes_num = len(attribute_name_to_label_encoder_dict) - 1 loss_weights = [1.0] + (np.ones(extra_attributes_num) / extra_attributes_num).tolist() model.compile(optimizer=Adam(), loss=classification_loss_function_list, loss_weights=loss_weights, metrics={ "grapheme_classification_output": ["accuracy"], "consonant_diacritic_classification_output": ["accuracy"], "grapheme_root_classification_output": ["accuracy"], "vowel_diacritic_classification_output": ["accuracy"] }) # Print the summary of the model print("Summary of model:") model.summary() lq.models.summary(model) # Load weights from the pretrained model if len(pretrained_model_file_path) > 0: assert os.path.isfile(pretrained_model_file_path) print("Loading weights from {} ...".format(pretrained_model_file_path)) model.load_weights(pretrained_model_file_path) return model, preprocess_input
x, y = loadXY(paths[batch_start:limit], "trainingSet/") yield (x, y) batch_start += batch_size batch_end += batch_size vggModel = VGG16(include_top=False, weights="imagenet", input_shape=(None, None, 3)) transfer_layer = vggModel.get_layer('block5_conv3') vggModel = Model(inputs=vggModel.input, outputs=transfer_layer.output) vggModel.trainable = False newModel = Sequential() newModel.add(vggModel) #newModel.add(UpSampling2D((2,2))) newModel.add(BatchNormalization()) newModel.add(Conv2D(512, (3, 3), padding='same', activation="relu")) newModel.add(BatchNormalization()) newModel.add(Conv2D(512, (3, 3), padding='same', activation="relu")) newModel.add(UpSampling2D((2, 2))) newModel.add(BatchNormalization()) newModel.add(Conv2D(256, (3, 3), padding='same', activation="relu")) newModel.add(BatchNormalization()) newModel.add(Conv2D(256, (3, 3), padding='same', activation="relu"))