def perceptual_stitched_loss(y_true, y_pred):
"""
calculates the perceptual loss for the 4rd block of VGG16, which has a shape of 8x8 and is evaluated on
pictures in [0,255].
Since this project works with pictures in [-1,1], a empirically evaluated scale factor is introduced.
:param y_true:
:param y_pred:
:return: the weighted perceptual loss of y_true and y_pred
"""
global SCALE
global percentage_perceptual
covered_area = y_true[:, :, :, -3:]
y_true = y_true[:, :, :, :-3]
# create the VGG16 Network and freeze its weights
my_vgg16 = VGG16(include_top=False,
weights='imagenet',
input_shape=[64, 64, 3])
my_vgg16.trainable = False
for l in my_vgg16.layers:
l.trainable = False
model = Model(inputs=my_vgg16.input,
outputs=my_vgg16.get_layer('block4_conv3').output)
model.trainable = False
# preprocessing, since VGG16 works with data in the range of [0,255], so the images have to be reverted
# the images are multiplied with the coverage-matrix in order to have all the non-covered pixels black
yt_new = preprocess_to_caffe(
revert_zero_center(y_true * covered_area) * 255.0)
yp_new = preprocess_to_caffe(
revert_zero_center(y_pred * covered_area) * 255.0)
return SCALE * percentage_perceptual * K.mean(
mean_squared_error(model(yt_new), model(yp_new)))
def get_transfer_model(num_classes, batch_size=20):
model = VGG16(include_top=True, weights='imagenet')
input_shape = model.layers[0].output_shape[1:3]
transfer_layer = model.get_layer('block5_pool')
conv_model = Model(inputs=model.input,
outputs=transfer_layer.output)
# Start a new Keras Sequential model.
new_model = Sequential()
# Add the convolutional part of the VGG16 model from above.
new_model.add(conv_model)
# Flatten the output of the VGG16 model because it is from a
# convolutional layer.
new_model.add(Flatten())
# Add a dense (aka. fully-connected) layer.
# This is for combining features that the VGG16 model has
# recognized in the image.
new_model.add(Dense(1024, activation='relu'))
# Add a dropout-layer which may prevent overfitting and
# improve generalization ability to unseen data e.g. the test-set.
new_model.add(Dropout(0.5))
# Add the final layer for the actual classification.
new_model.add(Dense(num_classes, activation='softmax'))
optimizer = Adam(lr=1e-5)
loss = 'categorical_crossentropy'
metrics = ['categorical_accuracy']
conv_model.trainable = False
for layer in conv_model.layers:
layer.trainable = False
new_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
return new_model
def vgg_loss_fixed_img(y_true, y_pred):
vgg19 = VGG19(include_top=False,
weights='imagenet',
input_shape=image_shape)
vgg19.trainable = False
# Make trainable as False
for layer in vgg19.layers:
layer.trainable = False
model = Model(inputs=vgg19.input,
outputs=vgg19.get_layer('block5_conv4').output)
model.trainable = False
return bk.mean(bk.square(model(y_true) - model(y_pred)))
def create_res_dem_bc(self, kernel_initializer = 'he_normal', img_flat_len = 1024, only_emb = False):
attr_input = layers.Input(shape = (50,), name = 'attr')
word_emb = layers.Input(shape = (self.wv_len,), name = 'wv')
imag_classifier = layers.Input(shape = (img_flat_len,), name = 'img')
label = layers.Input(shape = (1,), name = 'label')
attr_dense = layers.Dense(self.wv_len, use_bias = True, kernel_initializer=kernel_initializer,
kernel_regularizer = l2(1e-4), name = 'attr_dense')(attr_input)
ini_dem_model = self.create_dem_bc(kernel_initializer = 'he_normal',
img_flat_len = img_flat_len,
only_emb = True)
ini_dem_model.load_weights('./only_emb.h5')
ini_dem_model_part = Model(inputs = ini_dem_model.inputs[2],
outputs = ini_dem_model.outputs[0])
ini_dem_model_part.trainable = False
ini_attr_word_emb_dense = ini_dem_model_part([word_emb])
if only_emb:
attr_word_emb = word_emb
else:
attr_word_emb = layers.Concatenate(name = 'attr_word_emb')([word_emb, attr_dense])
attr_word_emb_dense = self.full_connect_layer(attr_word_emb, hidden_dim = [
int(img_flat_len * 2),
int(img_flat_len * 1.5),
int(img_flat_len * 1.25),
int(img_flat_len)
], \
activation = 'relu', resnet = False, drop_out_ratio = 0.2)
attr_word_emb_dense = layers.Lambda(lambda x: x[0] + x[1])([attr_word_emb_dense, ini_attr_word_emb_dense])
attr_x_img = layers.Lambda(lambda x: x[0] * x[1], name = 'attr_x_img')([attr_word_emb_dense, imag_classifier])
# attr_x_img = layers.Concatenate(name = 'attr_x_img')([attr_word_emb_dense, imag_classifier])
attr_img_input = layers.Input(shape = (img_flat_len,), name = 'attr_img_input')
# attr_img_input = layers.Input(shape = (img_flat_len * 2,), name = 'attr_img_input')
proba = self.full_connect_layer(attr_img_input, hidden_dim = [1], activation = 'sigmoid')
attr_img_model = Model(inputs = attr_img_input, outputs = proba, name = 'attr_x_img_model')
out = attr_img_model([attr_x_img])
bc_loss = K.mean(binary_crossentropy(label, out))
model = Model([imag_classifier, attr_input, word_emb, label], outputs = [attr_word_emb_dense, out])
model.add_loss(bc_loss)
model.compile(optimizer=Adam(lr=1e-4), loss=None)
return model
# Add the final layer for the actual classification.
new_model.add(Dense(num_classes, activation='softmax'))
optimizer = Adam(lr=1e-5)
loss = 'categorical_crossentropy'
metrics = ['categorical_accuracy']
def print_layer_trainable():
for layer in conv_model.layers:
print("{0}:\t{1}".format(layer.trainable, layer.name))
print_layer_trainable()
conv_model.trainable = False
for layer in conv_model.layers:
layer.trainable = False
print_layer_trainable()
new_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
epochs = 5
steps_per_epoch = 50
history = new_model.fit_generator(generator=generator_train,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
class_weight=class_weight,
validation_data=generator_test,
validation_steps=steps_test)
# plot_training_history(history)
# recognized in the image.
new_model.add(Dense(1024, activation='relu'))
# Add a dropout-layer which may prevent overfitting and
# improve generalization ability to unseen data e.g. the test-set.
new_model.add(Dropout(0.5))
# Add the final layer for the actual classification.
new_model.add(Dense(num_classes, activation='softmax'))
optimizer = Adam(lr=1e-5)
loss = 'categorical_crossentropy'
metrics = ['categorical_accuracy']
# since we are not using any pretrained weights, we set all layers to 'trainable'
conv_model.trainable = True
for layer in conv_model.layers:
layer.trainable = 1
# compile model
new_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
# train the model
history = new_model.fit_generator(generator=generator_train,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
class_weight=class_weight,
validation_data=generator_test,
validation_steps=steps_test)
result = new_model.evaluate_generator(generator_test, steps=steps_test)
def create_model(learning_rate, num_dense_layers,
num_dense_nodes, activation):
"""
Hyper-parameters:
learning_rate: Learning-rate for the optimizer.
num_dense_layers: Number of dense layers.
num_dense_nodes: Number of nodes in each dense layer.
activation: Activation function for all layers.
"""
model = VGG16(include_top=True, weights='imagenet')
# We have checked separately and found the last convolutional layer of VGG-16 model is called 'block5_pool'
# We refer to this layer as the Transfer Layer because its output will be re-routed to our new fully-connected neural network
# which will do the classification for our dataset
transfer_layer = model.get_layer('block5_pool')
# Next is to create a new model using Keras API
# First we take the part of the VGG16 model from its input-layer to the output of the transfer-layer
conv_model = Model(inputs=model.input, outputs=transfer_layer.output)
# Next we will build a new model on top of this
# Start a new Keras Sequential model.
new_model = Sequential()
# Add the convolutional part of the VGG16 model from above.
new_model.add(conv_model)
# Flatten the 4-rank output of the convolutional layers
# to 2-rank that can be input to a fully-connected / dense layer.
new_model.add(Flatten())
# Add fully-connected / dense layers.
# The number of layers is a hyper-parameter we want to optimize.
for i in range(num_dense_layers):
# Name of the layer. This is not really necessary
# because Keras should give them unique names.
name = 'layer_dense_{0}'.format(i+1)
# Add the dense / fully-connected layer to the model.
# This has two hyper-parameters we want to optimize:
# The number of nodes and the activation function.
new_model.add(Dense(num_dense_nodes,
activation=activation,
name=name))
# Last fully-connected / dense layer with softmax-activation
# for use in classification.
new_model.add(Dense(num_classes, activation='softmax'))
# Use the Adam method for training the network.
# Our objective is to find the best learning-rate for the Adam method.
optimizer = Adam(lr=learning_rate)
# In Transfer Learning we intend to reuse the pre-trained VGG16 model as it is, so we will disable training for all its layers
conv_model.trainable = False
for layer in conv_model.layers:
layer.trainable = False
new_model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
return new_model
def init_model(backbone_model_name, freeze_backbone_model, input_shape,
attribute_name_to_label_encoder_dict, use_batchnormalization,
dropout_rate, kernel_regularization_factor,
bias_regularization_factor, gamma_regularization_factor,
beta_regularization_factor, evaluation_only,
pretrained_model_file_path):
def _add_regularizers(model, kernel_regularization_factor,
bias_regularization_factor,
gamma_regularization_factor,
beta_regularization_factor):
update_model_required = False
for layer in model.layers:
if not np.isclose(kernel_regularization_factor, 0.0) and hasattr(
layer, "kernel_regularizer"):
update_model_required = True
layer.kernel_regularizer = l2(kernel_regularization_factor)
if not np.isclose(bias_regularization_factor, 0.0) and hasattr(
layer, "bias_regularizer"):
if layer.use_bias:
update_model_required = True
layer.bias_regularizer = l2(bias_regularization_factor)
if not np.isclose(gamma_regularization_factor, 0.0) and hasattr(
layer, "gamma_regularizer"):
if layer.scale:
update_model_required = True
layer.gamma_regularizer = l2(gamma_regularization_factor)
if not np.isclose(beta_regularization_factor, 0.0) and hasattr(
layer, "beta_regularizer"):
if layer.center:
update_model_required = True
layer.beta_regularizer = l2(beta_regularization_factor)
if update_model_required:
print("Adding regularizers ...")
# https://github.com/keras-team/keras/issues/2717#issuecomment-447570737
vanilla_weights = model.get_weights()
model = model_from_json(model.to_json(),
custom_objects={
"tf": tf,
"swish": tf.nn.swish
})
model.set_weights(vanilla_weights)
return model
# Initiate the backbone model
query_result = BackboneWrapper().query_by_model_name(backbone_model_name)
assert query_result is not None, "Backbone {} is not supported.".format(
backbone_model_name)
model_instantiation, preprocess_input, _ = query_result
backbone_model_weights = None if len(
pretrained_model_file_path) > 0 else "imagenet"
backbone_model = model_instantiation(input_shape=input_shape,
weights=backbone_model_weights,
include_top=False)
if freeze_backbone_model:
for layer in backbone_model.layers:
layer.trainable = False
# Add GlobalAveragePooling2D
global_average_pooling_tensor = GlobalAveragePooling2D()(
backbone_model.output)
# https://arxiv.org/pdf/1801.07698v1.pdf Section 3.2.2 Output setting
# https://arxiv.org/pdf/1807.11042.pdf
classification_embedding_tensor = global_average_pooling_tensor
if use_batchnormalization:
classification_embedding_tensor = BatchNormalization()(
classification_embedding_tensor)
if dropout_rate > 0:
classification_embedding_tensor = Dropout(
rate=dropout_rate)(classification_embedding_tensor)
# Add categorical crossentropy loss
classification_output_tensor_list = []
for attribute_name, label_encoder in attribute_name_to_label_encoder_dict.items(
):
classification_output_tensor = Dense(
units=len(label_encoder.classes_),
activation="softmax",
name="{}_classification_output".format(attribute_name))(
classification_embedding_tensor)
classification_output_tensor_list.append(classification_output_tensor)
classification_loss_function_list = [
"categorical_crossentropy"
] * len(classification_output_tensor_list)
# Define the model
model = Model(inputs=[backbone_model.input],
outputs=classification_output_tensor_list)
model = _add_regularizers(model, kernel_regularization_factor,
bias_regularization_factor,
gamma_regularization_factor,
beta_regularization_factor)
if evaluation_only:
print("Freezing the whole model in the evaluation_only mode ...")
model.trainable = False
# Compile the model
extra_attributes_num = len(attribute_name_to_label_encoder_dict) - 1
loss_weights = [1.0] + (np.ones(extra_attributes_num) /
extra_attributes_num).tolist()
model.compile(optimizer=Adam(),
loss=classification_loss_function_list,
loss_weights=loss_weights,
metrics={
"grapheme_classification_output": ["accuracy"],
"consonant_diacritic_classification_output": ["accuracy"],
"grapheme_root_classification_output": ["accuracy"],
"vowel_diacritic_classification_output": ["accuracy"]
})
# Print the summary of the model
print("Summary of model:")
model.summary()
lq.models.summary(model)
# Load weights from the pretrained model
if len(pretrained_model_file_path) > 0:
assert os.path.isfile(pretrained_model_file_path)
print("Loading weights from {} ...".format(pretrained_model_file_path))
model.load_weights(pretrained_model_file_path)
return model, preprocess_input
x, y = loadXY(paths[batch_start:limit], "trainingSet/")
yield (x, y)
batch_start += batch_size
batch_end += batch_size
vggModel = VGG16(include_top=False,
weights="imagenet",
input_shape=(None, None, 3))
transfer_layer = vggModel.get_layer('block5_conv3')
vggModel = Model(inputs=vggModel.input, outputs=transfer_layer.output)
vggModel.trainable = False
newModel = Sequential()
newModel.add(vggModel)
#newModel.add(UpSampling2D((2,2)))
newModel.add(BatchNormalization())
newModel.add(Conv2D(512, (3, 3), padding='same', activation="relu"))
newModel.add(BatchNormalization())
newModel.add(Conv2D(512, (3, 3), padding='same', activation="relu"))
newModel.add(UpSampling2D((2, 2)))
newModel.add(BatchNormalization())
newModel.add(Conv2D(256, (3, 3), padding='same', activation="relu"))
newModel.add(BatchNormalization())
newModel.add(Conv2D(256, (3, 3), padding='same', activation="relu"))
