def build_baseline_model():
input_shape = (160, 120, 3)
input = Input(shape=input_shape)
m = Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
activation='tanh',
kernel_regularizer=l1(0.0001))(input)
m = MaxPool2D(pool_size=(2, 2))(m)
m = Dropout(rate=0.1)(m)
m = Flatten()(m)
m = Dense(256,
activation='relu',
kernel_initializer='he_normal',
kernel_regularizer=l1(0.0001))(m)
m = Dropout(rate=0.12)(m)
output = Dense(5, activation='softmax')(m)
m = Model(inputs=input, outputs=output)
m.compile(optimizer=_opt,
loss=_loss,
metrics=_metrics)
return m
def build_category_parinject_model(nr_categories,
sentence_length,
vocabulary_size,
embedding_matrix,
embedding_dimensions=200,
loss_function='categorical_crossentropy',
optimizer='RMSprop',
nr_nodes=256,
nr_gpus=1,
concat_add='add',
trainable_embedding=False,
dropout=True,
cat_embedding=False,
attribute_included=False):
if cat_embedding == False:
ce1 = Input(shape=(nr_categories, ))
ce2 = Dense(nr_nodes, activation='relu')(ce1)
else:
if attribute_included == False:
ce1 = Input(shape=(1, ))
ce2 = Embedding(nr_categories, nr_nodes, trainable=True)(ce1)
ce2 = Reshape((nr_nodes, ))(ce2)
else:
ce1 = Input(shape=(nr_categories, ))
ce2 = Embedding(nr_categories,
nr_nodes,
trainable=True,
mask_zero=True)(ce1)
ce2 = Flatten()(ce2)
ce2 = Dense(nr_nodes, activation='relu')(ce2)
ce3 = RepeatVector(sentence_length)(ce2)
input_desc = Input((sentence_length, ))
se2 = Embedding(vocabulary_size,
embedding_dimensions,
trainable=trainable_embedding,
weights=[embedding_matrix],
mask_zero=True)(input_desc)
model = concatenate([ce3, se2])
# Encode --> add LSTM layers
if dropout:
model = Dropout(0.5)(model)
model = LSTM(nr_nodes, activation='sigmoid')(model)
# Add decoder / prediction layers
model = Dense(nr_nodes, activation='relu')(model)
if dropout:
model = Dropout(0.5)(model)
output = Dense(vocabulary_size, activation='softmax')(model)
model = k.Model(inputs=[ce1, input_desc], outputs=output)
model.compile(loss=loss_function,
optimizer=optimizer,
metrics=["accuracy"])
return model
def build_par_inject_model(cnn_input,
sentence_length,
vocabulary_size,
embedding_matrix,
embedding_dimensions=200,
loss_function='categorical_crossentropy',
optimizer='RMSprop',
image_manipulation=False,
nodes_per_layer=256,
dropout=True,
trainable_embedding=False):
input_desc = Input((sentence_length, ))
# Get image feature
fe1 = cnn_input
fe2 = Dense(nodes_per_layer, activation='relu')(fe1)
if dropout:
print("ADDED DROPOUT")
fe2 = Dropout(0.5)(fe2)
# Embed the text
embedding = Embedding(vocabulary_size,
embedding_dimensions,
trainable=trainable_embedding,
weights=[embedding_matrix],
mask_zero=True)(input_desc)
# Use RepeatVector to generate the correct layer size
fe3 = RepeatVector(sentence_length)(fe2)
# Concatenate the image features with the embedding
model = concatenate([fe3, embedding])
# Encode --> add LSTM layers
if dropout:
model = Dropout(0.5)(model)
model = LSTM(nodes_per_layer, activation='sigmoid')(model)
# Add decoder / prediction layers
model = Dense(nodes_per_layer, activation='relu')(model)
if dropout:
model = Dropout(0.5)(model)
output = Dense(vocabulary_size, activation='softmax')(model)
model = k.Model(inputs=[cnn_input, input_desc], outputs=output)
model.compile(loss=loss_function,
optimizer=optimizer,
metrics=["accuracy"])
return model
output_layer = Dense(n_classes, activation='softmax')(model)
model = Model(base_model.input, output_layer)
# Save model structure to file
with open(savepath + 'model.txt', 'w') as txtfile:
model.summary(print_fn=lambda x: txtfile.write(x + '\n'))
# Freeze layers
for layer in base_model.layers:
layer.trainable = False
# Compile and train the classifier
model.compile(optimizer=tf.keras.optimizers.SGD(lr=0.2,
momentum=0.9,
decay=0.01),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
history = model.fit(train_generator,
epochs=5,
validation_data=validation_generator)
# Unfreeze middle convolution layers and retrain with low LR
for layer in base_model.layers:
layer.trainable = True
model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.0005),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
