def train_model(self, themes_weight: ThemeWeights,
dataset: TrainValidationDataset, voc_size: int,
keras_callback: LambdaCallback):
article_length = dataset.article_length
theme_count = dataset.theme_count
model = tf.keras.Sequential([
keras.layers.Embedding(input_dim=voc_size,
input_length=article_length,
output_dim=self.embedding_output_dim,
mask_zero=True),
Dropout(0.3),
keras.layers.Conv1D(filters=64,
kernel_size=3,
input_shape=(voc_size,
self.embedding_output_dim),
activation=tf.nn.relu),
#keras.layers.MaxPooling1D(3),
#keras.layers.Bidirectional(keras.layers.LSTM(64)),
keras.layers.GlobalAveragePooling1D(),
Dropout(0.3),
keras.layers.Dense(theme_count, activation=tf.nn.sigmoid)
])
model.compile(optimizer=tf.keras.optimizers.Adam(clipnorm=1),
loss=WeightedBinaryCrossEntropy(
themes_weight.weight_array()),
metrics=[
AUC(multi_label=True),
BinaryAccuracy(),
TruePositives(),
TrueNegatives(),
FalseNegatives(),
FalsePositives(),
Recall(),
Precision()
],
run_eagerly=True)
model.summary()
self.__model__ = model
if self.__plot_directory is not None:
self.plot_model(self.__plot_directory)
# Fix for https://github.com/tensorflow/tensorflow/issues/38988
model._layers = [
layer for layer in model._layers if not isinstance(layer, dict)
]
callbacks = [ManualInterrupter(), keras_callback]
model.fit(dataset.trainData,
epochs=self.epochs,
steps_per_epoch=dataset.train_batch_count,
validation_data=dataset.validationData,
validation_steps=dataset.validation_batch_count,
callbacks=callbacks)
def train(self) -> TrainedModel:
"""
Trained the given model with data passed in the constructor.
"""
if self._trained:
raise Exception("Multiple training is not supported.")
theme_weights = ThemeWeights(self._theme_stats, self._theme_tokenizer)
self.logger.info("Parameters:")
self.logger.info("Batch size: %d", self.batch_size)
self._model.train_model(theme_weights, self._dataset,
self._article_tokenizer.voc_size, self)
return TrainedModel(self._model, self._article_tokenizer,
self._theme_tokenizer)
def train_model(self, themes_weight: ThemeWeights,
dataset: TrainValidationDataset,
voc_size: int,
keras_callback: LambdaCallback):
conv_reg = 0#0.015
dense_reg = 0#0.015
dropout_conv = 0.1#0.2
dropout_dense = 0.1#0.2
article_length = dataset.article_length
theme_count = dataset.theme_count
input = keras.layers.Input(shape=(dataset.article_length,))
layer = keras.layers.Embedding(input_dim=voc_size, input_length=article_length, output_dim=self.embedding_output_dim,
mask_zero=True)(input)
layer = Dropout(dropout_conv)(layer)
# Each convolution will have filter looking for some specific word combinations. For example a filter might
# return a small value except for "apple iphone".
conv1 = keras.layers.Conv1D(filters=64, kernel_size=2, input_shape=(voc_size, self.embedding_output_dim),
activation=tf.nn.relu, kernel_regularizer=l2(l=conv_reg))(layer)
conv1 = keras.layers.GlobalMaxPooling1D()(conv1)
conv1 = keras.layers.Dense(32, activation=tf.nn.relu)(conv1)
conv1 = Dropout(dropout_conv)(conv1)
conv2 = keras.layers.Conv1D(filters=64, kernel_size=3, input_shape=(voc_size, self.embedding_output_dim),
activation=tf.nn.relu, kernel_regularizer=l2(l=conv_reg))(layer)
conv2 = keras.layers.GlobalMaxPooling1D()(conv2)
conv2 = keras.layers.Dense(32, activation=tf.nn.relu)(conv2)
conv2 = Dropout(dropout_conv)(conv2)
conv3 = keras.layers.Conv1D(filters=64, kernel_size=1, input_shape=(voc_size, self.embedding_output_dim),
activation=tf.nn.relu, kernel_regularizer=l2(l=conv_reg))(layer)
conv3 = keras.layers.GlobalMaxPooling1D()(conv3)
conv3 = keras.layers.Dense(32, activation=tf.nn.relu)(conv3)
conv3 = Dropout(dropout_conv)(conv3)
conv4 = keras.layers.Conv1D(filters=40, kernel_size=5, input_shape=(voc_size, self.embedding_output_dim),
activation=tf.nn.relu, kernel_regularizer=l2(l=conv_reg))(layer)
conv4 = keras.layers.GlobalMaxPooling1D()(conv4)
conv4 = keras.layers.Dense(20, activation=tf.nn.relu)(conv4)
conv4 = Dropout(dropout_conv)(conv4)
layer = keras.layers.Concatenate()([conv1, conv2, conv3, conv4])
layer = keras.layers.Dense(32, activation=tf.nn.relu, kernel_regularizer=l2(l=conv_reg))(layer)
layer = keras.layers.Dropout(dropout_dense)(layer)
# layer = keras.layers.Dense(64, activation=tf.nn.relu, kernel_regularizer=l2(l=conv_reg))(layer)
# layer = keras.layers.Dropout(dropout_dense)(layer)
layer = keras.layers.Dense(theme_count, activation=tf.nn.sigmoid, kernel_regularizer=l2(l=dense_reg))(layer)
model = keras.Model(inputs=input, outputs=layer)
model.compile(optimizer=tf.keras.optimizers.Adam(clipnorm=1, learning_rate=0.00009),
loss=WeightedBinaryCrossEntropy(themes_weight.weight_array()),
metrics=[AUC(multi_label=True), BinaryAccuracy(), TruePositives(),
TrueNegatives(), FalseNegatives(), FalsePositives(),
Recall(), Precision()],
run_eagerly=None)
model.summary()
self._model = model
if self.__plot_directory is not None:
self.plot_model(self.__plot_directory)
# Fix for https://github.com/tensorflow/tensorflow/issues/38988
model._layers = [layer for layer in model._layers if not isinstance(layer, dict)]
callbacks = [ManualInterrupter(), keras_callback]
model.fit(dataset.trainData,
epochs=self.epochs,
steps_per_epoch=dataset.train_batch_count,
validation_data=dataset.validationData,
validation_steps=dataset.validation_batch_count,
callbacks=callbacks,
)
def train_model(self, themes_weight: ThemeWeights,
dataset: TrainValidationDataset, voc_size: int,
keras_callback: LambdaCallback):
input = keras.layers.Input(shape=(dataset.article_length))
outputs: List[keras.layers.Layer] = []
for i in range(0, dataset.theme_count):
print("")
dense = keras.layers.Embedding(
input_dim=voc_size, output_dim=self.embedding_size)(input)
ltsm = keras.layers.Bidirectional(
keras.layers.LSTM(self.LTSM_output_size,
recurrent_dropout=0.2,
dropout=0.2))(dense)
dropout = keras.layers.Dropout(0.2)(ltsm)
dense2 = keras.layers.Dense(units=self.dense2_output_size,
activation=tf.nn.relu)(dropout)
output = keras.layers.Dense(
units=1,
activation=tf.nn.sigmoid,
name=str(i),
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(dense2)
outputs.append(output)
if len(outputs) > 1:
outputs = [keras.layers.concatenate(outputs)]
else:
outputs = [outputs]
model = keras.Model(inputs=[input], outputs=outputs)
model.compile(
optimizer=tf.keras.optimizers.Adam(clipnorm=1, clipvalue=0.5),
#loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
loss=WeightedBinaryCrossEntropy(
weights=themes_weight.weight_list(), from_logits=True),
# loss = {"0" : tf.keras.losses.BinaryCrossentropy(from_logits=True),
# "1" : tf.keras.losses.BinaryCrossentropy(from_logits=True)},
metrics=[
metrics.AUC(multi_label=True),
metrics.BinaryAccuracy(),
metrics.TruePositives(),
metrics.TrueNegatives(),
metrics.FalseNegatives(),
metrics.FalsePositives(),
metrics.Recall(),
metrics.Precision()
],
run_eagerly=False)
model.summary()
keras.utils.plot_model(model,
self.__model_name__ + '.png',
show_shapes=True)
callbacks = [ManualInterrupter, keras_callback]
# model.fit(self.dataset.trainData, epochs=15, steps_per_epoch=self.dataset.train_batch_count,
# validation_data=self.dataset.validationData, validation_steps=self.dataset.validation_batch_count,
# callbacks=callbacks, class_weight=self.theme_weight)
# model.fit(self.dataset.trainData, epochs=10, steps_per_epoch=self.dataset.train_batch_count,
# validation_data=self.dataset.validationData, validation_steps=self.dataset.validation_batch_count,
# callbacks=callbacks, class_weight={ 0 : 1, 1 : 7.8, 2 : 4.3})
model.fit(dataset.trainData,
epochs=40,
steps_per_epoch=dataset.train_batch_count,
validation_data=dataset.validationData,
validation_steps=dataset.validation_batch_count,
callbacks=callbacks)
self.__model__ = model