def train(self, dataPointsTrain, dataPointsValidation, dataAugmentation):
def precision(y_true, y_pred):
from keras import backend as Kend
"""Precision metric.
Only computes a batch-wise average of precision.
Computes the precision, a metric for multi-label classification of
how many selected items are relevant.
"""
true_positives = Kend.sum(Kend.round(Kend.clip(y_true * y_pred, 0, 1)))
predicted_positives = Kend.sum(Kend.round(Kend.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + Kend.epsilon())
return precision
def recall(y_true, y_pred):
from keras import backend as Kend
"""Recall metric.
Only computes a batch-wise average of recall.
Computes the recall, a metric for multi-label classification of
how many relevant items are selected.
"""
true_positives = Kend.sum(Kend.round(Kend.clip(y_true * y_pred, 0, 1)))
possible_positives = Kend.sum(Kend.round(Kend.clip(y_true, 0, 1)))
recall = true_positives / (possible_positives + Kend.epsilon())
return recall
def fbeta_score(y_true, y_pred, beta=0.5):
from keras import backend as Kend
"""Computes the F score.
The F score is the weighted harmonic mean of precision and recall.
Here it is only computed as a batch-wise average, not globally.
This is useful for multi-label classification, where input samples can be
classified as sets of labels. By only using accuracy (precision) a model
would achieve a perfect score by simply assigning every class to every
input. In order to avoid this, a metric should penalize incorrect class
assignments as well (recall). The F-beta score (ranged from 0.0 to 1.0)
computes this, as a weighted mean of the proportion of correct class
assignments vs. the proportion of incorrect class assignments.
With beta = 1, this is equivalent to a F-measure. With beta < 1, assigning
correct classes becomes more important, and with beta > 1 the metric is
instead weighted towards penalizing incorrect class assignments.
"""
if beta < 0:
raise ValueError('The lowest choosable beta is zero (only precision).')
# If there are no true positives, fix the F score at 0 like sklearn.
if Kend.sum(Kend.round(Kend.clip(y_true, 0, 1))) == 0:
return 0
p = precision(y_true, y_pred)
r = recall(y_true, y_pred)
bb = beta ** 2
fbeta_score = (1 + bb) * (p * r) / (bb * p + r + Kend.epsilon())
return fbeta_score
def fmeasure(y_true, y_pred):
"""Computes the f-measure, the harmonic mean of precision and recall.
Here it is only computed as a batch-wise average, not globally.
"""
return fbeta_score(y_true, y_pred, beta=1)
if not self.logManager is None:
self.logManager.newLogSession("Training Model")
# optimizer = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
# optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-5)
optimizer = Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
if not self.logManager is None:
self.logManager.write("Training Strategy: " + str(optimizer.get_config()))
self.model.compile(loss="categorical_crossentropy",
optimizer=optimizer,
metrics=['accuracy', 'categorical_accuracy', fbeta_score, recall, precision])
filepath = self.experimentManager.modelDirectory + "/weights.best.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_categorical_accuracy', verbose=1, save_best_only=True,
mode='max')
early_stopping = EarlyStopping(monitor='val_loss', mode="min", patience=25)
reduce_lr = ReduceLROnPlateau(factor=0.5, monitor='val_loss', min_lr=1e-5, patience=2)
callbacks_list = [checkpoint, early_stopping, reduce_lr]
if dataAugmentation:
# this will do preprocessing and realtime data augmentation
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=True, # apply ZCA whitening
rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0, # randomly shift images horizontally (fraction of total width)
height_shift_range=0, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(dataPointsTrain.dataX)
# fit the model on the batches generated by datagen.flow()
history_callback = self.model.fit_generator(
datagen.flow(dataPointsTrain.dataX, dataPointsTrain.dataY, shuffle=True,
batch_size=self.batchSize),
steps_per_epoch=dataPointsTrain.dataX.shape[0] / self.batchSize,
epochs=self.numberOfEpochs,
validation_data=(dataPointsValidation.dataX, dataPointsValidation.dataY),
callbacks=callbacks_list)
else:
history_callback = self.model.fit(dataPointsTrain.dataX, dataPointsTrain.dataY,
batch_size=self.batchSize,
epochs=self.numberOfEpochs,
validation_data=(dataPointsValidation.dataX, dataPointsValidation.dataY),
shuffle=True,
callbacks=callbacks_list)
if not self.logManager is None:
self.logManager.write(str(history_callback.history))
self.logManager.endLogSession()
if not self.plotManager is None:
self.plotManager.createTrainingPlot(history_callback, self.modelName)
self.model.load_weights(self.experimentManager.modelDirectory + "/weights.best.hdf5")
self.model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy', 'categorical_accuracy', fbeta_score, recall, precision])
def train(self, dataPointsTrain, dataPointsValidation, dataAugmentation):
self.logManager.newLogSession("Creating Histogram plot: Training data")
self.plotManager.createDataArousalValenceHistogram(
dataPointsTrain, "train")
self.logManager.newLogSession(
"Creating Histogram plot: Validation data")
self.plotManager.createDataArousalValenceHistogram(
dataPointsValidation, "validation")
self.logManager.newLogSession("Training Model")
optimizer = Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
self.optimizerType = "Adamax"
self.logManager.write("--- Training Optimizer: " +
str(self.optimizerType))
self.logManager.write("--- Training Strategy: " +
str(optimizer.get_config()))
self.logManager.write("--- Training Batchsize: " + str(self.batchSize))
self.logManager.write("--- Training Number of Epochs: " +
str(self.numberOfEpochs))
self.model.compile(loss="mean_absolute_error",
optimizer=optimizer,
metrics=['mse', metrics.ccc])
filepath = self.experimentManager.modelDirectory + "/weights.best.hdf5"
checkPoint = ModelCheckpoint(
filepath,
monitor='val_arousal_output_mean_squared_error',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
reduce_lr = ReduceLROnPlateau(
monitor='val_valence_output_mean_squared_error',
factor=0.2,
patience=5,
min_lr=0.0001,
verbose=1)
history_callback = self.model.fit(
dataPointsTrain.dataX,
[dataPointsTrain.dataY[:, 0], dataPointsTrain.dataY[:, 1]],
batch_size=self.batchSize,
epochs=self.numberOfEpochs,
validation_data=(dataPointsValidation.dataX, [
dataPointsValidation.dataY[:, 0], dataPointsValidation.dataY[:,
1]
]),
shuffle=True,
callbacks=[checkPoint, reduce_lr])
self.logManager.write(str(history_callback.history))
self.plotManager.createTrainingPlot(history_callback)
self.logManager.endLogSession()
def train(self, dataPointsTrain, dataPointsValidation, dataAugmentation):
if not self.logManager is None:
self.logManager.newLogSession("Training Model")
# optimizer = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
# optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-5)
optimizer = Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
if not self.logManager is None:
self.logManager.write("Training Strategy: " +
str(optimizer.get_config()))
self.model.compile(loss="categorical_crossentropy",
optimizer=optimizer,
metrics=[
'accuracy', metrics.fbeta_score, metrics.recall,
metrics.precision
])
filepath = self.experimentManager.modelDirectory + "/weights.best.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_categorical_accuracy',
verbose=1,
save_best_only=True,
mode='max')
early_stopping = EarlyStopping(monitor='val_loss',
mode="min",
patience=25)
reduce_lr = ReduceLROnPlateau(factor=0.5,
monitor='val_loss',
min_lr=1e-5,
patience=2)
callbacks_list = [checkpoint, early_stopping, reduce_lr]
#concatenation = concatenate([armsmodel, lipsModel, audioModel])
#audioModel = dataPointsTrain.dataX[:, 0]
#print "Shape1:", numpy.array(dataPointsTrain.dataX[:, 0]).shape
armsTrain = []
lipsTrain = []
audioTrain = []
for i in dataPointsTrain.dataX:
armsTrain.append(i[0])
lipsTrain.append(i[1])
audioTrain.append(i[2])
armsTrain = numpy.array(armsTrain)
lipsTrain = numpy.array(lipsTrain)
audioTrain = numpy.array(audioTrain)
armsValidation = []
lipsValidation = []
audioValidation = []
for i in dataPointsValidation.dataX:
armsValidation.append(i[0])
lipsValidation.append(i[1])
audioValidation.append(i[2])
armsValidation = numpy.array(armsValidation)
lipsValidation = numpy.array(lipsValidation)
audioValidation = numpy.array(audioValidation)
history_callback = self.model.fit(
[armsTrain, lipsTrain, audioTrain],
dataPointsTrain.dataY,
batch_size=self.batchSize,
epochs=self.numberOfEpochs,
validation_data=([armsValidation, lipsValidation,
audioValidation], dataPointsValidation.dataY),
shuffle=True,
callbacks=callbacks_list)
if not self.logManager is None:
self.logManager.write(str(history_callback.history))
self.logManager.endLogSession()
if not self.plotManager is None:
self.plotManager.createTrainingPlot(history_callback,
self.modelName)
# convert the labels from integers to vectors
trainY = LabelBinarizer().fit_transform(trainY)
testY = LabelBinarizer().fit_transform(testY)
# initialize the optimizer and model
print("[INFO] compiling model...")
#opt = SGD(lr=0.01)
dataAugmentation = True
LR,EPS = 0.01, 0.1
#opt = SGD(lr=LR,momentum=0.9)
#print("Learning Rate: ",LR)#,"\tEpsilon: ",EPS)
#opt = Adagrad(lr=LR,epsilon=EPS) #LR should be 0.01 and eps 0.1 for this optimizer
#opt = Adam()
opt = Adamax(lr=LR)
print("Network Parameters:\n",opt.get_config())
model = MohlerNet4.build(width=32,height=32,depth=3,classes=3)
model.compile(loss="categorical_crossentropy", optimizer=opt,
metrics=["accuracy"])
# train the network
print("[INFO] training network...")
numEpochs = 50
batch_size = 16
if dataAugmentation == True:
train_datagen = IDG(
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
