def test_binary_accuracy(self):
acc_obj = metrics.BinaryAccuracy(name='my_acc')
# check config
assert acc_obj.name == 'my_acc'
assert acc_obj.stateful
assert len(acc_obj.weights) == 2
assert acc_obj.dtype == 'float32'
# verify that correct value is returned
result_t = acc_obj([[1], [0]], [[1], [0]])
result = K.eval(result_t)
assert result == 1 # 2/2
# check y_pred squeeze
result_t = acc_obj([[1], [1]], [[[1]], [[0]]])
result = K.eval(result_t)
assert np.isclose(result, 3. / 4., atol=1e-3)
# check y_true squeeze
result_t = acc_obj([[[1]], [[1]]], [[1], [0]])
result = K.eval(result_t)
assert np.isclose(result, 4. / 6., atol=1e-3)
# check with sample_weight
result_t = acc_obj([[1], [1]], [[1], [0]], [[0.5], [0.2]])
result = K.eval(result_t)
assert np.isclose(result, 4.5 / 6.7, atol=1e-3)
def test_metrics_correctness_with_dataset(self):
layers = [
keras.layers.Dense(8,
activation='relu',
input_dim=4,
kernel_initializer='ones'),
keras.layers.Dense(1,
activation='sigmoid',
kernel_initializer='ones')
]
model = testing_utils.get_model_from_layers(layers, (4, ))
model.compile(loss='binary_crossentropy',
metrics=['accuracy',
metrics_module.BinaryAccuracy()],
optimizer='rmsprop',
run_eagerly=testing_utils.should_run_eagerly())
np.random.seed(123)
x = np.random.randint(10, size=(100, 4)).astype(np.float32)
y = np.random.randint(2, size=(100, 1)).astype(np.float32)
dataset = tf.data.Dataset.from_tensor_slices((x, y))
dataset = dataset.batch(10)
outs = model.evaluate(dataset, steps=10)
self.assertEqual(np.around(outs[1], decimals=1), 0.5)
self.assertEqual(np.around(outs[2], decimals=1), 0.5)
y = np.zeros((100, 1), dtype=np.float32)
dataset = tf.data.Dataset.from_tensor_slices((x, y))
dataset = dataset.repeat(100)
dataset = dataset.batch(10)
outs = model.evaluate(dataset, steps=10)
self.assertEqual(outs[1], 0.)
self.assertEqual(outs[2], 0.)
def TrainModel(model, base_model, model_name):
task = Task.init(project_name="Ex3ModelTrains", task_name=model_name)
reporter = TrainsReporter()
# Show a summary of the model. Check the number of trainable parameters
model.summary()
# Compile the model
model.compile(loss=keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=keras.optimizers.Adam(),
metrics=[metrics.BinaryAccuracy()])
# Train the model
model.fit(train_ds,
steps_per_epoch=train_ds.samples / train_ds.batch_size,
epochs=20,
validation_data=valid_ds,
validation_steps=valid_ds.samples / valid_ds.batch_size,
callbacks=[reporter],
verbose=1)
# Unfreeze the base_model. Note that it keeps running in inference mode
# since we passed `training=False` when calling it. This means that
# the batchnorm layers will not update their batch statistics.
# This prevents the batchnorm layers from undoing all the training
# we've done so far.
base_model.trainable = True
reporter.epoch_ref = 20
score = model.evaluate(test_ds)
print('Test evaluation Score:', model.evaluate(test_ds))
print('validation evaluation Score:', model.evaluate(valid_ds))
model.compile(
optimizer=keras.optimizers.Adam(1e-5), # Low learning rate
loss=keras.losses.BinaryCrossentropy(from_logits=True),
metrics=[keras.metrics.BinaryAccuracy()],
)
model.fit(train_ds,
steps_per_epoch=train_ds.samples / train_ds.batch_size,
epochs=10,
validation_data=valid_ds,
validation_steps=valid_ds.samples / valid_ds.batch_size,
callbacks=[reporter],
verbose=1)
score = model.evaluate(test_ds)
print('Test evaluation Score:', model.evaluate(test_ds))
print('validation evaluation Score:', model.evaluate(valid_ds))
def all_experiment_model(hp):
PRE_TRAINED_MODEL = Xception(input_shape=(IMG_HEIGHT, IMG_WIDTH, 3),
include_top=False,
pooling='avg',
weights='imagenet')
for layer in PRE_TRAINED_MODEL.layers:
layer.trainable = False
x = layers.Flatten()(PRE_TRAINED_MODEL.output)
for i in range(
hp.Int('number_of_dense_dropout_blocks', min_value=0,
max_value=2)):
x = layers.Dense(hp.Int(f'dense_units_{i}',
min_value=1024,
max_value=4096,
step=512),
activation='relu')(x)
x = layers.Dropout(
hp.Float(f'dropout_probability_{i}',
min_value=0,
max_value=0.3,
step=0.05))(x)
x = layers.Dense(hp.Int(f'penultimate_dense_unit',
min_value=1024,
max_value=4096,
step=512),
activation='relu')(x)
x = layers.Dense(1, activation='sigmoid')(x)
MODEL = Model(PRE_TRAINED_MODEL.input, x)
chosen_loss = hp.Choice(
'loss', values=['SigmoidFocalCrossEntropy', 'BinaryCrossentropy'])
MODEL.compile(optimizer=hp.Choice('optimiser',
values=['Adam', 'RMSprop', 'SGD']),
loss=eval(chosen_loss)(),
metrics=[
metrics.BinaryAccuracy(name='acc'),
metrics.AUC(name='auc'),
metrics.FalsePositives(name='fp')
])
return MODEL
def optimiser_experiment_model(hp):
PRE_TRAINED_MODEL = Xception(input_shape=(IMG_HEIGHT, IMG_WIDTH, 3),
include_top=False,
pooling='avg',
weights='imagenet')
for layer in PRE_TRAINED_MODEL.layers:
layer.trainable = False
x = layers.Flatten()(PRE_TRAINED_MODEL.output)
x = layers.Dense(1, activation='sigmoid')(x)
MODEL = Model(PRE_TRAINED_MODEL.input, x)
MODEL.compile(optimizer=hp.Choice('optimiser',
values=['Adam', 'RMSprop', 'SGD']),
loss='binary_crossentropy',
metrics=[
metrics.BinaryAccuracy(name='acc'),
metrics.AUC(name='auc'),
metrics.FalsePositives(name='fp')
])
return MODEL
def loss_experiment_model(hp):
PRE_TRAINED_MODEL = Xception(input_shape=(IMG_HEIGHT, IMG_WIDTH, 3),
include_top=False,
pooling='avg',
weights='imagenet')
for layer in PRE_TRAINED_MODEL.layers:
layer.trainable = False
x = layers.Flatten()(PRE_TRAINED_MODEL.output)
x = layers.Dense(1, activation='sigmoid')(x)
MODEL = Model(PRE_TRAINED_MODEL.input, x)
chosen_loss = hp.Choice(
'loss', values=['SigmoidFocalCrossEntropy', 'BinaryCrossentropy'])
MODEL.compile(optimizer='adam',
loss=eval(chosen_loss)(),
metrics=[
metrics.BinaryAccuracy(name='acc'),
metrics.AUC(name='auc'),
metrics.FalsePositives(name='fp')
])
return MODEL
model.add(Dense(units_9, activation=activation_9))
model.add(Dropout(dropout_9))
model.add(Dense(units_10, activation=activation_10))
model.add(Dropout(dropout_10))
model.add(Dense(units_11, activation=activation_11))
model.add(Dropout(dropout_11))
model.add(Dense(units_12, activation=activation_12))
model.add(Dropout(dropout_12))
model.add(Dense(nb_classes, activation='sigmoid'))
METRICS = [
metrics.BinaryAccuracy(name='ACCURACY'),
metrics.Precision(name='PRECISION'),
metrics.Recall(name='RECALL'),
metrics.AUC(name='AUC'),
metrics.TruePositives(name='TP'),
metrics.TrueNegatives(name='TN'),
metrics.FalsePositives(name='FP'),
metrics.FalseNegatives(name='FN')]
model.compile(loss='binary_crossentropy',
optimizer=compile_optimizer,
metrics=METRICS)
# GENERATORS
train_datagen = ImageDataGenerator(
rescale=1. / 255,
weights = 'imagenet')
for layer in PRE_TRAINED_MODEL.layers:
layer.trainable = False
x = layers.Flatten()(PRE_TRAINED_MODEL.output)
x = layers.Dense(1024, activation = 'relu')(x)
x = layers.Dropout(0.2)(x)
x = layers.Dense(1, activation = 'sigmoid')(x)
MODEL = Model(PRE_TRAINED_MODEL.input, x)
MODEL.compile(optimizer = RMSprop(lr = LEARNING_RATE),
loss = 'binary_crossentropy',
metrics = [
metrics.BinaryAccuracy(name = 'acc'),
metrics.AUC(name = 'auc'),
metrics.FalsePositives(name = 'fp')
])
EARLY_STOPPING = EarlyStopping(monitor='fp',
verbose=1,
patience=50,
mode='max',
restore_best_weights=True)
CLASS_WEIGHT = {0: 0.001694915254237288,
1: 0.00017733640716439085}
# CLASS_WEIGHT = {0: 9,
# 1: 1}
# CLASS_WEIGHT = {0: 1,
def set_model_prediction_multi_label(model_path, test_set, custom_objects, with_f1=False):
"""Compute metrics of a set of models for multilabel model. Computed metrics are : Loss, Accuracy, F1-Score, Macro F1-Score.
Args:
model_path (string): path of folder that contains models.
test_set (pandas.DataFrame): DataFrame with column path for images path and label.
custom_objects (dict): Dict of custom objects to load with model.
with_f1 (bool, optional): True if the model is train with F1-Score metrics, otherwise False. Defaults to False.
"""
test_generator = generator(test_set['path'].to_numpy(),
test_set[['label_culture','label_coffee']].to_numpy(),
eurosat_params['mean'],
eurosat_params['std'],
batch_size=len(test_set))
prediction_set = []
evaluate = []
X, y = next(test_generator)
for path in os.listdir(model_path):
if path.split(".")[1] == 'h5':
restored_model = None
if with_f1:
restored_model = load_model(os.path.join(model_path, path), custom_objects, compile=False)
restored_model.compile(optimizer=Adam(learning_rate=0.00001), loss='binary_crossentropy',metrics=[metrics.BinaryAccuracy(name='accuracy'),metrics.Precision(name='precision'),metrics.Recall(name='recall'),f1_score_keras])
else :
restored_model = load_model(os.path.join(model_path, path), custom_objects)
evaluate.append(restored_model.evaluate(test_generator, steps=1))
prediction_set.append(np.where(restored_model.predict(X) > 0.5, 1, 0))
predictions = []
for pred in zip(*prediction_set):
culture_pred, coffee_pred = zip(*pred)
predictions.append(np.array([np.argmax(np.bincount(culture_pred)),
np.argmax(np.bincount(coffee_pred))]))
cm = multilabel_confusion_matrix(y, predictions)
plot_confusion_matrix(cm[0], ["Culture", "No-Culture"],"Confusion Matrix\nCulture vs No-Culture\nDenseNet 64x64")
plot_confusion_matrix(cm[1], ["Coffee", "Other"],"Confusion Matrix\nCoffee vs other\nDenseNet 64x64")
if with_f1:
losses, accs, precisions, recalls, f1 = zip(*evaluate)
else :
losses, accs, precisions, recalls = zip(*evaluate)
print("Global metrics")
print(f"Mean accuracy : {round(np.mean(accs),4)}")
print(f"Stdev accuracy : {round(np.std(accs),4)}")
print("\n")
print(f"Mean loss : {round(np.mean(losses),4)}")
print(f"Stdev loss : {round(np.std(losses),4)}")
print("\n")
culture_pred, coffee_pred = zip(*predictions)
culture_true, coffee_true = zip(*y)
print("Culture vs no-culture")
print(f"F1-Score Culture: {round(f1_score(culture_true, culture_pred, pos_label=0),4)}")
print(f"F1-Score No culture: {round(f1_score(culture_true, culture_pred, pos_label=1),4)}")
print(f"Macro F1-Score : {round(f1_score(culture_true, culture_pred, average='macro'),4)}")
print(f"\n")
print("Coffee vs other")
print(f"F1-Score Coffee: {round(f1_score(coffee_true, coffee_pred, pos_label=0),4)}")
print(f"F1-Score Other: {round(f1_score(coffee_true, coffee_pred, pos_label=1),4)}")
print(f"Macro F1-Score : {round(f1_score(coffee_true, coffee_pred, average='macro'),4)}")
def set_model_prediction(model_path, test_set, custom_objects, with_f1=False, labels=["Coffee", "Other"], title="title"):
"""Compute metrics of a set of models. Computed metrics are : Loss, Accuracy, F1-Score, Macro F1-Score.
Args:
model_path (string): path of folder that contains models.
test_set (pandas.DataFrame): DataFrame with column path for images path and label.
custom_objects (dict): Dict of custom objects to load with model.
with_f1 (bool, optional): True if the model is train with F1-Score metrics, otherwise False. Defaults to False.
labels (list, optional): Label for confusion Matrix. Defaults to ["Coffee", "Other"].
title (str, optional): Title of confusion matrix. Defaults to "title".
"""
test_generator = generator(test_set['path'].to_numpy(),
test_set['label'].to_numpy(),
eurosat_params['mean'],
eurosat_params['std'],
batch_size=len(test_set))
prediction_set = []
evaluate = []
X, y = next(test_generator)
for path in os.listdir(model_path):
if path.split(".")[1] == 'h5':
restored_model = None
if with_f1:
restored_model = load_model(os.path.join(model_path, path), custom_objects, compile=False)
restored_model.compile(optimizer=Adam(learning_rate=0.00001), loss='binary_crossentropy',metrics=[metrics.BinaryAccuracy(name='accuracy'),metrics.Precision(name='precision'),metrics.Recall(name='recall'),f1_score_keras])
else :
restored_model = load_model(os.path.join(model_path, path), custom_objects)
evaluate.append(restored_model.evaluate(test_generator, steps=1))
prediction_set.append(np.where(restored_model.predict(X) > 0.5, 1, 0).reshape(-1).tolist())
predictions = []
for pred in zip(*prediction_set):
predictions.append(np.argmax(np.bincount(pred)))
cm = confusion_matrix(y, predictions)
plot_confusion_matrix(cm, labels,title)
if with_f1:
losses, accs, precisions, recalls, f1 = zip(*evaluate)
else :
losses, accs, precisions, recalls = zip(*evaluate)
print(f"Mean accuracy : {round(np.mean(accs),4)}")
print(f"Stdev accuracy : {round(np.std(accs),4)}")
print("\n")
print(f"Mean loss : {round(np.mean(losses),4)}")
print(f"Stdev loss : {round(np.std(losses),4)}")
print("\n")
print(f"F1-Score {labels[0]}: {round(f1_score(y, predictions, pos_label=0),4)}")
print(f"F1-Score {labels[1]}: {round(f1_score(y, predictions, pos_label=1),4)}")
print(f"Macro F1-Score : {round(f1_score(y, predictions, average='macro'),4)}")
def test_binary_accuracy_threshold(self):
acc_obj = metrics.BinaryAccuracy(threshold=0.7)
result_t = acc_obj([[1], [1], [0], [0]], [[0.9], [0.6], [0.4], [0.8]])
result = K.eval(result_t)
assert np.isclose(result, 0.5, atol=1e-3)
