def cnn_model(tp):
model = Sequential()
model.add(
Conv2D(tp["filter_1"], (tp["kernel_1"], tp["kernel_1"]),
padding="same",
activation='relu',
input_shape=(32, 32, 1)))
model.add(MaxPooling2D((tp["pool_1"], tp["pool_1"])))
model.add(
Conv2D(tp["filter_2"], (tp["kernel_2"], tp["kernel_2"]),
padding="same",
activation='relu'))
model.add(
Conv2D(tp["filter_3"], (tp["kernel_3"], tp["kernel_3"]),
padding="same",
activation='relu'))
model.add(Flatten())
model.add(Dense(tp["dense_units_1"], activation='relu'))
model.add(Dropout(tp["dropout_1"]))
model.add(Dense(tp["dense_units_2"], activation='relu'))
model.add(Dropout(tp["dropout_2"]))
model.add(Dense(tp["dense_units_3"], activation='relu'))
model.add(Dense(1, activation="sigmoid"))
model.compile(
loss="binary_crossentropy",
optimizer=Adam(lr=tp['learning_rate']),
metrics=["accuracy", metrics.AUC(name="auc")],
)
return model
def my_model(self):
input_shape = (self.dim, self.dim, 3)
nclass = self.nclass
input_ = Input(shape=input_shape)
conv1 = self.conv2d_bn(input_, 64, kernel_size=(3, 3), strides=(2, 2))
pool1 = MaxPool2D(pool_size=(3, 3), strides=(2, 2),
padding='same')(conv1)
conv2 = self.basic_block(64, 2, is_first_layer=False)(pool1)
pool2 = GlobalAvgPool2D()(conv2)
output_ = Dense(nclass, activation='softmax')(pool2)
model = Model(inputs=input_, outputs=output_)
model.compile(loss="categorical_crossentropy",
optimizer="adagrad",
metrics=[
"accuracy",
metrics.AUC(),
metrics.Precision(),
metrics.Recall()
])
return model
def test_auc(self, distribution):
label_prediction = ([0, 0, 1, 1], [0, 0.5, 0.3, 0.9])
with distribution.scope():
metric = metrics.AUC(num_thresholds=3)
self.evaluate([v.initializer for v in metric.variables])
updates = distribution.run(metric, args=label_prediction)
self.evaluate(updates)
self.assertAllClose(metric.result(), 0.75)
def __init__(self):
self.num_conv2d_layers=1
self.filters_2d=[16,32]
self.kernel_size_2d=[[3,3], [3,3]]
self.mpool_size_2d=[[2,2], [2,2]]
self.metric_type_map = {
'precision' : metrics.Precision(),
'recall' : metrics.Recall(),
'AUC' : metrics.AUC(),
'accuracy' : metrics.Accuracy(),
}
def test_unweighted(self):
self.setup()
auc_obj = metrics.AUC(num_thresholds=self.num_thresholds)
result = auc_obj(self.y_true, self.y_pred)
# tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]
# recall = [2/2, 1/(1+1), 0] = [1, 0.5, 0]
# fp_rate = [2/2, 0, 0] = [1, 0, 0]
# heights = [(1 + 0.5)/2, (0.5 + 0)/2] = [0.75, 0.25]
# widths = [(1 - 0), (0 - 0)] = [1, 0]
expected_result = (0.75 * 1 + 0.25 * 0)
assert np.allclose(K.eval(result), expected_result, atol=1e-3)
def __init__(self, X_train, y_train, X_val, y_val, X_test, y_test,
**kwargs):
self.model_dir = MODEL_DIR
(self.X_train, self.y_train) = (X_train, y_train)
(self.X_val, self.y_val) = (X_val, y_val)
(self.X_test, self.y_test) = (X_test, y_test)
self.metrics = [metrics.Recall(), metrics.Precision(), metrics.AUC()]
self.model = self.set_architecture()
self.callback = self.compile_model()
def test_weighted_roc_interpolation(self):
self.setup()
auc_obj = metrics.AUC(num_thresholds=self.num_thresholds)
result = auc_obj(self.y_true,
self.y_pred,
sample_weight=self.sample_weight)
# tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
# recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
# fp_rate = [3/3, 0, 0] = [1, 0, 0]
# heights = [(1 + 0.571)/2, (0.571 + 0)/2] = [0.7855, 0.2855]
# widths = [(1 - 0), (0 - 0)] = [1, 0]
expected_result = (0.7855 * 1 + 0.2855 * 0)
assert np.allclose(K.eval(result), expected_result, atol=1e-3)
def initializeNN():
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LeakyReLU
from keras.layers import Dropout
from keras import regularizers
from keras import metrics
#import tensorflow_addons as tfa
### Define metrics
metrics = [
metrics.CategoricalAccuracy(name="accuracy"),
metrics.FalseNegatives(name="fn"),
metrics.FalsePositives(name="fp"),
metrics.TrueNegatives(name="tn"),
metrics.TruePositives(name="tp"),
metrics.Precision(name="precision"),
metrics.Recall(name="recall"),
metrics.AUC(name='auc') #,
#tfa.metrics.CohenKappa(name='kappa')
]
# define the keras model
nn = Sequential()
nn.add(Dense(256, input_dim=102,
kernel_regularizer='l1')) #, activation='relu'))
nn.add(LeakyReLU(alpha=0.1))
nn.add(Dropout(0.1))
nn.add(Dense(128)) #, activation='relu'))#,kernel_regularizer='l1'))
nn.add(LeakyReLU(alpha=0.1))
nn.add(Dropout(0.1))
nn.add(Dense(64)) #, activation='relu'))#,kernel_regularizer='l1'))
nn.add(LeakyReLU(alpha=0.1))
nn.add(Dropout(0.1))
nn.add(Dense(64)) #, activation='relu'))#,kernel_regularizer='l1'))
nn.add(LeakyReLU(alpha=0.1))
nn.add(Dropout(0.1))
nn.add(Dense(31, activation='softmax'))
nn.compile(loss='categorical_crossentropy',
optimizer='Adamax',
metrics=metrics)
return nn
def test_manual_thresholds(self):
self.setup()
# Verify that when specified, thresholds are used instead of num_thresholds.
auc_obj = metrics.AUC(num_thresholds=2, thresholds=[0.5])
assert auc_obj.num_thresholds == 3
assert np.allclose(auc_obj.thresholds, [0.0, 0.5, 1.0], atol=1e-3)
result = auc_obj(self.y_true, self.y_pred)
# tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]
# recall = [2/2, 1/(1+1), 0] = [1, 0.5, 0]
# fp_rate = [2/2, 0, 0] = [1, 0, 0]
# heights = [(1 + 0.5)/2, (0.5 + 0)/2] = [0.75, 0.25]
# widths = [(1 - 0), (0 - 0)] = [1, 0]
expected_result = (0.75 * 1 + 0.25 * 0)
assert np.allclose(K.eval(result), expected_result, atol=1e-3)
def test_weighted_roc_minoring(self):
self.setup()
auc_obj = metrics.AUC(num_thresholds=self.num_thresholds,
summation_method='minoring')
result = auc_obj(self.y_true,
self.y_pred,
sample_weight=self.sample_weight)
# tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
# recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
# fp_rate = [3/3, 0, 0] = [1, 0, 0]
# heights = [min(1, 0.571), min(0.571, 0)] = [0.571, 0]
# widths = [(1 - 0), (0 - 0)] = [1, 0]
expected_result = (0.571 * 1 + 0 * 0)
assert np.allclose(K.eval(result), expected_result, atol=1e-3)
def dot_arci(
self,
dense_size,
dropout_rate,
output_dim,
):
input1 = Input(shape=(dense_size,), dtype='float32')
input2 = Input(shape=(dense_size,), dtype='float32')
input3 = Input(shape=(1,), dtype='float32')
input4 = Input(shape=(1,), dtype='float32')
question_dot12 = Multiply()([input3, input1])
question_dot23 = Multiply()([input3, input2])
action_dot12 = Multiply()([input4, input1])
action_dot23 = Multiply()([input4, input2])
question_mlp12 = Dense(dense_size, activation='relu')(question_dot12)
question_mlp23 = Dense(dense_size, activation='relu')(question_dot23)
action_mlp12 = Dense(dense_size, activation='relu')(action_dot12)
action_mlp23 = Dense(dense_size, activation='relu')(action_dot23)
concatenated_tensor = Concatenate(axis=1)(
[
question_mlp12,
question_mlp23,
action_mlp12,
action_mlp23
]
)
dropout = Dropout(rate=dropout_rate)(concatenated_tensor)
relevance_qa_dense = Dense(dense_size, activation='relu')(dropout)
output = Dense(output_dim, activation='softmax')(relevance_qa_dense)
model = Model(
inputs=[
input1,
input2,
input3,
input4
],
outputs=output
)
model.compile(
loss = 'binary_crossentropy',
optimizer='adam',
metrics = [metrics.AUC()]
)
print('Successfully build the dot arci model!')
return model
def test_weighted_pr_minoring(self):
self.setup()
auc_obj = metrics.AUC(num_thresholds=self.num_thresholds,
curve='PR',
summation_method='minoring')
result = auc_obj(self.y_true,
self.y_pred,
sample_weight=self.sample_weight)
# tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
# precision = [7/(7+3), 4/4, 0] = [0.7, 1, 0]
# recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
# heights = [min(0.7, 1), min(1, 0)] = [0.7, 0]
# widths = [(1 - 0.571), (0.571 - 0)] = [0.429, 0.571]
expected_result = (0.7 * 0.429 + 0 * 0.571)
assert np.allclose(K.eval(result), expected_result, atol=1e-3)
def train_cnn_model(x_train, y_train):
x_train = array(x_train)
x_train = x_train.reshape((len(x_train), 3, int(len(x_train[0])/3), 1))
y_train = array(y_train)
#create model
cnn_model = Sequential()
cnn_model.add(Conv2D(64,
kernel_size=3,
activation='relu',
input_shape=(3,21,1),
padding='same'))
cnn_model.add(layers.BatchNormalization(1))
cnn_model.add(Conv2D(64,
kernel_size=3,
activation='relu',
padding='same'))
cnn_model.add(layers.BatchNormalization(1))
cnn_model.add(MaxPooling2D(2,2))
cnn_model.add(Flatten())
cnn_model.add(Dense(512, activation = 'relu'))
cnn_model.add(Dense(1, activation='sigmoid'))
# compile and fit
cnn_model.compile(optimizer='Adam',
loss='binary_crossentropy',
metrics=['acc',
metrics.AUC(),
metrics.FalseNegatives(),
metrics.Recall(),
metrics.Precision(),
metrics.FalseNegatives(),
metrics.TrueNegatives(),
metrics.FalsePositives(),
metrics.TruePositives()])
cnn_history = cnn_model.fit(x_train, y_train,
epochs=100,
batch_size=16,
validation_split=0.2,
callbacks=[callbacks.EarlyStopping(monitor='val_loss', patience=5),
callbacks.LearningRateScheduler(scheduler)])
print("finish training cnn model")
return cnn_model, cnn_history
def test_config(self):
auc_obj = metrics.AUC(num_thresholds=100,
curve='PR',
summation_method='majoring',
name='auc_1')
assert auc_obj.name == 'auc_1'
assert len(auc_obj.weights) == 4
assert auc_obj.num_thresholds == 100
assert auc_obj.curve == metrics_utils.AUCCurve.PR
assert auc_obj.summation_method == metrics_utils.AUCSummationMethod.MAJORING
# Check save and restore config.
auc_obj2 = metrics.AUC.from_config(auc_obj.get_config())
assert auc_obj2.name == 'auc_1'
assert len(auc_obj2.weights) == 4
assert auc_obj2.num_thresholds == 100
assert auc_obj2.curve == metrics_utils.AUCCurve.PR
assert auc_obj2.summation_method == metrics_utils.AUCSummationMethod.MAJORING
def all_experiment_model(hp):
PRE_TRAINED_MODEL = Xception(input_shape=(IMG_HEIGHT, IMG_WIDTH, 3),
include_top=False,
pooling='avg',
weights='imagenet')
for layer in PRE_TRAINED_MODEL.layers:
layer.trainable = False
x = layers.Flatten()(PRE_TRAINED_MODEL.output)
for i in range(
hp.Int('number_of_dense_dropout_blocks', min_value=0,
max_value=2)):
x = layers.Dense(hp.Int(f'dense_units_{i}',
min_value=1024,
max_value=4096,
step=512),
activation='relu')(x)
x = layers.Dropout(
hp.Float(f'dropout_probability_{i}',
min_value=0,
max_value=0.3,
step=0.05))(x)
x = layers.Dense(hp.Int(f'penultimate_dense_unit',
min_value=1024,
max_value=4096,
step=512),
activation='relu')(x)
x = layers.Dense(1, activation='sigmoid')(x)
MODEL = Model(PRE_TRAINED_MODEL.input, x)
chosen_loss = hp.Choice(
'loss', values=['SigmoidFocalCrossEntropy', 'BinaryCrossentropy'])
MODEL.compile(optimizer=hp.Choice('optimiser',
values=['Adam', 'RMSprop', 'SGD']),
loss=eval(chosen_loss)(),
metrics=[
metrics.BinaryAccuracy(name='acc'),
metrics.AUC(name='auc'),
metrics.FalsePositives(name='fp')
])
return MODEL
def test_config_manual_thresholds(self):
auc_obj = metrics.AUC(num_thresholds=None,
curve='PR',
summation_method='majoring',
name='auc_1',
thresholds=[0.3, 0.5])
assert auc_obj.name == 'auc_1'
assert len(auc_obj.weights) == 4
assert auc_obj.num_thresholds == 4
assert np.allclose(auc_obj.thresholds, [0.0, 0.3, 0.5, 1.0], atol=1e-3)
assert auc_obj.curve == metrics_utils.AUCCurve.PR
assert auc_obj.summation_method == metrics_utils.AUCSummationMethod.MAJORING
# Check save and restore config.
auc_obj2 = metrics.AUC.from_config(auc_obj.get_config())
assert auc_obj2.name == 'auc_1'
assert len(auc_obj2.weights) == 4
assert auc_obj2.num_thresholds == 4
assert auc_obj2.curve == metrics_utils.AUCCurve.PR
assert auc_obj2.summation_method == metrics_utils.AUCSummationMethod.MAJORING
def test_weighted_pr_interpolation(self):
self.setup()
auc_obj = metrics.AUC(num_thresholds=self.num_thresholds, curve='PR')
result = auc_obj(self.y_true,
self.y_pred,
sample_weight=self.sample_weight)
# auc = (slope / Total Pos) * [dTP - intercept * log(Pb/Pa)]
# tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
# P = tp + fp = [10, 4, 0]
# dTP = [7-4, 4-0] = [3, 4]
# dP = [10-4, 4-0] = [6, 4]
# slope = dTP/dP = [0.5, 1]
# intercept = (TPa+(slope*Pa) = [(4 - 0.5*4), (0 - 1*0)] = [2, 0]
# (Pb/Pa) = (Pb/Pa) if Pb > 0 AND Pa > 0 else 1 = [10/4, 4/0] = [2.5, 1]
# auc * TotalPos = [(0.5 * (3 + 2 * log(2.5))), (1 * (4 + 0))]
# = [2.416, 4]
# auc = [2.416, 4]/(tp[1:]+fn[1:])
# expected_result = (2.416 / 7 + 4 / 7)
expected_result = 0.345 + 0.571
assert np.allclose(K.eval(result), expected_result, atol=1e-3)
def create_model(self, activation_function, alpha):
# create the model
self.activation_function = activation_function
self.model = Sequential()
self.alpha = alpha
hidden_nodes = int(
len(self.training_traces) / (alpha *
(len(self.labels) + self.num_target)))
self.hidden_nodes = hidden_nodes
self.model.add(LSTM(hidden_nodes))
self.model.add(Dense(1, activation=activation_function))
self.model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[
'accuracy',
metrics.AUC(),
metrics.Precision(),
metrics.Recall()
])
def optimiser_experiment_model(hp):
PRE_TRAINED_MODEL = Xception(input_shape=(IMG_HEIGHT, IMG_WIDTH, 3),
include_top=False,
pooling='avg',
weights='imagenet')
for layer in PRE_TRAINED_MODEL.layers:
layer.trainable = False
x = layers.Flatten()(PRE_TRAINED_MODEL.output)
x = layers.Dense(1, activation='sigmoid')(x)
MODEL = Model(PRE_TRAINED_MODEL.input, x)
MODEL.compile(optimizer=hp.Choice('optimiser',
values=['Adam', 'RMSprop', 'SGD']),
loss='binary_crossentropy',
metrics=[
metrics.BinaryAccuracy(name='acc'),
metrics.AUC(name='auc'),
metrics.FalsePositives(name='fp')
])
return MODEL
def train_lstm_model(x_train, y_train):
x_train = array(x_train)
x_train = x_train.reshape((len(x_train), 1, len(x_train[0])))
print("x_train.shape", x_train.shape)
print(x_train[0])
y_train = array(y_train)
print("y_train.shape", y_train.shape)
# imrpove log: use batch size 16 and add one more lstm layer
lstm_model = Sequential()
lstm_model.add(LSTM(16,
input_shape=(1, 63),
return_sequences=True))
lstm_model.add(LSTM(16, ))
lstm_model.add(layers.Dense(1, activation='sigmoid'))
lstm_model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['acc',
metrics.AUC(),
metrics.FalseNegatives(),
metrics.Recall(),
metrics.Precision(),
metrics.FalseNegatives(),
metrics.TrueNegatives(),
metrics.FalsePositives(),
metrics.TruePositives()])
lstm_history = lstm_model.fit(x_train, y_train,
epochs=100,
batch_size=16,
validation_split=0.2,
callbacks=[callbacks.EarlyStopping(monitor='val_loss', patience=5),
callbacks.LearningRateScheduler(scheduler)])
print("finish training lstm model")
return lstm_model, lstm_history
def loss_experiment_model(hp):
PRE_TRAINED_MODEL = Xception(input_shape=(IMG_HEIGHT, IMG_WIDTH, 3),
include_top=False,
pooling='avg',
weights='imagenet')
for layer in PRE_TRAINED_MODEL.layers:
layer.trainable = False
x = layers.Flatten()(PRE_TRAINED_MODEL.output)
x = layers.Dense(1, activation='sigmoid')(x)
MODEL = Model(PRE_TRAINED_MODEL.input, x)
chosen_loss = hp.Choice(
'loss', values=['SigmoidFocalCrossEntropy', 'BinaryCrossentropy'])
MODEL.compile(optimizer='adam',
loss=eval(chosen_loss)(),
metrics=[
metrics.BinaryAccuracy(name='acc'),
metrics.AUC(name='auc'),
metrics.FalsePositives(name='fp')
])
return MODEL
# Total params: 24,131,585
# Trainable params: 2,328,801 (2.098.176 from our Dense 1024, and 230.625 (1025 * 225) from our output layer)
# Non-trainable params: 21,802,784
"""
# Compile
# We use categorical_crossentropy since our model is trying to classify categorical result
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=[
"accuracy",
metrics.AUC(
num_thresholds=200,
curve="ROC",
summation_method="interpolation",
name=None,
dtype=None,
thresholds=None,
multi_label=False,
label_weights=None,
)
])
# Callbacks
my_callbacks = [
#tf.keras.callbacks.EarlyStopping(monitor="val_accuracy", patience=5),
tf.keras.callbacks.ModelCheckpoint(
filepath=outputs + "/model.{epoch:02d}-{val_accuracy:.3f}.h5"),
tf.keras.callbacks.TensorBoard(log_dir=logs, histogram_freq=1)
]
# Fit
for layer in PRE_TRAINED_MODEL.layers:
layer.trainable = False
x = layers.Flatten()(PRE_TRAINED_MODEL.output)
x = layers.Dense(1024, activation = 'relu')(x)
x = layers.Dropout(0.2)(x)
x = layers.Dense(1, activation = 'sigmoid')(x)
MODEL = Model(PRE_TRAINED_MODEL.input, x)
MODEL.compile(optimizer = RMSprop(lr = LEARNING_RATE),
loss = 'binary_crossentropy',
metrics = [
metrics.BinaryAccuracy(name = 'acc'),
metrics.AUC(name = 'auc'),
metrics.FalsePositives(name = 'fp')
])
EARLY_STOPPING = EarlyStopping(monitor='fp',
verbose=1,
patience=50,
mode='max',
restore_best_weights=True)
CLASS_WEIGHT = {0: 0.001694915254237288,
1: 0.00017733640716439085}
# CLASS_WEIGHT = {0: 9,
# 1: 1}
# CLASS_WEIGHT = {0: 1,
# 1: 1}
def test_invalid_summation_method(self):
with pytest.raises(Exception):
metrics.AUC(summation_method='Invalid')
def test_invalid_curve(self):
with pytest.raises(Exception):
metrics.AUC(curve='Invalid')
def test_invalid_num_thresholds(self):
with pytest.raises(Exception):
metrics.AUC(num_thresholds=-1)
with pytest.raises(Exception):
metrics.AUC(num_thresholds=1)
model.add(Dense(units_10, activation=activation_10))
model.add(Dropout(dropout_10))
model.add(Dense(units_11, activation=activation_11))
model.add(Dropout(dropout_11))
model.add(Dense(units_12, activation=activation_12))
model.add(Dropout(dropout_12))
model.add(Dense(nb_classes, activation='sigmoid'))
METRICS = [
metrics.BinaryAccuracy(name='ACCURACY'),
metrics.Precision(name='PRECISION'),
metrics.Recall(name='RECALL'),
metrics.AUC(name='AUC'),
metrics.TruePositives(name='TP'),
metrics.TrueNegatives(name='TN'),
metrics.FalsePositives(name='FP'),
metrics.FalseNegatives(name='FN')]
model.compile(loss='binary_crossentropy',
optimizer=compile_optimizer,
metrics=METRICS)
# GENERATORS
train_datagen = ImageDataGenerator(
rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
auxiliary_output = Dense(2, activation='sigmoid', name='aux_output')(gru_out) auxiliary_input = Input(shape=(13,), name='aux_input') e = Embedding(output_dim=512, input_dim=10000, input_length=13)(auxiliary_input) cnn_out = Conv1D(32,3)(e) cnn_out = Dropout(0.5)(cnn_out) cnn_out = Flatten()(cnn_out) x = concatenate([gru_out, cnn_out]) x = Dense(64, activation='relu')(x) x = Dense(64, activation='relu')(x) x = Dense(64, activation='relu')(x) main_output = Dense(2, activation='sigmoid', name='main_output')(x) model = Model(input=[main_input, auxiliary_input], output=[main_output, auxiliary_output]) model.compile(optimizer='rmsprop', loss='binary_crossentropy',loss_weights=[1., 0.2],metrics=[metrics.AUC()]) print(time_series_Mat.shape) es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10) model.fit([time_series_Mat,num_Norm_Train], [target_Train, target_Train], nb_epoch=200, batch_size=32, callbacks=[es]) print(time_series_Mat_Val.shape) print(num_Norm_Val.shape) print(type(time_series_Mat_Val)) print(type(num_Norm_Val)) predictions = model.predict([time_series_Mat_Val,num_Norm_Val]) from sklearn.metrics import classification_report predictions_bool = np.argmax(predictions[0], axis=1) import sklearn.metrics as metrics print(len(predictions)) predictions = np.asarray(predictions) y_pred = (predictions > 0.5)
def run(model_name, train_file, val_file, num_classes, filename, dropout, input_shape_arg = (224,224,3)):
"""
fit dataset and run training process
Arguments:\n
train_file --> h5 file, fit to model\n
val_file --> h5 file, for validation\n
num_classes --> int, total classes \n
dropout_value --> float, range 0 - 1 for dropout\n
epoch --> int\n
batch_size --> int, [8, 16, 32, 64, 128, 256, etc.]\n
input_shape_arg --> shape of image (W,H,C)\n
lr_value --> learning rate value\n
optimizer --> Adam, SGD\n
Returns:\n
model\n
x_test\n
y_test
"""
# preprocessing data
X_train, Y_train, X_val, Y_val = dataset_preprocess(num_classes, train_file, val_file)
_epoch = 80
lr_value_array = [1e-3, 1e-4]
if model_name == "resnet50":
batch_size_array = [8, 16, 32]
LABEL = ["e-3(8)", "e-3(16)", "e-3(32)", "e-4(8)", "e-4(16)", "e-4(32)"]
elif model_name == "resnet18":
batch_size_array = [16, 32, 64]
LABEL = ["e-3(16)", "e-3(32)", "e-3(64)", "e-4(16)", "e-4(32)", "e-4(64)"]
HP = []
for lr in lr_value_array:
for bs in batch_size_array:
hp = Hyperparameter("adam", lr, bs, dropout)
HP.append(hp)
HISTORY = []
ROC = []
for hp in HP:
K.clear_session()
model = None
# compile model
if model_name == "resnet50":
print("resnet50")
model = ResNet50(input_shape=input_shape_arg, classes=int(num_classes), dropout_value=hp.dropout)
model.compile(optimizer=hp.get_optimizer(), loss='categorical_crossentropy', metrics=['accuracy', metrics.AUC(), metrics.Precision(), metrics.Recall()])
elif model_name == "resnet18":
print("resnet18")
model = ResNet18(input_shape=input_shape_arg, classes=int(num_classes), dropout_value=hp.dropout)
model.compile(optimizer=hp.get_optimizer(), loss='categorical_crossentropy', metrics=['accuracy', metrics.AUC(), metrics.Precision(), metrics.Recall()])
elif model_name == "vgg19":
print("VGG19")
# configure model input
base_model = applications.vgg19.VGG19(weights= None, include_top=False, input_shape= input_shape_arg)
# configure model output
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dropout(hp.dropout(x))
out = Dense(int(num_classes), activation= 'softmax')(x)
# combine model then compile
model = Model(inputs = base_model.input, outputs = out)
model.compile(optimizer= hp.get_optimizer(), loss='categorical_crossentropy', metrics=['accuracy'])
elif model_name == "vgg16":
print("VGG16")
# configure model input
base_model = applications.vgg16.VGG16(weights= None, include_top=False, input_shape= input_shape_arg)
# configure model output
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dropout(hp.dropout(x))
out = Dense(int(num_classes), activation= 'softmax')(x)
# combine model then compile
model = Model(inputs = base_model.input, outputs = out)
model.compile(optimizer= hp.get_optimizer(), loss='categorical_crossentropy', metrics=['accuracy'])
# optimizer == adam
# train the model
history = model.fit(X_train, Y_train, epochs = _epoch, batch_size = hp.batch_size,
validation_data=(X_val, Y_val),
shuffle=True)
HISTORY.append(history)
del model
print(f"DONE for: {hp.optim}-{hp.lr_value}-{hp.batch_size}-{hp.dropout}")
plt.figure(1)
mpl.style.use('seaborn')
i = 0
for history in HISTORY:
plt.plot(history.history["val_accuracy"], f"C{i}", label=LABEL[i])
i = i+1
plt.ylabel('val_acc')
plt.xlabel('epoch')
plt.title(f"Accuracy {filename}")
plt.legend()
plt.savefig(f"ACC-{filename}.png")
print("VAL ACC:")
i = 0
for history in HISTORY:
print(LABEL[i])
i = i + 1
print("auc: {}" .format(statistics.mean( history.history["val_auc_1"] )) )
print("recall: {}" .format(statistics.mean( history.history["val_recall_1"] )) )
print("Prec: {}" .format(statistics.mean( history.history["val_precision_1"] )) )
print("MEAN: {}" .format(statistics.mean( history.history["val_accuracy"] )) )
print("STD: {}" .format(statistics.pstdev( history.history['val_accuracy'] )) )
def test_unweighted_all_correct(self):
self.setup()
auc_obj = metrics.AUC()
result = auc_obj(self.y_true, self.y_true)
assert K.eval(result) == 1