def evaluate_model(model, data_reader, test_dir, batch_size):
"""Test the model on test dataset attained from test_dir.
Args:
model: the trained keras model
data_reader: the TfRecords datareader used to load training and validation datasets
test_dir: string representing the directory containing the test TfRecords files
Returns:
eval_dict: dictionary containing important evaulation metrics
"""
#Create evaluation metrics
aucroc_calculator = metrics.AUC(multi_label=True)
aucpr_calculator = metrics.AUC(multi_label=True, curve='PR')
pr_calculator = metrics.PrecisionAtRecall(0.7)
rp_calculator = metrics.RecallAtPrecision(0.7)
#Prepare data
batch_num = 0
test_dataset = data_reader.get_dataset(test_dir,
batch_size=batch_size,
type="validate")
test_dataset = tfds.as_numpy(test_dataset)
for batch in test_dataset:
test_input = tf.convert_to_tensor(batch[0])
test_labels = tf.convert_to_tensor(batch[1])
predictions = model.predict(test_input)
loss_val = loss.custom_crossentropy(test_labels, predictions)
#Update Metrics
aucroc_calculator.update_state(test_labels, predictions)
aucpr_calculator.update_state(test_labels, predictions)
pr_calculator.update_state(test_labels, predictions)
rp_calculator.update_state(test_labels, predictions)
print(f"Batch Number {batch_num} with loss {loss_val}.")
batch_num += 1
#Get results
auc_roc = aucroc_calculator.result()
auc_pr = aucpr_calculator.result()
precision = pr_calculator.result()
recall = rp_calculator.result()
eval_dict = {
"AUCPR": auc_pr,
"AUCROC": auc_roc,
"precision": precision,
"recall": recall
}
return eval_dict
def test_eval(model, test_df, y_test):
test_data = BertPreprocessing(
test_df[["sentence1", "sentence2"]].values.astype("str"),
y_test,
batch_size=config.batch_size,
shuffle=False,
)
y_pred = model.predict(test_data)
size = y_pred.shape[0]
y_test = y_test[:size, :]
accuracy = metrics.CategoricalAccuracy()
accuracy.update_state(y_test, y_pred)
precision = metrics.Precision()
precision.update_state(y_test, y_pred)
recall = metrics.Recall()
recall.update_state(y_test, y_pred)
f1 = tfa.metrics.F1Score(num_classes=3, average="macro")
f1.update_state(y_test, y_pred)
auc = metrics.AUC()
auc.update_state(y_test, y_pred)
print(f"""
Accuracy: {accuracy.result().numpy()}
Precision: {precision.result().numpy()}
Recall: {recall.result().numpy()}
F1 score: {f1.result().numpy()}
AUC: {auc.result().numpy()}
""")
def eval_use_model(model_name, path_model_file, test_file, class_num):
"""
evaluating model by using entire model (weights, architecture, optimizers, etc.)
Arguments:\n
model_name --> String, Resnet50/Resnet18/VGG16/VGG19
path_model_file --> String, path which store .hdf5 of model's weight\n
test_file --> String, path to which store .h5 file of test dataset
class_num --> Int, number of class/label\n
Returns:\n
none
"""
# Load model weights
new_model = Model()
new_model = load_model(path_model_file)
new_model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=[
metrics.AUC(),
metrics.CategoricalAccuracy(),
metrics.TruePositives(),
metrics.TrueNegatives(),
metrics.FalsePositives(),
metrics.FalseNegatives()
])
# retrieve X_test, Y_test
X_test, Y_test = retrieve_test_dataset(test_file, int(class_num))
for i in range(4):
hasil = new_model.evaluate(X_test, Y_test)
print(new_model.metrics_names)
print(hasil)
def define_model(self, length, vocab_size, num_outcome_classes):
'''
Defines and compiles a convolutional neural network model
Parameters
___________
length: int
Max length of the text strings
vocab_size: int
Vocabulary size of the text documents
'''
input = Input(shape=(length, ))
embed = Embedding(vocab_size, 200)(input)
gru = Bidirectional(GRU(128, return_sequences=True))(embed)
drop1 = SpatialDropout1D(0.2)(gru)
conv = Conv1D(filters=256, kernel_size=4, activation='relu')(drop1)
pool = MaxPooling1D(pool_size=2)(conv)
att = Attention()([gru, pool])
flat = Flatten()(att)
drop2 = Dropout(.2)(flat)
dense1 = Dense(64, activation='relu')(drop2)
output = Dense(num_outcome_classes, activation='softmax')(dense1)
model = Model(inputs=input, outputs=output)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['categorical_accuracy',
metrics.AUC()])
self.model = model
def define_model(self, length, vocab_size):
'''
Defines and compiles a convolutional neural network model
Parameters
___________
length: int
Max length of the text strings
vocab_size: int
Vocabulary size of the text documents
'''
model = Sequential()
model.add(Embedding(vocab_size, 125, input_length=length))
model.add(SpatialDropout1D(0.2))
model.add(GRU(64))
#model.add(Bidirectional(GRU(75)))
model.add(Dropout(0.25))
model.add(Dense(3, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy', metrics.AUC()])
#model = Model(inputs=inputs1, outputs=outputs)
#model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', metrics.AUC()])
self.model = model
def confusion_matrix_other_metric(self):
return [
metrics.Accuracy(name='acc'),
metrics.Precision(name='precision'),
metrics.Recall(name='recall'),
metrics.AUC(name='auc'),
]
def train_network(training_set, training_labels, save_path='network.keras'):
global model
print('----TRAINING----')
model = Sequential([
Flatten(input_shape=training_set[0].shape),
Dense(256, activation='relu'),
Dense(64, activation='relu'),
Dense(10, activation='softmax'),
])
model_metrics = [
metrics.CategoricalAccuracy(),
metrics.Recall(),
metrics.AUC(),
metrics.SensitivityAtSpecificity(.8),
metrics.SpecificityAtSensitivity(.8), f1_score, fbeta_score
]
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=model_metrics)
print(model.summary())
train_time = time.time_ns()
history = model.fit(training_set,
training_labels,
epochs=2,
batch_size=32,
validation_split=0)
print((time.time_ns() - train_time) / 1000000)
print(str(history.history['loss'])[1:-1].replace(',', ''))
print(str(history.history['categorical_accuracy'])[1:-1].replace(',', ''))
model.save(save_path)
def define_model(self, length, vocab_size, num_outcome_classes):
'''
Defines and compiles a convolutional neural network model
Parameters
___________
length: int
Max length of the text strings
vocab_size: int
Vocabulary size of the text documents
'''
model = Sequential()
model.add(Input(shape=(length, )))
model.add(Embedding(vocab_size, 200))
model.add(Bidirectional(GRU(128, return_sequences=True)))
model.add(SpatialDropout1D(0.2))
model.add(Conv1D(filters=64, kernel_size=4, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dropout(.2))
model.add(Dense(32, activation='relu'))
model.add(Dense(num_outcome_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['categorical_accuracy',
metrics.AUC()])
self.model = model
def get_metrics():
acc = 'accuracy'
auc = metrics.AUC(num_thresholds=200,
curve='ROC',
name='auc',
thresholds=None,
multi_label=False)
fp = metrics.FalsePositives(thresholds=[0.001, 0.01, 0.1, 1.0], name='FP')
tp = metrics.TruePositives(thresholds=[0.001, 0.01, 0.1, 1.0], name='TP')
return [acc] #, auc, fp, tp]
def train_din(mode=1):
hidden_unit = 64
batch_size = 32
learning_rate = 1
epochs = 50
with open(electronics.pkl_dataset, 'rb') as f:
train_set = np.array(pickle.load(f))
test_set = pickle.load(f)
cate_list = pickle.load(f)
user_count, item_count, cate_count, max_sl = pickle.load(f)
train_user, train_item, train_hist, train_sl, train_y = input_data(
train_set, max_sl)
test_user, test_item, test_hist, test_sl, test_y = input_data(
test_set, max_sl)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
log_dir = 'logs/' + current_time
tensorboard = callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1,
write_graph=True,
write_grads=False,
write_images=True,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None,
embeddings_data=None,
update_freq=500)
# model checkpoint
check_path = 'save/din_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
checkpoint = callbacks.ModelCheckpoint(check_path,
save_weights_only=True,
verbose=1,
period=1)
model = DIN(user_count, item_count, cate_count, cate_list, hidden_unit)
model.summary()
optimizer = optimizers.SGD(learning_rate=learning_rate, decay=0.1)
model.compile(loss=losses.binary_crossentropy,
optimizer=optimizer,
metrics=[metrics.AUC()])
model.fit(
[train_user, train_item, train_hist, train_sl],
train_y,
epochs=epochs,
batch_size=batch_size,
validation_split=0.1,
# callbacks=[tensorboard, checkpoint]
)
print('test AUC: %f' %
model.evaluate([test_user, test_item, test_hist, test_sl], test_y))
def build_simple_model(dataset='Fashion Mnist',
opt='sgd',
hidden=None,
funcs=None,
loss=None,
metrics_list=None):
model = models.Sequential()
if dataset == 'CIFAR-10':
model.add(layers.Flatten(input_shape=[32, 32, 3]))
elif ('Fashion Mnist'):
model.add(layers.Flatten(input_shape=[28, 28]))
for i in hidden.keys():
model.add(layers.Dense(hidden[i], activation=funcs[i].lower()))
model.add(layers.Dense(10, activation="softmax"))
loss_dict = {
'Categorical Crossentropy': 'categorical_crossentropy',
'Binary Crossentropy': 'binary_crossentropy',
'Categorical Hinge': 'categorical_hinge',
'Huber loss': 'huber_loss'
}
metrics_dict = {
'auc':
metrics.AUC(),
'recall':
metrics.Recall(),
'accuracy':
metrics.CategoricalAccuracy()
if loss.startswith('Categorical') else metrics.Accuracy(),
'precision':
metrics.Precision(),
'categorical Hinge':
metrics.CategoricalHinge(),
'squared Hinge':
metrics.SquaredHinge(),
'Kullback-Leibler divergence':
metrics.KLDivergence(),
'mean absolute error':
metrics.MeanAbsoluteError(),
'mean squared error':
metrics.MeanSquaredError()
}
if metrics_list is not None and len(metrics_list) > 0:
metrics_list = [metrics_dict.get(m, m) for m in metrics_list]
else:
metrics_list = ['accuracy']
loss_f = loss_dict.get(loss)
model.compile(loss=loss_f, optimizer=opt, metrics=metrics_list)
return model
def inference(tfrecords_path, weights_path, wts_root):
""" Inference function to reproduce original model scores. This
script can be run as a standalone using python inference.py.
For more information try: `python inference.py -h`
Parameters
----------
tfrecords_path: str
The path to directory containing preprocessed tfrecords.
weights_path: str
The path to the combined model weights. A copy of the
weights can be found here:
https://gin.g-node.org/shashankbansal56/nondefaced-detector-reproducibility/src/master/pretrained_weights/combined
wts_root: str
The path to the root directory of all the model weights.
A copy of the weights can be found here:
https://gin.g-node.org/shashankbansal56/nondefaced-detector-reproducibility/src/master/pretrained_weights
"""
model = CombinedClassifier(input_shape=(128, 128),
dropout=0.4,
wts_root=wts_root,
trainable=False)
model.load_weights(os.path.abspath(weights_path))
model.trainable = False
dataset_test = get_dataset(
file_pattern=os.path.join(tfrecords_path, "data-test_*"),
n_classes=2,
batch_size=16,
volume_shape=(128, 128, 128),
plane="combined",
mode="test",
)
METRICS = [
metrics.BinaryAccuracy(name="accuracy"),
metrics.Precision(name="precision"),
metrics.Recall(name="recall"),
metrics.AUC(name="auc"),
]
model.compile(
loss=tf.keras.losses.binary_crossentropy,
optimizer=Adam(learning_rate=1e-3),
metrics=METRICS,
)
model.evaluate(dataset_test)
def evaluate_model(model, dataset, precision_at_recall, recall_at_precision):
"""Evaluate the model and dataset.
Args:
model: A keras model with input (video_matrix, class_feature_list) and 1 output denoting class relevance
dataset: tf.dataset attained from a Dataset class from readers.py
precision_at_recall: the recall float value for which to calculate precision
recall_at_precision: the precision float value for which to calculate recall
"""
segment_num = 0
#Create evaluation metrics
aucroc_calculator = metrics.AUC()
aucpr_calculator = metrics.AUC(curve='PR')
pr_calculator = metrics.PrecisionAtRecall(precision_at_recall)
rp_calculator = metrics.RecallAtPrecision(recall_at_precision)
for input_data, label in dataset:
prediction = evaluate_example(model, input_data)
#Update Metrics
aucroc_calculator.update_state(label, prediction)
aucpr_calculator.update_state(label, prediction)
pr_calculator.update_state(label, prediction)
rp_calculator.update_state(label, prediction)
print(f"Processing segment number {segment_num}")
segment_num += 1
#Get results
auc_roc = aucroc_calculator.result()
auc_pr = aucpr_calculator.result()
precision = pr_calculator.result()
recall = rp_calculator.result()
eval_dict = {
"AUCPR": auc_pr,
"AUCROC": auc_roc,
"precision": precision,
"recall": recall
}
return eval_dict
def __get_metric(self, metric):
if metric == "auc":
return m.AUC()
elif metric == "accuracy":
return m.Accuracy()
elif metric == "binary_accuracy":
return m.BinaryAccuracy()
elif metric == "categorical_accuracy":
return m.CategoricalAccuracy()
elif metric == "binary_crossentropy":
return m.BinaryCrossentropy()
elif metric == "categorical_crossentropy":
return m.CategoricalCrossentropy()
elif metric == "sparse_categorical_crossentropy":
return m.SparseCategoricalCrossentropy()
elif metric == "kl_divergence":
return m.KLDivergence()
elif metric == "poisson":
return m.Poission()
elif metric == "mse":
return m.MeanSquaredError()
elif metric == "rmse":
return m.RootMeanSquaredError()
elif metric == "mae":
return m.MeanAbsoluteError()
elif metric == "mean_absolute_percentage_error":
return m.MeanAbsolutePercentageError()
elif metric == "mean_squared_logarithm_error":
return m.MeanSquaredLogarithmError()
elif metric == "cosine_similarity":
return m.CosineSimilarity()
elif metric == "log_cosh_error":
return m.LogCoshError()
elif metric == "precision":
return m.Precision()
elif metric == "recall":
return m.Recall()
elif metric == "true_positive":
return m.TruePositives()
elif metric == "true_negative":
return m.TrueNegatives()
elif metric == "false_positive":
return m.FalsePositives()
elif metric == "false_negative":
return m.FalseNegatives()
else:
raise Exception("specified metric not defined")
def __init__(self, n_features, n_classes):
print("##################### Init NN #####################")
self.N_FEATURES = n_features
self.N_CLASSES = n_classes
self.METRICS = [
'accuracy',
tkm.TruePositives(),
tkm.FalsePositives(name='fp'),
tkm.TrueNegatives(name='tn'),
tkm.FalseNegatives(name='fn'),
#tkm.BinaryAccuracy(name='accuracy'),
tkm.Precision(name='precision'),
tkm.Recall(name='recall'),
tkm.AUC(name='auc')
]
self.DATE = datetime.now().strftime("%d-%m_%H%M%S")
create_dir(self.DATE)
def main(input_dir, output_dir):
class_names = [path.basename(s) for s in glob(input_dir + "/*/")]
n_classes = len(class_names)
# image_gen = preprocessing.image.ImageDataGenerator(
# rescale=1.0 / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True
# )
image_gen = preprocessing.image.ImageDataGenerator(rescale=1.0 / 255)
train_gen = image_gen.flow_from_directory(
input_dir,
target_size=(224, 224),
batch_size=16,
class_mode="categorical", # "binary"
color_mode="rgb",
)
base_model = applications.mobilenet_v2.MobileNetV2(input_shape=(224, 224,
3),
include_top=False,
weights="imagenet",
pooling="max")
for layer in base_model.layers:
layer.trainable = False
y = base_model.output
y = layers.Dense(n_classes, activation="softmax")(y)
model = models.Model(inputs=base_model.inputs, outputs=y)
lr = 0.05
model.compile(
optimizer=optimizers.Adam(lr),
loss="categorical_crossentropy",
metrics=["accuracy", metrics.AUC()],
)
# print(model.summary())
model.fit(
train_gen,
validation_data=train_gen,
epochs=20,
callbacks=[
callbacks.ModelCheckpoint(output_dir, save_best_only=True),
callbacks.EarlyStopping(patience=2),
callbacks.LearningRateScheduler(
lambda epoch: lr * np.exp(-0.1 * (epoch - 1))),
callbacks.CSVLogger(path.join(output_dir, "metrics.csv")),
],
)
def load_simple_model(model_path='',
weights_path='',
opt='sgd',
loss=None,
metrics_list=None):
model = models.load_model(model_path)
model.load_weights(weights_path)
loss_dict = {
'Categorical Crossentropy': 'categorical_crossentropy',
'Binary Crossentropy': 'binary_crossentropy',
'Categorical Hinge': 'categorical_hinge',
'Huber loss': 'huber_loss'
}
metrics_dict = {
'auc':
metrics.AUC(),
'recall':
metrics.Recall(),
'accuracy':
metrics.CategoricalAccuracy()
if loss.startswith('Categorical') else metrics.Accuracy(),
'precision':
metrics.Precision(),
'categorical Hinge':
metrics.CategoricalHinge(),
'squared Hinge':
metrics.SquaredHinge(),
'Kullback-Leibler divergence':
metrics.KLDivergence(),
'mean absolute error':
metrics.MeanAbsoluteError(),
'mean squared error':
metrics.MeanSquaredError()
}
if metrics_list is not None and len(metrics_list) > 0:
metrics_list = [metrics_dict.get(m, m) for m in metrics_list]
else:
metrics_list = ['accuracy']
loss_f = loss_dict.get(loss)
model.compile(loss=loss_f, optimizer=opt, metrics=metrics_list)
return model
def get_model():
# define convolutional layer
# kernel size
filters = 4
kernel_size = 4
convolution_1d_layer = keras.layers.Conv1D(filters, kernel_size,
input_shape=(500, 1),
strides=1, padding='same',
activation="relu",
name="convolution_1d_layer")
# max pooling layer
max_pooling_layer = keras.layers.MaxPooling1D(pool_size=2, strides=2, padding="same", name="max_pooling_layer")
# reshape layer
reshape_layer = keras.layers.Flatten(name="reshape_layer")
# dropout layer
dropout_layer = keras.layers.Dropout(0.5, name="dropout_layer")
# full connect layer
full_connect_layer = keras.layers.Dense(128, activation="relu", name="full_connect_layer")
model = keras.Sequential()
model.add(convolution_1d_layer)
model.add(max_pooling_layer)
model.add(reshape_layer)
model.add(dropout_layer)
model.add(full_connect_layer)
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(
optimizer=optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08),
loss='binary_crossentropy',
metrics=['accuracy',metrics.AUC()]
)
return model
def experiment(self, under=False, ratio=3, plot=False):
METRICS = [
metrics.TruePositives(name='tp'),
metrics.FalsePositives(name='fp'),
metrics.TrueNegatives(name='tn'),
metrics.FalseNegatives(name='fn'),
metrics.BinaryAccuracy(name='accuracy'),
metrics.Precision(name='precision'),
metrics.Recall(name='recall'),
metrics.AUC(name='auc')
]
data = DataLoader()
model = LeNet(data.X, METRICS)
augmenter = Augmenter(data.X, data.Y)
if under:
data.X, data.Y = augmenter.undersample(ratio=ratio)
if self.augmentation.type == 1 or self.augmentation.type == 2:
data.X, data.Y = augmenter.duplicate(noise=self.augmentation.noise,
sigma=self.augmentation.sigma)
elif self.augmentation.type == 3:
data.X, data.Y = augmenter.SMOTE()
#data.normalize()
#print(len(data.X))
#print(len(data.valX))
data.summarize(test=False)
his = model.fit(data.X, data.Y, data.valX, data.valY)
RES, fpr, tpr = model.predict(data.testX, data.testY)
#self.model_summary(RES)
if plot:
self.plot(his)
self.ROC(fpr, tpr)
return RES
def train(args):
input_shape = (args.img_size, args.img_size, 3)
train_gen, valid_gen = get_generators(
args.data_path,
args.img_size,
args.batch_size,
args.test_img
)
model = get_model(input_shape, args.n_classes)
model.compile(
RMSprop(lr=args.learning_rate),
loss="categorical_crossentropy",
metrics=[metrics.AUC(name='auc'), 'accuracy']
)
es_callback = EarlyStopping(
monitor='val_auc',
mode='max',
patience=5,
verbose=1,
min_delta=0.005,
restore_best_weights=True
)
history = model.fit(
train_gen,
steps_per_epoch=train_gen.samples//args.batch_size,
epochs = args.num_epochs,
validation_data=valid_gen,
validation_steps=valid_gen.samples//args.batch_size,
callbacks= [es_callback],
verbose=1
)
model.save_weights(f"{args.weigths_path}/fish_classification")
def compile(self, model, train_generator, valid_generator):
""":arg
This function contain model compile and model fit process, input a model and output history and trained model
"""
start_time = time()
print("*" * 40, "Start {} Processing".format(model._name), "*" * 40)
# we use a lot of metric to evalute our binary classification result
METRICS = [
metrics.TruePositives(name='tp'),
metrics.FalsePositives(name='fp'),
metrics.TrueNegatives(name='tn'),
metrics.FalseNegatives(name='fn'),
metrics.BinaryAccuracy(name='binary_accuracy'),
#metrics.CategoricalAccuracy(name='accuracy'),
metrics.Precision(name='precision'),
metrics.Recall(name='recall'),
metrics.AUC(name='auc'),
# F1Score(num_classes = int(y_train.shape[1]), name='F1')
]
# define a optimizer
opt_rms = optimizers.RMSprop(lr = 1e-4, decay = 1e-5)
# define compile parameters
model.compile(loss = 'binary_crossentropy', optimizer = opt_rms, metrics = ['accuracy'])
# start to fit
history = model.fit(
train_generator,
steps_per_epoch=20,
epochs=5,
validation_data=valid_generator,
validation_steps=20
)
return history
def cross_validate(self, train_data_list, **build_kwargs):
"""
Performs cross-validation on the model.
Parameters
----------
train_data_list: list
A list where each element contains multiple images.
(Each element corresponds to a fold)
Returns
-------
Model
A trained model
"""
print(f"Cross validating {str(self)}")
# Create lists for holding the metric values
acc_per_fold = [] # accuracy
auc_per_fold = [] # AUC
loss_per_fold = [] # loss
# Create 2 empty lists for holding the training history losses and test history losses
fold_train_loss, fold_test_loss = list(), list()
# Iterate an index to size of folds
for i in range(len(train_data_list)):
# Extract validation data
x_val, y_val = train_data_list[i]
# Indices for train
indices_to_keep = np.delete(range(len(train_data_list)), i)
# Get train data
x_train, y_train = _build_train_data(train_data_list,
indices_to_keep)
# Compute class weights for balanced learning. Returns a list
weights = compute_class_weight("balanced",
classes=np.unique(y_train),
y=y_train)
# Convert the list do dict.
weights_dict = {idx: value for idx, value in enumerate(weights)}
# Build the model
baseline_model = self.build(
input_shape=x_train[0].shape,
**build_kwargs,
)
# Compile the model
baseline_model.compile(
optimizer=optimizers.Adam(),
loss=losses.BinaryCrossentropy(),
metrics=[metrics.BinaryAccuracy(),
metrics.AUC(name="auc")],
)
# Train the model with the training indices
history = baseline_model.fit(
x_train,
y_train,
validation_data=(x_val, y_val),
batch_size=64,
epochs=40,
class_weight=weights_dict,
verbose=0,
)
fold_train_loss.append(history.history["loss"])
fold_test_loss.append(history.history["val_loss"])
# Evaluate the model with the testing indices
scores = baseline_model.evaluate(x_val, y_val, verbose=0)
# Scores has format : ['loss', 'binary_accuracy', 'auc']
# Save the loss value on the val data
loss_per_fold.append(scores[0])
# Save the Accuracy value on the val data
acc_per_fold.append(scores[1])
# Save the Accuracy value on the val data
auc_per_fold.append(scores[2])
# Plot the loss curve
self._plot_losses(
fold_train_loss,
fold_test_loss,
)
# Display the results
print(
f"Accuracy: {acc_per_fold.mean():.2f} (+/- {acc_per_fold.std():.2f})"
)
print(f"AUC: {auc_per_fold.mean():.2f} (+/- {auc_per_fold.std():.2f})")
return baseline_model
def define_model(self, length, vocab_size, num_outcome_classes):
'''
Defines and compiles a convolutional neural network model
Parameters
___________
length: int
Max length of the text strings
vocab_size: int
Vocabulary size of the text documents
'''
model = Sequential()
model.add(Input(shape=(length,)))
model.add(Embedding(vocab_size, 250))
model.add(Dropout(.25))
model.add(Bidirectional(GRU(64)))
model.add(Dropout(.5))
model.add(Dense(32, activation='relu'))
model.add(Dense(num_outcome_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy', metrics.AUC()])
self.model = model
def train_model(self, themes_weight: List[float],
dataset: TrainValidationDataset, voc_size: int,
keras_callback: LambdaCallback):
article_length = dataset.article_length
theme_count = dataset.theme_count
model = tf.keras.Sequential([
# 1
# keras.layers.Embedding(input_dim=voc_size, output_dim=firstLayoutOutputDim),
# keras.layers.Dropout(0.2),
# keras.layers.Conv1D(200,3,input_shape=(ARTICLE_MAX_WORD_COUNT,firstLayoutOutputDim), activation=tf.nn.relu),
# keras.layers.GlobalAveragePooling1D(),
# keras.layers.Dense(250, activation=tf.nn.relu),
# keras.layers.Dense(theme_count, activation=tf.nn.softmax)
# 2
# keras.layers.Embedding(input_dim=voc_size, output_dim=firstLayoutOutputDim),
# keras.layers.LSTM(ltsmOutputDim, dropout=0.2, recurrent_dropout=0.2, activation='tanh'),
# keras.layers.Dense(theme_count, activation=tf.nn.softmax)
# 3
# keras.layers.Embedding(input_dim=self.voc_size, output_dim=embedding_output_dim),
# keras.layers.Bidirectional(keras.layers.LSTM(intermediate_dim, return_sequences=True)),
# # keras.layers.Dropout(0.1),
# keras.layers.Bidirectional(keras.layers.LSTM(last_dim, dropout=0.05, recurrent_dropout=0.05)),
# keras.layers.Dense(last_dim, activation=tf.nn.relu),
# keras.layers.Dense(self.theme_count, activation=tf.nn.softmax)
# 4
# keras.layers.Embedding(input_dim=self.voc_size, input_length=self.article_length, output_dim=embedding_output_dim),
# keras.layers.Bidirectional(keras.layers.LSTM(intermediate_dim, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)),
# keras.layers.Dropout(0.2),
# keras.layers.Bidirectional(keras.layers.LSTM(last_dim * 2, recurrent_dropout=0.2)), #was last_dim * 2
# keras.layers.Dense(last_dim, activation=tf.nn.relu),
# keras.layers.Dense(self.theme_count, activation=tf.nn.sigmoid)
# 5
#keras.layers.Embedding(input_dim=self.voc_size, input_length=self.article_length, output_dim=embedding_output_dim),
# keras.layers.Conv1D(filters=64, kernel_size=5, input_shape=(self.voc_size, embedding_output_dim), activation="relu"),
# keras.layers.MaxPool1D(4),
#keras.layers.Bidirectional(keras.layers.LSTM(intermediate_dim, recurrent_dropout=0.1)),
#keras.layers.Dense(last_dim, activation=tf.nn.relu),
#keras.layers.Dense(self.theme_count, activation=tf.nn.sigmoid)
#6
keras.layers.Embedding(input_dim=voc_size,
input_length=article_length,
output_dim=128,
mask_zero=True),
keras.layers.Bidirectional(
keras.layers.LSTM(128, recurrent_dropout=0.2, dropout=0.2)),
#keras.layers.Dropout(0.2),
#keras.layers.Dense(last_dim, activation=tf.nn.relu),
# keras.layers.Dense(self.theme_count, activation=tf.nn.sigmoid, use_bias=True,bias_initializer=tf.keras.initializers.Constant(-1.22818328))
keras.layers.Dense(theme_count,
activation=tf.nn.sigmoid,
kernel_regularizer=regularizers.l2(0.1),
activity_regularizer=regularizers.l1(0.05))
# 7
# keras.layers.Embedding(input_dim=self.voc_size, input_length=self.article_length,
# output_dim=embedding_output_dim),
# keras.layers.GlobalAvgPool1D(),
# keras.layers.Dense(last_dim, activation=tf.nn.relu),
# keras.layers.Dense(self.theme_count, activation=tf.nn.sigmoid)
])
model.summary()
model.compile(
optimizer=tf.keras.optimizers.Adam(clipnorm=1, clipvalue=0.5),
#loss=WeightedBinaryCrossEntropy(themes_weight, from_logits=True),
loss=keras.losses.BinaryCrossentropy(from_logits=True),
metrics=[
metrics.AUC(),
metrics.BinaryAccuracy(),
metrics.TruePositives(),
metrics.TrueNegatives(),
metrics.FalseNegatives(),
metrics.FalsePositives(),
metrics.Recall(),
metrics.Precision()
],
run_eagerly=self.run_eagerly)
keras.utils.plot_model(model, 'Model1.png', show_shapes=True)
cb_list = [ManualInterrupter, keras_callback]
model.fit(dataset.trainData,
epochs=10,
steps_per_epoch=dataset.train_batch_count,
validation_data=dataset.validationData,
validation_steps=dataset.validation_batch_count,
callbacks=cb_list,
class_weight={
0: 1,
1: themes_weight[0]
})
model.save("output/" + self.get_model_name() + ".h5")
model.save_weights("output/" + self.get_model_name() + "_weight.h5")
self.__model__ = model
def call(self, local_true, local_pred):
local_true = tf.convert_to_tensor(local_true, dtype=tf.float32)
local_pred = tf.convert_to_tensor(local_pred, dtype=tf.float32)
local_auc_roc = tf.math.add(tf.math.add(1., metrics.AUC(local_true, local_pred)))
return local_auc_roc
}
if __name__ == "__main__":
if len(argv) != 2 or argv[1] not in networks.keys():
options = "|".join(networks.keys())
print(f"usage: ./predict_review.py {options}")
else:
name = argv[1]
(X_train, Y_train, X_dev, Y_dev, X_test, Y_test) = prepare(
filename="./data/02_filter.json",
max_ingredients=int(networks[name]["max_ingredients"]),
)
model = Sequential(networks[name]["layers"])
model.compile(
loss=losses.binary_crossentropy,
metrics=[metrics.AUC()],
optimizer=optimizers.Adam(),
)
run_time = datetime.now().strftime("%Y%m%d-%H%M%S")
training = model.fit(
x=X_train,
y=Y_train,
batch_size=BATCH_SIZE,
callbacks=[
callbacks.TensorBoard(
log_dir=LOG_DIR.format(network=name, run=run_time),
histogram_freq=1,
)
],
epochs=5,
validation_data=(X_dev, Y_dev),
IMGSIZE = (int(args.imgsize[0]), int(args.imgsize[1]))
LOGDIR = args.logdir
DATA = args.data
BACKBONE = args.backbone
NAME = args.model
# --- define model metrics ---
METRICS = [
metrics.TruePositives(name="True_Positives"),
metrics.FalsePositives(name="False_Positives"),
metrics.TrueNegatives(name="True_Negatives"),
metrics.FalseNegatives(name="False_Negatives"),
metrics.BinaryAccuracy(name="Binary_Accuracy"),
metrics.Precision(name="Precision"),
metrics.Recall(name="Recall"),
metrics.AUC(name="AUC")
]
# --- tensorflow calbacks ---
date = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
if platform.system().lower() == "windows":
LOGDIR = LOGDIR + "\\" + NAME + "\\" + date
else:
LOGDIR = LOGDIR + "/" + NAME + "/" + date
if not os.path.isdir(LOGDIR):
os.makedirs(LOGDIR, exist_ok=True)
tensorboard = callbacks.TensorBoard(log_dir=LOGDIR,
histogram_freq=1,
profile_batch=1)
# Encode labels
onehot = mlu.EncoderHelper()
onehot.fit(y_train)
y_train = onehot.transform(y_train)
y_test = onehot.transform(y_test)
# Create model
model = create_model()
model.compile(
optimizer="adam",
loss="categorical_crossentropy",
metrics=[
met.CategoricalAccuracy(name="categorical_accuracy"),
met.Recall(name="recall"),
met.Precision(name="precision"),
met.AUC(name="auc"),
tfa_met.F1Score(num_classes=num_classes,
average="macro",
name="f1_macro"),
tfa_met.MatthewsCorrelationCoefficient(num_classes=num_classes,
name="mcc"),
],
)
# Fit data
# tfu.kill_tensorboard_pids()
proc = tfu.start_tensorboard(tfu.get_tb_logdir("gpu_test", False))
model.fit(
x=x_train,
y=y_train,
epochs=5,
X_train, X_test, y_train, y_test = train_test_split(trainLines, transformed_labels, test_size=.2, random_state=42, stratify=transformed_labels) length = encoder.max_length(X_train) vocab_size = encoder.vocab_size(X_train) X_train = encoder.encode_text(X_train) X_test = encoder.encode_text(X_test, test_data=True) inputs1 = Input(shape=(length,)) embedding1 = Embedding(vocab_size, 100)(inputs1) conv1 = Conv1D(filters=32, kernel_size=4, activation='relu')(embedding1) drop1 = Dropout(0.5)(conv1) pool1 = MaxPooling1D(pool_size=2)(drop1) flat1 = Flatten()(pool1) dense1 = Dense(64, activation='relu')(flat1) outputs = Dense(3, activation='softmax')(dense1) model = Model(inputs=inputs1, outputs=outputs) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', metrics.AUC()]) model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10, batch_size=128) test_text = "I have been an every dollar user for the past 18 months and occasionally have issues loading my USAA accounts. I followed your troubleshooting page and have deleted my bank account and attempted to reload it but hasn't loaded. Any help would be appreciated." test_text = ct.prepare_text(test_text) test_text = encoder.encode_text(test_text, test_data=True) model.predict(test_text)
dropout=0.3)
# Build the model and expose input and output sockets of graphsage model
# for link prediction
x_inp, x_out = graphsage.in_out_tensors()
prediction = link_classification(output_dim=1,
output_act="relu",
edge_embedding_method="ip")(x_out)
model = keras.Model(inputs=x_inp, outputs=prediction)
model.compile(
optimizer=keras.optimizers.Adam(lr=1e-3),
loss=keras.losses.binary_crossentropy,
metrics=[tfm.AUC(), "acc", utils.get_f1, utils.get_pres, utils.get_rec])
init_train_metrics = model.evaluate(train_flow)
init_test_metrics = model.evaluate(test_flow)
print("\nTrain Set Metrics of the initial (untrained) model:")
for name, val in zip(model.metrics_names, init_train_metrics):
print("\t{}: {:0.4f}".format(name, val))
print("\nTest Set Metrics of the initial (untrained) model:")
for name, val in zip(model.metrics_names, init_test_metrics):
print("\t{}: {:0.4f}".format(name, val))
epochs = 20
history = model.fit(train_flow,
