def evaluate(model, evalX, evalY, labelNames):
stats = {0: 0, 1: 0, 2: 0, 3: 0}
top5 = []
for i, image in enumerate(evalX):
if i % 10 == 0:
print("[INFO] Evaluated using {} images".format(i))
image = image.astype("float32") / 255.0
image = np.expand_dims(image, axis=0)
preds = model.predict(image)
top = np.argsort(-preds, axis=1)
if evalY[i] == top[0][0]:
stats[0] += 1
elif evalY[i] == top[0][1]:
stats[1] += 1
elif evalY[i] == top[0][2]:
stats[2] += 1
else:
stats[3] += 1
top5.append(
[top[0][0], top[0][1], top[0][2], top[0][3], top[0][4], evalY[i]])
numLabels = len(np.unique(evalY))
evalY = to_categorical(evalY, numLabels)
evalX = np.array(evalX, dtype=np.float32) / 255.0
predictions = model.predict(evalX)
report = classification_report(evalY.argmax(axis=1),
predictions.argmax(axis=1),
target_names=labelNames)
confusion = math.confusion_matrix(evalY.argmax(axis=1),
predictions.argmax(axis=1))
return stats, top5, report, confusion
def model_performance(model, x_test, y_test):
test_score = model.evaluate(x_test, y_test)
y_pred_prob = model.predict(x_test)
y_pred = np.argmax(y_pred_prob, axis=1)
y_test = np.argmax(y_test, axis=1)
confusion_mtx = confusion_matrix(y_test, y_pred)
return test_score, y_pred, confusion_mtx
def matriz_confusion(y_test, y_predecida):
# le hacemos el argmax para desaher el to categorical
matriz_conf = confusion_matrix(labels=argmax(y_test, axis=1),
predictions=argmax(y_predecida,
axis=1)).numpy()
matriz_normalizada = np.around(matriz_conf.astype('float') /
matriz_conf.sum(axis=1)[:, np.newaxis],
decimals=2)
clases = [i for i in range(NUM_CLASES)]
dataframe_matriz = pd.DataFrame(matriz_normalizada,
index=clases,
columns=clases)
figure = plt.figure(figsize=(8, 8))
sns.heatmap(dataframe_matriz, annot=True, cmap=plt.cm.Blues)
plt.tight_layout()
plt.ylabel('Valor real')
plt.xlabel('Valor predecido')
plt.show()
model.fit(fashion_mnist_train_images, fashion_mnist_train_marks, epochs=10)
test_loss, test_acc = model.evaluate(fashion_mnist_test_images,
fashion_mnist_test_marks,
verbose=1)
if test_acc > best_test_acc:
best_test_acc = test_acc
best_number = number
number += 1
print("best test accuracy = " + str(best_test_acc))
predict = models[best_number].predict(fashion_mnist_test_images)
p1 = []
for i in range(len(predict)):
p1.append(argmax(predict[i]))
print(math.confusion_matrix(fashion_mnist_test_marks, p1))
classes = {}
for i in range(len(fashion_mnist_test_marks)):
if not classes.__contains__(fashion_mnist_test_marks[i]):
classes[fashion_mnist_test_marks[i]] = []
classes[fashion_mnist_test_marks[i]].append(i)
# print(classes)
i = 0
for k in range(10):
for j in range(10):
i += 1
m = 0
num = 0
for index in classes[k]:
if predict[index][j] > m:
def launch(self) -> int:
"""Execute the :class:`ClassificationNeuralNetwork <neural_networks.classification_neural_network.ClassificationNeuralNetwork>` neural_networks.classification_neural_network.ClassificationNeuralNetwork object."""
# check input/output paths and parameters
self.check_data_params(self.out_log, self.err_log)
# Setup Biobb
if self.check_restart(): return 0
self.stage_files()
# load dataset
fu.log(
'Getting dataset from %s' %
self.io_dict["in"]["input_dataset_path"], self.out_log,
self.global_log)
if 'columns' in self.features:
labels = getHeader(self.io_dict["in"]["input_dataset_path"])
skiprows = 1
else:
labels = None
skiprows = None
data = pd.read_csv(self.io_dict["in"]["input_dataset_path"],
header=None,
sep="\s+|;|:|,|\t",
engine="python",
skiprows=skiprows,
names=labels)
targets_list = data[getTargetValue(self.target)].to_numpy()
X = getFeatures(self.features, data, self.out_log,
self.__class__.__name__)
fu.log('Features: [%s]' % (getIndependentVarsList(self.features)),
self.out_log, self.global_log)
# target
y = getTarget(self.target, data, self.out_log, self.__class__.__name__)
fu.log('Target: %s' % (str(getTargetValue(self.target))), self.out_log,
self.global_log)
# weights
if self.weight:
w = getWeight(self.weight, data, self.out_log,
self.__class__.__name__)
# shuffle dataset
fu.log('Shuffling dataset', self.out_log, self.global_log)
shuffled_indices = np.arange(X.shape[0])
np.random.shuffle(shuffled_indices)
np_X = X.to_numpy()
shuffled_X = np_X[shuffled_indices]
shuffled_y = targets_list[shuffled_indices]
if self.weight: shuffled_w = w[shuffled_indices]
# train / test split
fu.log('Creating train and test sets', self.out_log, self.global_log)
arrays_sets = (shuffled_X, shuffled_y)
# if user provide weights
if self.weight:
arrays_sets = arrays_sets + (shuffled_w, )
X_train, X_test, y_train, y_test, w_train, w_test = train_test_split(
*arrays_sets,
test_size=self.test_size,
random_state=self.random_state)
else:
X_train, X_test, y_train, y_test = train_test_split(
*arrays_sets,
test_size=self.test_size,
random_state=self.random_state)
# scale dataset
if self.scale:
fu.log('Scaling dataset', self.out_log, self.global_log)
X_train = scale(X_train)
# build model
fu.log('Building model', self.out_log, self.global_log)
model = self.build_model((X_train.shape[1], ), np.unique(y_train).size)
# model summary
stringlist = []
model.summary(print_fn=lambda x: stringlist.append(x))
model_summary = "\n".join(stringlist)
fu.log('Model summary:\n\n%s\n' % model_summary, self.out_log,
self.global_log)
# get optimizer
mod = __import__('tensorflow.keras.optimizers',
fromlist=[self.optimizer])
opt_class = getattr(mod, self.optimizer)
opt = opt_class(lr=self.learning_rate)
# compile model
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'mse'])
# fitting
fu.log('Training model', self.out_log, self.global_log)
# set an early stopping mechanism
# set patience=2, to be a bit tolerant against random validation loss increases
early_stopping = EarlyStopping(patience=2)
if self.weight:
sample_weight = w_train
class_weight = []
else:
# TODO: class_weight not working since TF 2.4.1 update
#fu.log('No weight provided, class_weight will be estimated from the target data', self.out_log, self.global_log)
fu.log('No weight provided', self.out_log, self.global_log)
sample_weight = None
class_weight = [
] #compute_class_weight('balanced', np.unique(y_train), y_train)
print(class_weight)
# fit the model
mf = model.fit(X_train,
y_train,
class_weight=class_weight,
sample_weight=sample_weight,
batch_size=self.batch_size,
epochs=self.max_epochs,
callbacks=[early_stopping],
validation_split=self.validation_size,
verbose=1)
fu.log('Total epochs performed: %s' % len(mf.history['loss']),
self.out_log, self.global_log)
train_metrics = pd.DataFrame()
train_metrics['metric'] = [
'Train loss', ' Train accuracy', 'Train MSE', 'Validation loss',
'Validation accuracy', 'Validation MSE'
]
train_metrics['coefficient'] = [
mf.history['loss'][-1], mf.history['accuracy'][-1],
mf.history['mse'][-1], mf.history['val_loss'][-1],
mf.history['val_accuracy'][-1], mf.history['val_mse'][-1]
]
fu.log(
'Training metrics\n\nTRAINING METRICS TABLE\n\n%s\n' %
train_metrics, self.out_log, self.global_log)
# confusion matrix
train_predictions = model.predict(X_train)
train_predictions = np.around(train_predictions, decimals=2)
norm_pred = []
[
norm_pred.append(np.argmax(pred, axis=0))
for pred in train_predictions
]
cnf_matrix_train = math.confusion_matrix(y_train, norm_pred).numpy()
np.set_printoptions(precision=2)
if self.normalize_cm:
cnf_matrix_train = cnf_matrix_train.astype(
'float') / cnf_matrix_train.sum(axis=1)[:, np.newaxis]
cm_type = 'NORMALIZED CONFUSION MATRIX'
else:
cm_type = 'CONFUSION MATRIX, WITHOUT NORMALIZATION'
fu.log(
'Calculating confusion matrix for training dataset\n\n%s\n\n%s\n' %
(cm_type, cnf_matrix_train), self.out_log, self.global_log)
# testing
if self.scale:
X_test = scale(X_test)
fu.log('Testing model', self.out_log, self.global_log)
test_loss, test_accuracy, test_mse = model.evaluate(X_test, y_test)
test_metrics = pd.DataFrame()
test_metrics['metric'] = ['Test loss', ' Test accuracy', 'Test MSE']
test_metrics['coefficient'] = [test_loss, test_accuracy, test_mse]
fu.log(
'Testing metrics\n\nTESTING METRICS TABLE\n\n%s\n' % test_metrics,
self.out_log, self.global_log)
# predict data from X_test
test_predictions = model.predict(X_test)
test_predictions = np.around(test_predictions, decimals=2)
tpr = tuple(map(tuple, test_predictions))
test_table = pd.DataFrame()
test_table['P' + np.array2string(np.unique(y_train))] = tpr
test_table['target'] = y_test
fu.log('TEST DATA\n\n%s\n' % test_table, self.out_log, self.global_log)
# confusion matrix
norm_pred = []
[
norm_pred.append(np.argmax(pred, axis=0))
for pred in test_predictions
]
cnf_matrix_test = math.confusion_matrix(y_test, norm_pred).numpy()
np.set_printoptions(precision=2)
if self.normalize_cm:
cnf_matrix_test = cnf_matrix_test.astype(
'float') / cnf_matrix_test.sum(axis=1)[:, np.newaxis]
cm_type = 'NORMALIZED CONFUSION MATRIX'
else:
cm_type = 'CONFUSION MATRIX, WITHOUT NORMALIZATION'
fu.log(
'Calculating confusion matrix for testing dataset\n\n%s\n\n%s\n' %
(cm_type, cnf_matrix_test), self.out_log, self.global_log)
# save test data
if (self.io_dict["out"]["output_test_table_path"]):
fu.log(
'Saving testing data to %s' %
self.io_dict["out"]["output_test_table_path"], self.out_log,
self.global_log)
test_table.to_csv(self.io_dict["out"]["output_test_table_path"],
index=False,
header=True)
# create test plot
if (self.io_dict["out"]["output_plot_path"]):
vs = np.unique(targets_list)
vs.sort()
if len(vs) > 2:
plot = plotResultsClassMultCM(mf.history, cnf_matrix_train,
cnf_matrix_test,
self.normalize_cm, vs)
fu.log(
'Saving confusion matrix plot to %s' %
self.io_dict["out"]["output_plot_path"], self.out_log,
self.global_log)
else:
plot = plotResultsClassBinCM(mf.history, train_predictions,
test_predictions, y_train, y_test,
cnf_matrix_train, cnf_matrix_test,
self.normalize_cm, vs)
fu.log(
'Saving binary classifier evaluator plot to %s' %
self.io_dict["out"]["output_plot_path"], self.out_log,
self.global_log)
plot.savefig(self.io_dict["out"]["output_plot_path"], dpi=150)
# save model and parameters
vs = np.unique(targets_list)
vs.sort()
vars_obj = {
'features': self.features,
'target': self.target,
'scale': self.scale,
'vs': vs.tolist(),
'type': 'classification'
}
variables = json.dumps(vars_obj)
fu.log('Saving model to %s' % self.io_dict["out"]["output_model_path"],
self.out_log, self.global_log)
with h5py.File(self.io_dict["out"]["output_model_path"],
mode='w') as f:
hdf5_format.save_model_to_hdf5(model, f)
f.attrs['variables'] = variables
return 0
def visualize_detections(images,
ground_truths,
detections,
id2code,
label_codes,
label_names,
out_dir='/tmp'):
"""Create visualizations.
Consist of the original image, the confusion matrix, ground truth labels
and the model predicted labels.
:param images: original images
:param ground_truths: ground truth labels
:param detections: the model label predictions
:param id2code: dictionary mapping label ids to their codes
:param label_codes: list with label codes
:param label_names: list with label names
:param out_dir: directory where the output visualizations will be saved
"""
max_id = max(id2code.values())
name_range = range(len(label_names))
for i in range(0, np.shape(detections)[0]):
fig = plt.figure(figsize=(17, 17))
# original image
ax1 = fig.add_subplot(2, 2, 1)
# TODO: expect also other data than S2
a = np.stack(
(images[i][:, :, 3], images[i][:, :, 2], images[i][:, :, 1]),
axis=2)
ax1.imshow((255 / a.max() * a).astype(np.uint8))
ax1.title.set_text('Actual image')
# ground truths
ax3 = fig.add_subplot(2, 2, 3)
ax3.set_title('Ground truth labels')
gt_labels = ground_truths[i]
gt_labels = onehot_decode(gt_labels, id2code)
ax3.imshow(gt_labels * 4)
# detections
ax4 = fig.add_subplot(2, 2, 4)
ax4.set_title('Predicted labels')
pred_labels = onehot_decode(detections[i], id2code)
ax4.imshow(pred_labels * 4)
# confusion matrix
ax2 = fig.add_subplot(2, 2, 2)
ax2.set_title('Confusion matrix')
conf_matrix = confusion_matrix(gt_labels[:, :, 0].flatten(),
pred_labels[:, :,
0].flatten(), max_id + 1)
# subset to existing classes
conf_matrix = conf_matrix.numpy()[label_codes][:, label_codes]
# normalize the confusion matrix
row_sums = conf_matrix.sum(axis=1)[:, np.newaxis]
# TODO: solve division by 0
cm_norm = np.around(conf_matrix.astype('float') / row_sums, decimals=2)
# visualize
ax2.imshow(cm_norm, cmap=plt.cm.Blues)
y_labels = [
'{}\n{}'.format(label_names[j], row_sums[j]) for j in name_range
]
plt.xticks(name_range, label_names)
plt.yticks(name_range, y_labels)
plt.xlabel('Predicted label')
plt.ylabel('True label')
# write percentage values (0.00 -- 1.00) into the confusion matrix
threshold = cm_norm.max() / 2. # used to decide for the font colour
for row in range(len(conf_matrix)):
for col in range(len(conf_matrix)):
if cm_norm[col, row] > threshold:
colour = 'white'
else:
colour = 'black'
# TODO: class names, not codes
ax2.text(row,
col,
cm_norm[col, row],
color=colour,
horizontalalignment='center')
plt.savefig(os.path.join(out_dir, str(i)))
plt.close()
r = model.fit(
# np.append(data_train, data_test, axis = 0),
# np.append(Y_multi_train, Y_multi_test, axis =0),
data_train, Y_multi_train,
epochs=10,
validation_data=(data_val, np.array(Y_multi_val),),
batch_size=64, verbose=1, shuffle=True, callbacks=callbacks_list
)
predicted_test = model.predict(data_test)>0.5 #predicting test data in trained model
predicted_test = predicted_test*1
predicted_val = model.predict(data_val)>0.5 #predicting test data in trained model
predicted_val = predicted_val*1
#Inspecting confusion matrix for val data set as well as test data set
val_con = math.confusion_matrix(Y_multi_test.argmax(axis = 1), predicted_test.argmax(axis = 1))
test_con = math.confusion_matrix(Y_multi_val.argmax(axis = 1), predicted_val.argmax(axis = 1))
print(val_con)
print(test_con)
test = "all the days the stock went low and it did not improve even at the end of the day"
test = clean_text_round1(test)
# print(test)
print(test)
test_sen = np.argmax(prediction_(test, model, T))
print(test_sen)
if test_sen == 0:
print("Postive sentance ")
elif test_sen == 1:
print("negative sentance")
else:
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
history = model.fit(X[train],
y[train],
epochs=N_EPOCHS,
batch_size=B_SIZE,
shuffle=1,
validation_split=0.2,
verbose=0)
fitness = model.evaluate(X[test], y[test])
accuracy_scores.append(fitness[1])
cm = confusion_matrix(y[test], tf.math.round(model.predict(X[test])))
cm_scores.append(cm.numpy())
y_pred = tf.math.round(model.predict(X[test])).numpy()
mcc_scores.append(matthews_corrcoef(y[test], y_pred))
print("\n\nEnd training! ")
accuracy_std_deviation = np.std(accuracy_scores)
print("accuracy:")
print(f"mean: {np.mean(accuracy_scores)}")
print(f"standard deviation: {accuracy_std_deviation}")
# compute confidence interval
cfi = scipy.stats.t.interval(0.95,
len(accuracy_scores) - 1,
loc=np.mean(accuracy_scores),
print('Loading dataset...')
X_train, y_train, X_val, y_val, X_test, y_test = loadAugmentedBinaryDatasetFromFiles(
)
print('Loading model...')
model = load_model(full_path)
print('Evaluating model on validation dataset...')
results = model.evaluate(X_val,
y_val,
batch_size=4,
return_dict=True,
verbose=0)
predictions = model.predict(X_val, batch_size=4, verbose=0)
predictions = [round(p[0]) for p in predictions]
conf_matrix = confusion_matrix(y_val, predictions)
# Define print results function
def printResults(res):
for metric in res:
print(" {}: {}".format(metric, res[metric]))
def printConfusionMatrix(cf):
print(cf.numpy())
print('Validation Results:')
printResults(results)
printConfusionMatrix(conf_matrix)
def plotAndSave(filename, history, model, X_val, y_val):
# Create the figure.
f = plt.figure(figsize=(15, 10))
f.tight_layout()
# Create subplots for each metric and confusion matrix.
ax1 = f.add_subplot(231, xlabel='Epoch', ylabel='Accuracy', ylim=[0, 1])
ax2 = f.add_subplot(232, xlabel='Epoch', ylabel='MSE', ylim=[0, 1])
ax3 = f.add_subplot(233, xlabel='Epoch', ylabel='Precision', ylim=[0, 1])
ax4 = f.add_subplot(234, xlabel='Epoch', ylabel='Recall', ylim=[0, 1])
ax5 = f.add_subplot(235, xlabel='Epoch', ylabel='Loss')
ax6 = f.add_subplot(236, xlabel='Predicted', ylabel='Actual')
ax1.plot(history.history['accuracy'], label='accuracy')
ax1.plot(history.history['val_accuracy'], label='val_accuracy')
ax1.legend(loc='lower right')
ax2.plot(history.history['mse'], label='mse')
ax2.plot(history.history['val_mse'], label='val_mse')
ax2.legend(loc='lower right')
ax3.plot(history.history['precision'], label='precision')
ax3.plot(history.history['val_precision'], label='val_precision')
ax3.legend(loc='lower right')
ax4.plot(history.history['recall'], label='recall')
ax4.plot(history.history['val_recall'], label='val_recall')
ax4.legend(loc='lower right')
ax5.plot(history.history['loss'], label='loss')
ax5.plot(history.history['val_loss'], label='val_loss')
ax5.set_yscale('log')
ax5.legend(loc='lower right')
# Make validation predictions and construct confusion matrix.
predictions = model.predict(X_val, batch_size=2)
predictions = [round(p[0]) for p in predictions]
# print("Predictions:", predictions)
# print("Y_val:", y_val)
cf = confusion_matrix(y_val, predictions)
cf = np.array(cf)
# Plot confusion matrix.
im_id = ax6.imshow(cf, interpolation='nearest')
ax6.set_xticks(np.arange(2))
ax6.set_yticks(np.arange(2))
ax6.set_xticklabels(['positive', 'negative'])
ax6.set_yticklabels(['positive', 'negative'])
ax6.set_title("Confusion Matrix")
f.colorbar(im_id)
plt.setp(ax6.get_yticklabels(),
rotation=90,
ha="center",
va="baseline",
rotation_mode="anchor")
for i in range(2):
for j in range(2):
ax6.text(j, i, cf[i, j], ha="center", va="center", color="w")
# Save the figure as an image.
f.savefig(filename)
# train_dataset,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
validation_data=(X_val, y_val),
# validation_data=val_dataset,
callbacks=[earlyStopping, mcp_save, reduce_lr_loss, csv_logger],
verbose=VERBOSITY_1)
plotAndSave('./baseline_results.png', history, model, X_val, y_val)
print('Evaluating results...')
# Generate confusion matrix again, for evaluation and training dataset.
cf_text = ""
predictions = model.predict(X_val, batch_size=BATCH_SIZE)
predictions = [round(p[0]) for p in predictions]
cf = confusion_matrix(y_val, predictions)
cf_text += "Validation\n" + str(cf.numpy())[1:-1].replace('\n ',
'\n') + "\n"
predictions = model.predict(X_train, batch_size=BATCH_SIZE)
predictions = [round(p[0]) for p in predictions]
cf = confusion_matrix(y_train, predictions)
cf_text += "Training\n" + str(cf.numpy())[1:-1].replace('\n ', '\n') + "\n"
# Save confusion matrices in text format into the file.
file = open('vgg16_confusion_matrix.txt', 'w')
file.write(cf_text)
file.close()
print(cf_text)
# Normalize the data
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
# Define our neural network model
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(28,28,1)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
# Compile the model
model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy'])
# Fit the model according to the parameters
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test))
# Evaluate the result, and generate confusion matrix
result = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', result[0])
print('Test accuracy:', result[1] * 100)
print('Confusion Matrix:', math.confusion_matrix(x_test, y_test))