def test_accuracy(self):
matrix = confusion_matrix(self.one_dimensional_result,
self.one_dimensional_expected)
actual = accuracy(matrix)
expected = (Y_AS_Y + N_AS_N) / (CLASS_Y + CLASS_N)
assert abs(actual - expected) < EPSILON
matrix = confusion_matrix(self.multi_dimensional_result,
self.multi_dimensional_expected)
actual = accuracy(matrix)
expected = (A_AS_A + B_AS_B + C_AS_C) / (CLASS_A + CLASS_B + CLASS_C)
assert abs(actual - expected) < EPSILON
def test_confusion_matrix(self):
actual = confusion_matrix(self.one_dimensional_result,
self.one_dimensional_expected)
expected = np.array([[Y_AS_Y, N_AS_Y], [Y_AS_N, N_AS_N]])
assert ((actual - expected) < EPSILON).all()
actual = confusion_matrix(self.multi_dimensional_result,
self.multi_dimensional_expected)
expected = np.array([[A_AS_A, B_AS_A,
C_AS_A], [A_AS_B, B_AS_B, C_AS_B],
[A_AS_C, B_AS_C, C_AS_C]])
assert ((actual - expected) < EPSILON).all()
def test_recall(self):
matrix = confusion_matrix(self.one_dimensional_result,
self.one_dimensional_expected)
actual = recall(matrix)
expected = np.array([Y_AS_Y / CLASS_Y, N_AS_N / CLASS_N])
assert (np.abs(actual - expected) < EPSILON).all()
matrix = confusion_matrix(self.multi_dimensional_result,
self.multi_dimensional_expected)
actual = recall(matrix)
expected = np.array(
[A_AS_A / CLASS_A, B_AS_B / CLASS_B, C_AS_C / CLASS_C])
assert (np.abs(actual - expected) < EPSILON).all()
def test_exception(self):
test = False
wrong_output = np.random.rand(2, 1000)
try:
confusion_matrix(wrong_output, self.one_dimensional_expected)
except ValueError:
test = True
assert test
test = False
try:
confusion_matrix(wrong_output, self.one_dimensional_expected)
except ValueError:
test = True
assert test
def test_precision(self):
matrix = confusion_matrix(self.one_dimensional_result,
self.one_dimensional_expected)
actual = precision(matrix)
expected = np.array(
[Y_AS_Y / (Y_AS_Y + N_AS_Y), N_AS_N / (Y_AS_N + N_AS_N)])
assert (np.abs(actual - expected) < EPSILON).all()
matrix = confusion_matrix(self.multi_dimensional_result,
self.multi_dimensional_expected)
actual = precision(matrix)
expected = np.array([
A_AS_A / (A_AS_A + B_AS_A + C_AS_A),
B_AS_B / (A_AS_B + B_AS_B + C_AS_B),
C_AS_C / (A_AS_C + B_AS_C + C_AS_C)
])
assert (np.abs(actual - expected) < EPSILON).all()
def test_f1_score(self):
matrix = confusion_matrix(self.one_dimensional_result,
self.one_dimensional_expected)
actual = f1_score(matrix)
a = np.array([Y_AS_Y / (Y_AS_Y + N_AS_Y), N_AS_N / (Y_AS_N + N_AS_N)])
b = np.array([Y_AS_Y / CLASS_Y, N_AS_N / CLASS_N])
expected = 2 * (a * b) / (a + b)
assert (np.abs(actual - expected) < EPSILON).all()
matrix = confusion_matrix(self.multi_dimensional_result,
self.multi_dimensional_expected)
actual = f1_score(matrix)
a = np.array([
A_AS_A / (A_AS_A + B_AS_A + C_AS_A),
B_AS_B / (A_AS_B + B_AS_B + C_AS_B),
C_AS_C / (A_AS_C + B_AS_C + C_AS_C)
])
b = np.array([A_AS_A / CLASS_A, B_AS_B / CLASS_B, C_AS_C / CLASS_C])
expected = 2 * (a * b) / (a + b)
assert (np.abs(actual - expected) < EPSILON).all()
fig = plt.figure(figsize=FIG_SIZE)
fig.subplots_adjust(wspace=0.3)
ax = fig.add_subplot(121)
ax2 = ax.twinx()
ax3 = fig.add_subplot(122)
lines = []
c_m = np.array([])
iteration = ""
for i in range(k):
trained, (learn, costs) = trainer.train(network, epochs=N, repeat=True)
prediction = trainer.evaluate(trained)
if c_m.shape != (0, ):
c_m = c_m + confusion_matrix(prediction, trainer.get_labels())
else:
c_m = confusion_matrix(prediction, trainer.get_labels())
line = ax.plot(learn, label="Learning Curve", linewidth=2.5)
if k != 1:
iteration = " iteration: {}".format(i + 1)
c = line[0].get_color()
else:
c = "r"
line2 = ax2.plot(costs,
label="MSE{}".format(iteration),
linestyle="--",
linewidth=2.5,
def train_evaluate(architecture: dict, dataset_name: str) -> NeuralNetwork:
"""
Train and evaluate a Network
:param architecture: Architecture of NeuralNetwork (above)
:param dataset_name: Dataset to use
:return: Trained Neural Network
"""
# import dataset
dataset = import_data("../data/{}.data".format(dataset_name))
dataset = oversample(dataset)
more_info = "(oversample)"
labels, encoding = one_hot_encoding(dataset[-1])
classes = list(encoding.keys())
dataset = np.delete(dataset, -1, 0)
dataset = np.delete(dataset, [0], 0)
# Initialize network
logging.info("Input size: {}\tOutput size: {}".format(
dataset.shape[0], len(encoding)))
network = NeuralNetwork(dataset.shape[0], architecture["INTERNAL_LAYERS"],
len(encoding), architecture["ACT_FUNCS"],
architecture["LR"])
# Define Trainer
trainer = StandardTrainer(dataset, labels.T, TRAIN_SIZE)
fig = plt.figure(figsize=FIG_SIZE)
fig.subplots_adjust(wspace=0.3)
ax = fig.add_subplot(121)
ax2 = ax.twinx()
ax3 = fig.add_subplot(122)
trained, (learn, costs) = trainer.train(network,
epochs=EPOCHS,
repeat=True)
prediction = trainer.evaluate(trained)
c_m = confusion_matrix(prediction, trainer.get_labels())
line = ax.plot(learn, label="Learning Curve", linewidth=2.5, c="b")
line2 = ax2.plot(costs, label="MSE", linestyle="--", linewidth=2.5, c="r")
lines = line + line2
ax.set_ylabel("Learning Curve", fontsize=FONT_SIZE)
ax.set_xlabel("Epochs", fontsize=FONT_SIZE)
ax.set_title("Network on {}\n".format(dataset_name), fontsize=TITLE_SIZE)
ax.grid()
ax2.set_ylabel("Cost", fontsize=FONT_SIZE)
ax2.grid()
labels = [l.get_label() for l in lines]
ax2.legend(lines, labels, fontsize=FONT_SIZE, loc="center right")
show_matrix(
ax3, c_m,
(classes, ["Predicted\n{}".format(a_class) for a_class in classes]),
"Confusion Matrix of Test Set\n", FONT_SIZE, TITLE_SIZE)
measures = {
"accuracy": accuracy(c_m),
"precision": precision(c_m),
"recall": recall(c_m),
"f1_score": f1_score(c_m)
}
print("Summary on {}:\n".format(dataset))
report_results(c_m)
plt.savefig("../results/Network_on_{}{}.png".format(
dataset_name, more_info))
return trained
N = int(1e5)
X_MIN = Y_MIN = -50
X_MAX = Y_MAX = 50
FIG_SIZE = (28, 12)
FONT_SIZE = 20
TITLE_SIZE = 30
np.random.seed(2)
if __name__ == '__main__':
dataset = import_data("../../data/iris.data")[4]
classes = np.unique(dataset)
labels, _ = one_hot_encoding(dataset)
prediction, _ = one_hot_encoding(
np.random.choice(["a", "b", "c"], size=labels.shape[0], replace=True))
matrix = confusion_matrix(prediction.T, labels.T)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=FIG_SIZE)
show_matrix(ax1, matrix, ([classes[0], classes[1], classes[2]], [
"Predicted\n" + classes[0], "Predicted\n" + classes[1],
"Predicted\n" + classes[2]
]), "Confusion matrix of a iris dataset\n", FONT_SIZE, TITLE_SIZE)
measures = np.zeros((3, 4))
ax2.matshow(measures, cmap="Greys")
to_show = np.zeros((3, 4))
to_show[0][0] = round(accuracy(matrix), 4)
to_show[1][0] = np.nan
to_show[2][0] = np.nan
_precision = precision(matrix)
to_show[0][1] = round(_precision[0], 4)
to_show[1][1] = round(_precision[1], 4)
to_show[2][1] = round(_precision[2], 4)
from useful.results import show_matrix, annotate
N = int(1e5)
X_MIN = Y_MIN = -50
X_MAX = Y_MAX = 50
FIG_SIZE = (24, 12)
FONT_SIZE = 20
TITLE_SIZE = 30
np.random.seed(2)
if __name__ == '__main__':
labels = Circle(30, (0, 0), (X_MIN, X_MAX), (Y_MIN, Y_MAX)).training_set(N)[2]
prediction = np.abs(np.random.randn(N))
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=FIG_SIZE)
matrix = confusion_matrix(prediction, labels)
show_matrix(ax1, matrix, (["Outside Circle", "Inside Circle"],
["Predicted\nOutside", "Predicted\nInside"]), "Confusion matrix of a circle\n",
FONT_SIZE, TITLE_SIZE, add_label=np.array([["TP: ", "FN: "], ["FP: ", "TN: "]]))
measures = np.zeros((2, 4))
ax2.matshow(measures, cmap="Greys")
to_show = np.zeros((2, 4))
to_show[0][0] = round(accuracy(matrix), 4)
_precision = precision(matrix)
to_show[0][1] = round(_precision[0], 4)
to_show[1][1] = round(_precision[1], 4)
_recall = recall(matrix)
to_show[0][2] = round(_recall[0], 4)
to_show[1][2] = round(_recall[1], 4)
_f1_score = f1_score(matrix)
to_show[0][3] = round(_f1_score[0], 4)
train_set, test_set = split_set(dataset, TRAIN_SIZE)
epochs = []
accuracies = []
recalls = []
for x1, x2, x3, x4, label in train_set.T.tolist():
epochs.append(
neuron.train([x1, x2, x3, x4], label)
)
prediction = []
labels = []
for y1, y2, y3, y4, label2 in test_set.T.tolist():
prediction.append(neuron.feed([y1, y2, y3, y4]))
labels.append(label2)
c_m = confusion_matrix(np.array(prediction), np.array(labels))
accuracies.append(accuracy(c_m))
recalls.append(recall(c_m)[0])
fig, ax = plt.subplots(figsize=FIG_SIZE)
ax.plot(epochs, accuracies, label="Accuracy", linewidth=3.5)
ax.plot(epochs, recalls, "--", label="Recall", linewidth=2.0)
ax.set_xlabel("Epochs", fontsize=20)
ax.set_ylabel("Measure", fontsize=20)
ax.set_title("Sigmoid neuron on Iris Dataset\n", fontsize=30)
ax.legend(fontsize=20)
ax.grid()
plt.savefig("../results/neuron_on_iris.png")