def main(config = None):
run = wandb.init(config=config, resume=True)
config = wandb.config
hl = [config.hidden_layer_size] * config.hidden_layers # Hidden layers
ol = [len(train_y[0])] # Output layers
n_hl = len(hl)
name = "hl_" + str(config.hidden_layers) + "_bs_" + str(config.batch_size) + "_ac_" + config.ac
run.name = name
logConfig.logConfig(config)
# Set Loss function here
loss_functions = [ "cross_entropy", "sq_loss" ]
W, b = train.train(train_x, train_y, val_x, val_y, d, hl, ol, config, loss_functions[0])
test_acc, test_loss, y, _y = accuracy_loss.get_accuracy_loss_and_prediction(W, b, test_x, test_y, n_hl, config.ac,loss_functions[0])
confusion_matrix= plot.confusion_matrix(labels, y, _y)
if len(loss_functions) == 2:
W_, b_ = train.train(train_x, train_y, val_x, val_y, d, hl, ol, config, loss_functions[1])
test_acc_, test_loss_ = accuracy_loss.get_accuracy_and_loss(W_, b_, test_x, test_y, n_hl, config.ac, loss_functions[1])
wandb.log( { "test_accuracy": test_acc, "test_loss": test_loss , "test_accuracy (Squared Loss)": test_acc_, "test_loss (Squared Loss)": test_loss_ , "Confusion Matrix": confusion_matrix } )
else:
wandb.log( { "test_accuracy": test_acc, "test_loss": test_loss }, {"Confusion Matrix": confusion_matrix } )
run.finish()
def confusion_matrix(self, prediction, testing_DTData):
sample_index = 0
cm = []
for i in testing_DTData.labels:
cm.append([0] * len(testing_DTData.labels))
for sample_index, label in enumerate(prediction):
if label != 'no sample':
real_label_index = testing_DTData.labels.index(
testing_DTData.dataset_labels[sample_index])
predict_label_index = testing_DTData.labels.index(label)
cm[real_label_index][predict_label_index] += 1
sample_index += 1
print(cm)
plot.confusion_matrix(cm=np.array(cm),
target_names=testing_DTData.labels,
normalize=False,
title="Confusion Matrix, Normalized")
def confusion_matrix_crsv(self, prediction):
# fold = len(prediction)
sample_index = 0
for list1 in prediction:
cm = []
for i in self.data.labels:
cm.append([0] * len(self.data.labels))
for label in list1:
if label != 'no sample':
real_label_index = self.data.labels.index(
self.data.dataset_labels[sample_index])
predict_label_index = self.data.labels.index(label)
cm[real_label_index][predict_label_index] += 1
sample_index += 1
print(cm)
plot.confusion_matrix(cm=np.array(cm),
target_names=self.data.labels,
normalize=False,
title="Confusion Matrix, Normalized")
def process_test_data(model, n=30):
l, y, _y = 2000, [], []
for i in range(l):
y.append(test_set[i][1][0].argmax())
_y = model.predict(test_set).argmax(axis=1)
test_acc = sum([_y[i] == y[i] for i in range(l)]) / l
confusion_matrix = plot.confusion_matrix(labels, y, _y)
])
tf.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
with utils.Watch('Fitting model'):
tf.fit(x_train, y_train, epochs=2) #, verbose=0)
with utils.Watch('Evaluating model'):
print(tf.evaluate(x_test, y_test))
with utils.Watch('Generating confusion matrix'):
y_pred = [numpy.argmax(p) for p in tf.predict(x_test)]
matrix = metrics.confusion_matrix(y_test, y_pred)
plot.confusion_matrix(matrix)
def predict(model):
test = pandas.read_csv('test.csv')
test /= 255.0
test = testx.values.reshape(len(test), 28, 28, 1)
# test = test.applymap(lambda x: x/255.0)
# test = numpy.array([expand_image(r) for r in test.to_numpy()])
with utils.Watch('Predicting test data'):
pred = pandas.DataFrame()
pred['ImageId'] = list(range(1, 28000 + 1))
pred['Label'] = [numpy.argmax(p) for p in tf.predict(test)]
pred.to_csv('predictions4.csv', index=False)
# a. creating the mlps
# - here we add just the best MLP (found via test)
classifiers = \
{'MLP': MLPClassifier(hidden_layer_sizes=(4,8,8,8,4,),activation='relu', solver='adam', max_iter=500),}
# b. training it and predicting
for mlp in classifiers:
# - training and testing to obtain the learning curve
train_sizes, train_scores, test_scores = learning_curve(classifiers[mlp], X, y,\
train_sizes=np.array([0.1, 0.25, 0.5, 0.75, 1. ]), cv=5)
plot.learning_curve(train_sizes,train_scores,test_scores)
# - cross validating the method
score = cross_validate(classifiers[mlp],X,y,return_train_score=True,return_estimator=True,cv=5)
print(mlp,np.array(score['train_score']).mean(),np.array(score['test_score']).mean())
# - performing the probabilistic estimation using the best fitted
# classificator into the cross validation
best_estimator = score['estimator'][list(score['test_score']).index(max(score['test_score']))]
y_pred = best_estimator.predict_proba(X)
y_prob = np.array([np.array([1,0]) if class_ == 0 else np.array([0,1]) for class_ in y])
y_dist = np.diag(distance.cdist(y_pred,y_prob,'chebyshev'))
# - plotting the probablistic result
y_sort = np.sort(y_dist)
plot.line_graph(np.arange(len(y_sort)),y_sort,'Amostra','Erro','predict_proba.pdf')
# - plotting confusion matrix
plot.confusion_matrix(best_estimator,X,y)
# That's all folks... :}
# Dimension of datapoints
d = w * h
config = {
"learning_rate": 0.001,
"epochs": 100,
"optimiser": "mgd",
"hidden_layers": 3,
"hidden_layer_size": 32,
"ac": "tanh",
"batch_size": 32,
"init_strategy": "xavier",
"weight_decay": 0.0005
}
# Data Preprocessing
# (train_x, train_y), (val_x, val_y), (test_x, test_y) = preprocessing.pre_process(d, n_labels, train_X, train_Y, test_X, test_Y)
# hl = [config["hidden_layer_size"]] * config["hidden_layers"] # Hidden layers
# ol = [len(train_y[0])] # Output layer
# # Model Training
# W, b = train.train(train_x[:n], train_y[:n], val_x[:n], val_y[:n], d, hl, ol, config)
# # Checking Accuracy
# a, l = accuracy_loss.get_accuracy_and_loss(W, b, train_x[:n], train_y[:n], len(hl), config["ac"])
# print("Accuracy: ", a, " Loss: ", l)
plot.confusion_matrix(labels, [0, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9],
[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 5, 7])