def test_F1_micro(self): cf_matrix = ConfusionMatrix(3) cf_matrix.matrix = TEST_MATRIX expected_f1_micro = 0.692307693 actual_f1_micro = cf_matrix.F1_micro() self.assertLess(abs(actual_f1_micro - expected_f1_micro), 10e-6)
def test_TP(self): cf_matrix = ConfusionMatrix(3) cf_matrix.matrix = TEST_MATRIX expected_TP = np.array([10, 20, 15]) actual_TP = cf_matrix._TP() self.assertTrue(np.equal(actual_TP, expected_TP).all())
def test_accuracy(self): cf_matrix = ConfusionMatrix(3) cf_matrix.matrix = TEST_MATRIX expected_acc = 0.692307692 actual_acc = cf_matrix.accuracy() self.assertLess(abs(actual_acc - expected_acc), 10e-6)
def test_F1_score(self): cf_matrix = ConfusionMatrix(3) cf_matrix.matrix = TEST_MATRIX expected_f1 = np.array([0.588235288, 0.740740741, 0.714285715]) actual_f1 = cf_matrix.F1_score() self.assertTrue(np.isclose(actual_f1, expected_f1).all())
def test_precision(self): cf_matrix = ConfusionMatrix(3) cf_matrix.matrix = TEST_MATRIX expected_precision = np.array([0.6666667, 0.740740741, 0.652173913]) actual_precision = cf_matrix.precision() self.assertTrue(np.isclose(actual_precision, expected_precision).all())
def test_FN(self): cf_matrix = ConfusionMatrix(3) cf_matrix.matrix = TEST_MATRIX expected_FN = np.array([9, 7, 4]) actual_FN = cf_matrix._FN() self.assertTrue(np.equal(actual_FN, expected_FN).all())
def test_FP(self): cf_matrix = ConfusionMatrix(3) cf_matrix.matrix = TEST_MATRIX expected_FP = np.array([5, 7, 8]) actual_FP = cf_matrix._FP() self.assertTrue(np.equal(actual_FP, expected_FP).all())
def test_recall(self): cf_matrix = ConfusionMatrix(3) cf_matrix.matrix = TEST_MATRIX expected_recall = np.array([0.526315789, 0.740740741, 0.789473684]) actual_recall = cf_matrix.recall() self.assertTrue(np.isclose(actual_recall, expected_recall).all())
def run_experiment(dataset_file, target_column, specs, onehot=False, regression=False, verbose=False, check_gradient=False): lr = specs[1] batch_size = specs[2] k_count = specs[3] lambd = specs[4] network_specs = specs[0] epochs = EPOCHS print('========== Starting Experiment ===========') print('Learning Rate: \t\t %.5f' % lr) print('Batch Size: \t\t %d' % batch_size) print('K-Folds: \t\t %d' % k_count) print('Lambda: \t\t %.4f' % lambd) print('Netowrk Architecture: \t %s' % "IN " + "-".join([str(x['size']) for x in network_specs]) + " OUT") kfolds = DatasetBuilder.read_dataset_from_csv_as_kfold( dataset_file, target_column, k_count) results = {'specs': specs, 'folds': []} for fold_idx, (train, test) in enumerate(kfolds.get_folds()): print('------ Starting Fold %d ----------' % (fold_idx + 1)) fold_results = {'epochs': {}} results['folds'].append(fold_results) nnet = NetworkBuilder.build_network_from_specs(lambd, lr, network_specs, regression) dataloader = DataLoader(train.drop(target_column, axis=1).values, train[[target_column]].values, batch_size, shuffle=True, onehot=onehot) prev_loss = 99999999 loss = 0 loss_count = 0 for epoch in range(epochs): for data in dataloader: X, Y = data pred = nnet.forward(X) loss += nnet.loss(pred, Y) loss_count += 1 nnet.backprop(Y) loss /= loss_count if (verbose): print("Epoch: %d \t Loss: %f" % (epoch, loss)) fold_results['epochs'][epoch] = { 'loss': loss, } if abs(loss - prev_loss) < EPSILON: if verbose: print('stopped because of small loss gain') break else: prev_loss = loss loss = 0 loss_count = 0 X_test = test.drop(target_column, axis=1).values Y_test = test[[target_column]].values Y_pred = nnet.forward(X_test) if not regression: cf_matrix = ConfusionMatrix(train[target_column].max() + 1) cf_matrix.update(Y_pred, Y_test, onehot) fold_results['cf_matrix'] = cf_matrix.matrix.tolist() fold_results['accuracy'] = cf_matrix.accuracy() fold_results['f1_macro'] = cf_matrix.F1_macro() else: fold_results['rmse'] = RegressionMetrics.rmse(Y_pred, Y_test) fold_results['mse'] = RegressionMetrics.mse(Y_pred, Y_test) fold_results['mean_error'] = RegressionMetrics.mean_error( Y_pred, Y_test) print(fold_results) if not regression: results['accuracy'] = sum([f['accuracy'] for f in results['folds'] ]) / len(results['folds']) results['f1_macro'] = sum([f['f1_macro'] for f in results['folds'] ]) / len(results['folds']) else: results['rmse'] = sum([f['rmse'] for f in results['folds']]) / len( results['folds']) return results