selected_test_X = test_X[possible_feature_sets[i]]
# First we run our non deterministic classifiers a number of times to average their score.
performance_tr_nn = 0
performance_tr_rf = 0
performance_tr_svm = 0
performance_te_nn = 0
performance_te_rf = 0
performance_te_svm = 0
for repeat in range(0, repeats):
print repeat
class_train_y, class_test_y, class_train_prob_y, class_test_prob_y = learner.feedforward_neural_network(
selected_train_X, train_y, selected_test_X, gridsearch=True)
performance_tr_nn += eval.accuracy(train_y, class_train_y)
performance_te_nn += eval.accuracy(test_y, class_test_y)
class_train_y, class_test_y, class_train_prob_y, class_test_prob_y = learner.random_forest(
selected_train_X, train_y, selected_test_X, gridsearch=True)
performance_tr_rf += eval.accuracy(train_y, class_train_y)
performance_te_rf += eval.accuracy(test_y, class_test_y)
class_train_y, class_test_y, class_train_prob_y, class_test_prob_y = learner.support_vector_machine_with_kernel(
selected_train_X, train_y, selected_test_X, gridsearch=True)
performance_tr_svm += eval.accuracy(train_y, class_train_y)
performance_te_svm += eval.accuracy(test_y, class_test_y)
overall_performance_tr_nn = performance_tr_nn / repeats
overall_performance_te_nn = performance_te_nn / repeats
overall_performance_tr_rf = performance_tr_rf / repeats
repeats = 20
for reg_param in reg_parameters:
performance_tr = 0
performance_te = 0
for i in range(0, repeats):
# besluiten of we dit gebruiken...
class_train_y, class_test_y, class_train_prob_y, class_test_prob_y = learner.feedforward_neural_network(
train_X,
train_y,
test_X,
hidden_layer_sizes=(250, ),
alpha=reg_param,
max_iter=500,
gridsearch=False)
performance_tr += eval.accuracy(train_y, class_train_y)
performance_te += eval.accuracy(test_y, class_test_y)
performance_training.append(performance_tr / repeats)
performance_test.append(performance_te / repeats)
plot.hold(True)
plot.semilogx(reg_parameters, performance_training, 'r-')
plot.semilogx(reg_parameters, performance_test, 'b:')
print performance_training
print performance_test
plot.xlabel('regularization parameter value')
plot.ylabel('accuracy')
plot.ylim([0.95, 1.01])
plot.legend(['training', 'test'], loc=4)
plot.hold(False)
# First we run our non deterministic classifiers a number of times to average their score.
performance_tr_nn = 0
performance_tr_rf = 0
performance_tr_svm = 0
performance_te_nn = 0
performance_te_rf = 0
performance_te_svm = 0
for repeat in range(0, repeats):
print repeat
class_train_y, class_test_y, class_train_prob_y, class_test_prob_y = learner.feedforward_neural_network(selected_train_X, train_y, selected_test_X, gridsearch=True, alpha=10, max_iter=50)
performance_tr_nn += eval.accuracy(train_y, class_train_y)
performance_te_nn += eval.accuracy(test_y, class_test_y)
"""
class_train_y, class_test_y, class_train_prob_y, class_test_prob_y = learner.random_forest(selected_train_X, train_y, selected_test_X, gridsearch=True)
performance_tr_rf += eval.accuracy(train_y, class_train_y)
performance_te_rf += eval.accuracy(test_y, class_test_y)
class_train_y, class_test_y, class_train_prob_y, class_test_prob_y = learner.support_vector_machine_with_kernel(selected_train_X, train_y, selected_test_X, gridsearch=True)
performance_tr_svm += eval.accuracy(train_y, class_train_y)
performance_te_svm += eval.accuracy(test_y, class_test_y)
"""
overall_performance_tr_nn = performance_tr_nn/repeats
overall_performance_te_nn = performance_te_nn/repeats
"""