# param_grid=param_dist)
# gs_rf.fit(training_set, class_set['diagnosis'])
# print(gs_rf.best_params_)
# end = time.time()
# print(end - start)
print('''
###############################
## CROSS VALIDATION ##
###############################
'''
)
# Cross validation
hf.cross_val_metrics(fit_RF, training_set, class_set['diagnosis'],
print_results = True)
print('''
###############################
## TEST SET CALCULATIONS ##
###############################
'''
)
print(pd.crosstab(predictions_RF,
test_class_set['diagnosis'],
rownames=['Predicted Values'],
colnames=['Actual Values']))
print("Here is our mean accuracy on the test set:\n {0: 0.3f}"\
.format(accuracy_RF))
# n_jobs = 3)
# cv_rf.fit(training_set, class_set)
# print('Best Parameters using grid search: \n',
# cv_rf.best_params_)
# end = time.time()
# print('Time taken in grid search: {0: .2f}'\
#.format(end - start))
# Test Set Calculations -------------------------------------
# Test error rate
test_error_rate_rf = 1 - accuracy_rf
# Confusion Matrix
test_crosstb = hf.create_conf_mat(test_class_set, predictions_rf)
# Print Variable Importance
hf.variable_importance(importances_rf, indices_rf)
# Cross validation
print('Cross Validation:')
hf.cross_val_metrics(fit_rf, training_set, class_set, print_results=True)
print('Confusion Matrix:')
print(test_crosstb, '\n')
print("Here is our mean accuracy on the test set:\n {0: 0.3f}"\
.format(accuracy_rf))
print("The test error rate for our model is:\n {0: .3f}"\
.format(test_error_rate_rf))
# 'activation': ["relu", "identity", "tanh", "logistic"]})
# gs.fit(training_set_scaled, class_set)
# print(gs.best_params_)
# end = time.time()
# print(end - start)
# Test Set Calculations -------------------------------------
# Test error rate
test_error_rate_nn = 1 - accuracy_nn
# Confusion Matrix
test_crosstb = hf.create_conf_mat(test_class_set, predictions_nn)
# Cross validation
print("Cross Validation:")
hf.cross_val_metrics(fit_nn,
training_set_scaled,
class_set,
'nn',
print_results=True)
print('Confusion Matrix:')
print(test_crosstb, '\n')
print("Here is our mean accuracy on the test set:\n {0: .3f}"\
.format(accuracy_nn))
print("The test error rate for our model is:\n {0: .3f}"\
.format(test_error_rate_nn))
# KNN -----------------------------------------------------------
metrics_knn = produce_model_metrics(fit_knn, test_set, test_class_set, 'knn')
# Call each value from dictionary
predictions_knn = metrics_knn['predictions']
accuracy_knn = metrics_knn['accuracy']
fpr = metrics_knn['fpr']
tpr = metrics_knn['tpr']
auc_knn = metrics_knn['auc']
# Test Error Rate
test_error_rate_knn = 1 - accuracy_knn
# Cross Validated Score
mean_cv_knn, std_error_knn = cross_val_metrics(fit_knn,
training_set,
class_set,
'knn',
print_results=False)
# RF ------------------------------------------------------------
metrics_rf = produce_model_metrics(fit_rf, test_set, test_class_set, 'rf')
# Call each value from dictionary
predictions_rf = metrics_rf['predictions']
accuracy_rf = metrics_rf['accuracy']
fpr2 = metrics_rf['fpr']
tpr2 = metrics_rf['tpr']
auc_rf = metrics_rf['auc']
# Test Error Rate
test_error_rate_rf = 1 - accuracy_rf
# 'learning_rate_init': [0.05, 0.01, 0.005, 0.001],
# 'hidden_layer_sizes': [4, 8, 12],
# 'activation': ["relu", "identity", "tanh", "logistic"]})
# gs.fit(training_set_scaled, class_set_scaled['diagnosis'])
# print(gs.best_params_)
# end = time.time()
# print(end - start)
print('''
################################
## CROSS VALIDATION ##
################################
'''
)
test_thing = hf.cross_val_metrics(fit_nn, training_set_scaled,
class_set['diagnosis'],
print_results = True)
print('''
###############################
## TEST SET CALCULATIONS ##
###############################
'''
)
print(pd.crosstab(predictions_nn,
test_class_set['diagnosis'],
rownames=['Predicted Values'],
colnames=['Actual Values']))
print("Here is our mean accuracy on the test set:\n {0: .3f}"\
.format(accuracy_nn))
