# param_grid=param_dist) # gs_rf.fit(training_set, class_set['diagnosis']) # print(gs_rf.best_params_) # end = time.time() # print(end - start) print(''' ############################### ## CROSS VALIDATION ## ############################### ''' ) # Cross validation hf.cross_val_metrics(fit_RF, training_set, class_set['diagnosis'], print_results = True) print(''' ############################### ## TEST SET CALCULATIONS ## ############################### ''' ) print(pd.crosstab(predictions_RF, test_class_set['diagnosis'], rownames=['Predicted Values'], colnames=['Actual Values'])) print("Here is our mean accuracy on the test set:\n {0: 0.3f}"\ .format(accuracy_RF))
# n_jobs = 3) # cv_rf.fit(training_set, class_set) # print('Best Parameters using grid search: \n', # cv_rf.best_params_) # end = time.time() # print('Time taken in grid search: {0: .2f}'\ #.format(end - start)) # Test Set Calculations ------------------------------------- # Test error rate test_error_rate_rf = 1 - accuracy_rf # Confusion Matrix test_crosstb = hf.create_conf_mat(test_class_set, predictions_rf) # Print Variable Importance hf.variable_importance(importances_rf, indices_rf) # Cross validation print('Cross Validation:') hf.cross_val_metrics(fit_rf, training_set, class_set, print_results=True) print('Confusion Matrix:') print(test_crosstb, '\n') print("Here is our mean accuracy on the test set:\n {0: 0.3f}"\ .format(accuracy_rf)) print("The test error rate for our model is:\n {0: .3f}"\ .format(test_error_rate_rf))
# 'activation': ["relu", "identity", "tanh", "logistic"]}) # gs.fit(training_set_scaled, class_set) # print(gs.best_params_) # end = time.time() # print(end - start) # Test Set Calculations ------------------------------------- # Test error rate test_error_rate_nn = 1 - accuracy_nn # Confusion Matrix test_crosstb = hf.create_conf_mat(test_class_set, predictions_nn) # Cross validation print("Cross Validation:") hf.cross_val_metrics(fit_nn, training_set_scaled, class_set, 'nn', print_results=True) print('Confusion Matrix:') print(test_crosstb, '\n') print("Here is our mean accuracy on the test set:\n {0: .3f}"\ .format(accuracy_nn)) print("The test error rate for our model is:\n {0: .3f}"\ .format(test_error_rate_nn))
# KNN ----------------------------------------------------------- metrics_knn = produce_model_metrics(fit_knn, test_set, test_class_set, 'knn') # Call each value from dictionary predictions_knn = metrics_knn['predictions'] accuracy_knn = metrics_knn['accuracy'] fpr = metrics_knn['fpr'] tpr = metrics_knn['tpr'] auc_knn = metrics_knn['auc'] # Test Error Rate test_error_rate_knn = 1 - accuracy_knn # Cross Validated Score mean_cv_knn, std_error_knn = cross_val_metrics(fit_knn, training_set, class_set, 'knn', print_results=False) # RF ------------------------------------------------------------ metrics_rf = produce_model_metrics(fit_rf, test_set, test_class_set, 'rf') # Call each value from dictionary predictions_rf = metrics_rf['predictions'] accuracy_rf = metrics_rf['accuracy'] fpr2 = metrics_rf['fpr'] tpr2 = metrics_rf['tpr'] auc_rf = metrics_rf['auc'] # Test Error Rate test_error_rate_rf = 1 - accuracy_rf
# 'learning_rate_init': [0.05, 0.01, 0.005, 0.001], # 'hidden_layer_sizes': [4, 8, 12], # 'activation': ["relu", "identity", "tanh", "logistic"]}) # gs.fit(training_set_scaled, class_set_scaled['diagnosis']) # print(gs.best_params_) # end = time.time() # print(end - start) print(''' ################################ ## CROSS VALIDATION ## ################################ ''' ) test_thing = hf.cross_val_metrics(fit_nn, training_set_scaled, class_set['diagnosis'], print_results = True) print(''' ############################### ## TEST SET CALCULATIONS ## ############################### ''' ) print(pd.crosstab(predictions_nn, test_class_set['diagnosis'], rownames=['Predicted Values'], colnames=['Actual Values'])) print("Here is our mean accuracy on the test set:\n {0: .3f}"\ .format(accuracy_nn))