def run_sgd(X_train, X_test, y_train, y_test): ''' Function to fit Stochastic Gradient Descent Model and predict on test set Input: Train and Test Data. Output: Dataframe with metric scores. Confusion Matrix and ROC Curve plots. ''' print('SGD Results:') # Fit model and get predictions model = SGDClassifier() X_train_res, y_train_res = helper_functions.smote_train(X_train, y_train) clf = model.fit(X_train_res, y_train_res) calibrator = CalibratedClassifierCV(clf, cv='prefit') model_fit = calibrator.fit(X_train_res, y_train_res) y_hat_test = model.predict(X_test) # Calculate metrics prec = precision_score(y_test, y_hat_test) recall = recall_score(y_test, y_hat_test) acc = accuracy_score(y_test, y_hat_test) f1 = f1_score(y_test, y_hat_test) print() print("Model Metrics:") print(f"Precision: {prec}") print(f"Recall: {recall}") print(f"Accuracy: {acc}") print(f"F1_Score: {f1}") # Plot Confusion Matrix skplt.metrics.plot_confusion_matrix(y_test, y_hat_test, figsize = (4,4)) plt.ylim([1.5, -.5]) plt.tight_layout() plt.show() # Plot ROC Curve fpr, tpr, thresholds = roc_curve(y_test, calibrator.predict_proba(X_test)[:,1]) AUC = auc(fpr, tpr) scores = [prec, recall, acc, f1, AUC] print() print(f'AUC: {AUC}') plt.plot(fpr, tpr, lw = 2, label = 'ROC Curve', color = 'orange') plt.plot([0,1], [0,1], lw = 2, linestyle = '--', color = 'r') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.title('ROC Curve and AUC') plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.tight_layout() plt.show() print('-'*75) return pd.DataFrame({ 'Model' : ['SGD' for i in scores], 'Metric' : ['Precision', 'Recall', 'Accuracy', 'F1_Score', 'AUC'], 'Score' : scores})
def grid_search_estimators(X_train, y_train): ''' Function to grid search for best n value (# of estimators for random forest) Input: Training features and target Output: Optimized n (n_estimators) parameter for Random Forest ''' N = list(range(70, 125, 5)) hyperparameters = dict(n_estimators=N) rf = RandomForestClassifier() clf = GridSearchCV(rf, hyperparameters, cv=5, verbose=0) X_train_res, y_train_res = helper_functions.smote_train(X_train, y_train) grid = clf.fit(X_train_res, y_train_res) n = grid.best_estimator_.get_params()['n_estimators'] return n
def grid_search_neighbors(X_train, y_train): ''' Function to grid search for best k value (# of neighbors for knn) Input: Training features and target Output: Optimized k (n_neighbors) parameter for KNN ''' K = list(range(1, 9, 2)) hyperparameters = dict(n_neighbors=K) knn = KNeighborsClassifier() clf = GridSearchCV(knn, hyperparameters, cv=5, verbose=0) X_train_res, y_train_res = helper_functions.smote_train(X_train, y_train) grid = clf.fit(X_train_res, y_train_res) k = grid.best_estimator_.get_params()['n_neighbors'] return k
def run_linear_svc(X_train, X_test, y_train, y_test): ''' Function to fit Linear Support Vector Machine Model and predict on test set Input: Train and Test Data. Output: Dataframe with metric scores. Confusion Matrix. ''' print('Linear SVC Results:') # Fit model and get predictions model = LinearSVC() X_train_res, y_train_res = helper_functions.smote_train(X_train, y_train) model_fit = model.fit(X_train_res, y_train_res) y_hat_test = model.predict(X_test) # Calculate metrics prec = precision_score(y_test, y_hat_test) recall = recall_score(y_test, y_hat_test) acc = accuracy_score(y_test, y_hat_test) f1 = f1_score(y_test, y_hat_test) print() print("Model Metrics:") print(f"Precision: {prec}") print(f"Recall: {recall}") print(f"Accuracy: {acc}") print(f"F1_Score: {f1}") # Plot Confusion Matrix skplt.metrics.plot_confusion_matrix(y_test, y_hat_test, figsize = (4,4)) plt.ylim([1.5, -.5]) plt.tight_layout() plt.show() scores = [prec, recall, acc, f1] print('-'*75) return pd.DataFrame({ 'Model' : ['Linear_SVC' for i in scores], 'Metric' : ['Precision', 'Recall', 'Accuracy', 'F1_Score'], 'Score' : scores})
def grid_search_CP(X_train, y_train): ''' Function to grid search for best C and penalty for Logistic Regression Input: Training features and target Output: Optimized C and Penalty parameters for Logistic Regression ''' penalty = ['l1', 'l2'] C = np.arange(.1, 50, .5) hyperparameters = dict(C=C, penalty=penalty) lr = LogisticRegression() clf = GridSearchCV(lr, hyperparameters, cv=5, verbose=0) X_train_res, y_train_res = helper_functions.smote_train(X_train, y_train) grid = clf.fit(X_train_res, y_train_res) c = grid.best_estimator_.get_params()['C'] p = grid.best_estimator_.get_params()['penalty'] return c, p
def run_random_forest(X_train, X_test, y_train, y_test, n): ''' Function to fit Random Forest Model and predict on test set Input: Train and Test Data. Optimized n parameter. Output: Dataframe with metric scores. Confusion Matrix and ROC Curve plots. Feature Importance Plot. ''' print('Random Forest Results:') # Fit model and get predictions model = RandomForestClassifier(n_estimators = n) X_train_res, y_train_res = helper_functions.smote_train(X_train, y_train) model_fit = model.fit(X_train_res, y_train_res) y_hat_test = model.predict(X_test) # Calculate metrics prec = precision_score(y_test, y_hat_test) recall = recall_score(y_test, y_hat_test) acc = accuracy_score(y_test, y_hat_test) f1 = f1_score(y_test, y_hat_test) print() print("Model Metrics:") print(f"Precision: {prec}") print(f"Recall: {recall}") print(f"Accuracy: {acc}") print(f"F1_Score: {f1}") # Plot Confusion Matrix skplt.metrics.plot_confusion_matrix(y_test, y_hat_test, figsize = (4,4)) plt.ylim([1.5, -.5]) plt.tight_layout() plt.show() # Plot ROC Curve fpr, tpr, thresholds = roc_curve(y_test, model.predict_proba(X_test)[:,1]) AUC = auc(fpr, tpr) scores = [prec, recall, acc, f1, AUC] print() print(f'AUC: {AUC}') plt.plot(fpr, tpr, lw = 2, label = 'ROC Curve', color = 'orange') plt.plot([0,1], [0,1], lw = 2, linestyle = '--', color = 'r') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.title('ROC Curve and AUC') plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.tight_layout() plt.show() # Plot feature importances features = list(X_train.columns) importances = model.feature_importances_ indices = np.argsort(importances) plt.figure(figsize = (10,10)) plt.title('Feature Importances') plt.barh(range(len(indices)), importances[indices], color='darkblue', align='center') plt.yticks(range(len(indices)), [features[i] for i in indices]) plt.xlabel('Relative Importance') plt.tight_layout() plt.show() print('-'*75) return pd.DataFrame({ 'Model' : ['Random_Forest' for i in scores], 'Metric' : ['Precision', 'Recall', 'Accuracy', 'F1_Score', 'AUC'], 'Score' : scores})
def run_logreg(X_train, X_test, y_train, y_test, C, penalty): ''' Function to fit Logistic Regression Model and predict on test set Input: Train and Test Data. Optimized C and Penalty parameters Output: Dataframe with metric scores. Confusion Matrix and ROC Curve plots. ''' # Print test data balances y_pos = y_test.target.value_counts()[1] drug_user_percent = round(y_pos/ len(y_test), 2) print(f'Drug user percent: {drug_user_percent * 100}%') print() print('Logisitic Regression Results:') # Fit model and get predictions model = LogisticRegression(C = C, penalty = penalty, fit_intercept = False, solver = 'liblinear') X_train_res, y_train_res = helper_functions.smote_train(X_train, y_train) model_fit = model.fit(X_train_res, y_train_res) y_hat_test = model.predict(X_test) # Calculate metrics prec = precision_score(y_test, y_hat_test) recall = recall_score(y_test, y_hat_test) acc = accuracy_score(y_test, y_hat_test) f1 = f1_score(y_test, y_hat_test) print() print("Model Metrics:") print(f"Precision: {prec}") print(f"Recall: {recall}") print(f"Accuracy: {acc}") print(f"F1_Score: {f1}") # Plot Confusion Matrix skplt.metrics.plot_confusion_matrix(y_test, y_hat_test, figsize = (4,4)) plt.ylim([1.5, -.5]) plt.tight_layout() plt.show() # Plot ROC Curve fpr, tpr, thresholds = roc_curve(y_test, model.predict_proba(X_test)[:,1]) AUC = auc(fpr, tpr) scores = [prec, recall, acc, f1, AUC] print() print(f'AUC: {AUC}') plt.plot(fpr, tpr, lw = 2, label = 'ROC Curve', color = 'orange') plt.plot([0,1], [0,1], lw = 2, linestyle = '--', color = 'r') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.title('ROC Curve and AUC') plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.tight_layout() plt.show() print('-'*75) return pd.DataFrame({ 'Model' : ['Logistic_Regression' for i in scores], 'Metric' : ['Precision', 'Recall', 'Accuracy', 'F1_Score', 'AUC'], 'Score' : scores})