def base_model(X_train, y_train,X_test, y_test): model = LogisticRegression(multi_class='auto',solver='lbfgs') visualizer = ROCAUC(model, classes=['dancehall','reggae','soca','pop']) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) visualizer.show()
def evaluation(estimator, X, Y, x, y): classes = [Y[1], Y[0]] f, (ax, ax1, ax2) = plt.subplots(1, 3, figsize=(18, 6)) #Confusion Matrix cmm = ConfusionMatrix(model=estimator, ax=ax1, classes=classes, label_encoder={ 0.0: 'Negativo', 1.0: 'Positivo' }) cmm.score(x, y) #ROCAUC viz = ROCAUC(model=estimator, ax=ax2) viz.fit(X, Y) viz.score(x, y) #Learning Curve cv_strategy = StratifiedKFold(n_splits=3) sizes = np.linspace(0.3, 1.0, 10) visualizer = LearningCurve(estimator, ax=ax, cv=cv_strategy, scoring='roc_auc', train_sizes=sizes, n_jobs=4) visualizer.fit(X, Y) cmm.poof(), viz.poof(), visualizer.poof() plt.show()
def eva_model(c, n, X, y, X_test, y_test, class_names, outdir): model = svm.LinearSVC(class_weight='balanced', dual=False, max_iter=10000, C=c) rfe = RFE(model, n_features_to_select=n) ## learning curve plt.clf() viz_LC = LearningCurve( rfe, scoring='f1_weighted', n_jobs=4 ) viz_LC.fit(X, y) viz_LC.show(outpath=outdir + '/LC.png') ## classification report plt.clf() viz_CR = ClassificationReport(rfe, classes=class_names, support=True) viz_CR.fit(X, y) viz_CR.score(X_test, y_test) viz_CR.show(outpath=outdir + '/CR.png') ## confusion matrix plt.clf() viz_CM = ConfusionMatrix(rfe, classes=class_names) viz_CM.fit(X, y) viz_CM.score(X_test, y_test) viz_CM.show(outpath=outdir + '/CM.png') ## precision recall curve plt.clf() viz_PRC = PrecisionRecallCurve(rfe, per_class=True, iso_f1_curves=True, fill_area=False, micro=False, classes=class_names) viz_PRC.fit(X, y) viz_PRC.score(X_test, y_test) viz_PRC.show(outpath=outdir + '/PRC.png',size=(1080,720)) ## class prediction error plt.clf() viz_CPE = ClassPredictionError( rfe, classes=class_names ) viz_CPE.fit(X, y) viz_CPE.score(X_test, y_test) viz_CPE.show(outpath=outdir + '/CPE.png') ## ROCAUC plt.clf() viz_RA = ROCAUC(rfe, classes=class_names, size=(1080,720)) viz_RA.fit(X, y) viz_RA.score(X, y) viz_RA.show(outpath=outdir + '/RA.png') fit = rfe.fit(X,y) y_predict = fit.predict(X_test) f1 = f1_score(y_test, y_predict, average='weighted') features_retained_RFE = X.columns[rfe.get_support()].values feature_df =pd.DataFrame(features_retained_RFE.tolist()) feature_df.to_csv(outdir + '/features.csv', sep='\t', index=False) return f1
def ROC_Curve(Model, X, y): X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.15) model = Model visualizer = ROCAUC(Model) visualizer.fit(X_train, y_train) # Fit the training data to the visualizer visualizer.score(X_test, y_test) # Evaluate the model on the test data visualizer.show()
def ROC_AUC(model, classes, X_train, Y_train, X_test, Y_test): from yellowbrick.classifier import ROCAUC # Instantiate the visualizer with the classification model visualizer = ROCAUC(model, classes=classes) visualizer.fit(X_train, Y_train) # Fit the training data to the visualizer visualizer.score(X_test, Y_test) # Evaluate the model on the test data g = visualizer.poof()
def plotting_ROC_curve_yellowbrick(model, labels, X_train, y_train, X_test, y_test): st.subheader('ROC Curve') visualizer = ROCAUC(model, classes=labels) visualizer.fit(X_train, y_train) # Fit the training data to the visualizer visualizer.score(X_test, y_test) # Evaluate the model on the test data visualizer.show() st.pyplot() return
def log_roc_auc_chart(classifier, X_train, X_test, y_train, y_test, experiment=None): """Log ROC-AUC chart. Make sure you created an experiment by using ``neptune.create_experiment()`` before you use this method. Tip: Check `Neptune documentation <https://docs.neptune.ai/integrations/scikit_learn.html>`_ for the full example. Args: classifier (:obj:`classifier`): | Fitted sklearn classifier object X_train (:obj:`ndarray`): | Training data matrix X_test (:obj:`ndarray`): | Testing data matrix y_train (:obj:`ndarray`): | The classification target for training y_test (:obj:`ndarray`): | The classification target for testing experiment (:obj:`neptune.experiments.Experiment`, optional, default is ``None``): | Neptune ``Experiment`` object to control to which experiment you log the data. | If ``None``, log to currently active, and most recent experiment. Returns: ``None`` Examples: .. code:: python3 rfc = RandomForestClassifier() rfc.fit(X_train, y_train) neptune.init('my_workspace/my_project') exp = neptune.create_experiment() log_roc_auc_chart(rfc, X_train, X_test, y_train, y_test, experiment=exp) """ assert is_classifier( classifier), 'classifier should be sklearn classifier.' exp = _validate_experiment(experiment) try: fig, ax = plt.subplots() visualizer = ROCAUC(classifier, is_fitted=True, ax=ax) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) visualizer.finalize() exp.log_image('charts_sklearn', fig, image_name='ROC-AUC') plt.close(fig) except Exception as e: print('Did not log ROC-AUC chart. Error {}'.format(e))
def classification_sanity_check(model, X_train, X_test, y_train, y_test, classes=None): visualizer = ROCAUC(model, micro=False, macro=False, classes=classes) visualizer.fit(X_train, y_train) # Fit the training data to the visualizer visualizer.score(X_test, y_test) # Evaluate the model on the test data visualizer.poof()
def roc_curve(self, classes) -> None: visualizer = ROCAUC(self.trained_model, classes=classes) visualizer.fit(self.X_train, self.y_train) # Fit the training data to the visualizer visualizer.score(self.X_test, self.y_test) # Evaluate the model on the test data save_dir = f"{self.plots_dir}/roc_curve_{self.model_id}.png" visualizer.show(outpath=save_dir) if not LOCAL: upload_to_s3(save_dir, f'plots/roc_curve_{self.model_id}.png', bucket=S3_BUCKET_NAME) plt.clf()
def create_roc_auc_chart(classifier, X_train, X_test, y_train, y_test): """Create ROC-AUC chart. Tip: Check Sklearn-Neptune integration `documentation <https://docs-beta.neptune.ai/essentials/integrations/machine-learning-frameworks/sklearn>`_ for the full example. Args: classifier (:obj:`classifier`): | Fitted sklearn classifier object X_train (:obj:`ndarray`): | Training data matrix X_test (:obj:`ndarray`): | Testing data matrix y_train (:obj:`ndarray`): | The classification target for training y_test (:obj:`ndarray`): | The classification target for testing Returns: ``neptune.types.File`` object that you can assign to run's ``base_namespace``. Examples: .. code:: python3 import neptune.new.integrations.sklearn as npt_utils rfc = RandomForestClassifier() rfc.fit(X_train, y_train) run = neptune.init(project='my_workspace/my_project') run['visuals/roc_auc'] = npt_utils.create_roc_auc_chart(rfc, X_train, X_test, y_train, y_test) """ assert is_classifier( classifier), 'classifier should be sklearn classifier.' chart = None try: fig, ax = plt.subplots() visualizer = ROCAUC(classifier, is_fitted=True, ax=ax) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) visualizer.finalize() chart = neptune.types.File.as_image(fig) plt.close(fig) except Exception as e: print('Did not log ROC-AUC chart. Error {}'.format(e)) return chart
def rocauc(X, y, model, outpath, **kwargs): # Create a new figure and axes _, ax = plt.subplots() # Instantiate the classification model and visualizer visualizer = ROCAUC(model, ax=ax, **kwargs) # Create the train and test data X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) # Save to disk visualizer.poof(outpath=outpath)
def evaluate_model(clf_, X_tr, X_te, y_tr, y_te, cls_rpt_tr=False, show=True, cls_labels=None, binary=False): """Takes any classifier, train/test data for X/y, labels for graph (optional). Will output (if show) a Sklearn Classification Report and Confusion Matrix along with a Yellowbrick ROC/AUC curve and Feature Importance graph (if a tree). Otherwise will return training/test predictions.""" import sklearn.metrics as metrics import matplotlib.pyplot as plt from yellowbrick.classifier import ROCAUC ## Fit and predict y_hat_trn, y_hat_tes = fit_n_pred(clf_, X_tr, X_te, y_tr) if show: ## Classification Report / Scores if cls_rpt_tr: print('Classification Report Train') print(metrics.classification_report(y_tr,y_hat_trn)) else: print('Classification Report Test') print(metrics.classification_report(y_te,y_hat_tes)) ## Confusion Matrix fig, ax = plt.subplots(figsize=(10,5), ncols=2) metrics.plot_confusion_matrix(clf_,X_te,y_te,cmap="YlOrRd", normalize='true',ax=ax[0]) ax[0].set(title='Confusion Matrix Test Data') ax[0].grid(False) roc = ROCAUC(clf_, classes=cls_labels, ax=ax[1]) roc.fit(X_tr, y_tr) roc.score(X_te, y_te) roc.finalize() plt.tight_layout() plt.show() if binary: try: imps = plot_importance(clf_, X_tr) except: imps = None else: return y_hat_trn, y_hat_tes
def generate_roc_auc(self, X_train, y_train, X_test, y_test, **kwargs): """ Given the training and testing sets, computes the ROC AUC metrics for the given model and returns a ROC AUC plotly figure. :param X_train: the training feature set. :param y_train: the training target set. :param X_test: the testing feature set. :param y_test: the testing target set. """ visualizer = ROCAUC(self.model, classes=self.classes) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) roc_data = visualizer.tpr layout = go.Layout(yaxis = dict( scaleratio = 1, range=[-.1, 1] ), xaxis=dict( range=[-.1, 1], #scaleratio = 1 ), ) fig = go.Figure(layout=layout) for tr in roc_data.keys(): trace = go.Scatter( x = [i / len(roc_data[tr]) for i in range(len(roc_data[tr]))], y = roc_data[tr], name = f'{tr}' if type(tr) != int else f'{self.classes[tr]}', line = dict(shape = 'hv') ) fig.add_trace(trace) lin_line = go.Scatter( x = [0,1], y = [0,1], name = 'linear_line', line = dict(dash='dash') ) fig.add_trace(lin_line) return fig
def rocauc(dataset): if dataset == "binary": X, y = load_occupancy() model = GaussianNB() elif dataset == "multiclass": X, y = load_game() X = OrdinalEncoder().fit_transform(X) model = RidgeClassifier() else: raise ValueError("uknown dataset") X_train, X_test, y_train, y_test = tts(X, y, test_size=0.2) oz = ROCAUC(model, ax=newfig()) oz.fit(X_train, y_train) oz.score(X_test, y_test) savefig(oz, "rocauc_{}".format(dataset))
def evaluate_model(model,x_test,y_test,coef_): prediction = model.predict(x_test) acc_test = accuracy_score(y_test,prediction) acc_train = accuracy_score(y_train,model.predict(X_train)) print("\n") print("Accuracy Score(Test): " + str(acc_test)) print("Accuracy Score(Train): " + str(acc_train)) print("Difference between train and test accuracy = {0}".format(abs(acc_test-acc_train))) print("Roc Auc Score: "+ str(roc_auc_score(y_test,prediction))) print("\n") print("Classification Report:") print(classification_report(y_test,prediction)) # confusion matrix plt.figure() cm = confusion_matrix(y_test,prediction) sns.heatmap(cm,annot=True,cmap="YlGnBu",fmt="d") plt.title("Confusion Matrix(1:Churned, 0:Not Churned)") plt.show() # roc-curve plt.figure() visualizer = ROCAUC(model, classes=["Not Churn", "Churn"]) visualizer.fit(X_train, y_train) # Fit the training data to the visualizer visualizer.score(X_test, y_test) # Evaluate the model on the test data visualizer.show() # Finalize and show the figure plt.show() if coef_: feature_imp ={} for idx,col_name in enumerate(X_train.columns): feature_imp[col_name] = model.coef_[0][idx] feature_imp = pd.DataFrame(feature_imp.items(),columns = ["Feature","Feature Importance"]) feature_imp.set_index("Feature",inplace=True) ax = feature_imp.plot(kind="bar",fontsize=10,color="red") ax.set_title("Future Importance",fontdict={"fontsize":12,"fontweight":"bold"}) ax.set_ylabel("Coef_") plt.show()
def rocauc(ax): from yellowbrick.classifier import ROCAUC from sklearn.linear_model import LogisticRegression # Specify the features of interest and the classes of the target features = ["temperature", "relative humidity", "light", "C02", "humidity"] target = "occupancy" classes = ['unoccupied', 'occupied'] # Load the data splits = load_data('occupancy', cols=features, target=target, tts=True) X_train, X_test, y_train, y_test = splits estimator = LogisticRegression() visualizer = ROCAUC(estimator, ax=ax) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) return visualizer
def showROC(): # Load the classification data set data = load_data('occupancy') # Specify the features of interest and the classes of the target features = ["temperature", "relative humidity", "light", "C02", "humidity"] classes = ['unoccupied', 'occupied'] # Extract the numpy arrays from the data frame X = data[features].as_matrix() y = data.occupancy.as_matrix() # Create the train and test data X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) # Instantiate the classification model and visualizer logistic = LogisticRegression() visualizer = ROCAUC(logistic) visualizer.fit(X_train, y_train) # Fit the training data to the visualizer visualizer.score(X_test, y_test) # Evaluate the model on the test data g = visualizer.poof() # Draw/show/poof the data
def plot(X, Y): X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2) oz = ROCAUC(GaussianNB()) oz.fit(X_train, y_train) oz.score(X_test, y_test) oz.poof()
y_pred = model.predict(X_test) # Making the Confusion Matrix from sklearn.metrics import confusion_matrix import seaborn as sn import matplotlib.pyplot as plt cmET = confusion_matrix(y_test, y_pred) sn.heatmap(cmET, cmap='Blues_r', annot=True, xticklabels='1234', yticklabels='1234') plt.xlabel('Predicted Label') plt.ylabel('Actual Label') plt.show() #ROCAUC Plot from sklearn.preprocessing import OrdinalEncoder, LabelEncoder from yellowbrick.classifier import ROCAUC # Encode the non-numeric columns X = OrdinalEncoder().fit_transform(X) y = LabelEncoder().fit_transform(y) # Instaniate the classification model and visualizer visualizer = ROCAUC(model, Damage=[1, 2, 3, 4]) visualizer.fit(X_train, y_train) # Fit the training data to the visualizer visualizer.score(X_test, y_test) # Evaluate the model on the test data visualizer.show() # Finalize and render the figure
####################################################### # STEP 5: Accuracy check ######################################################### from sklearn import metrics prediction_test = model.predict(X_test) ##Check accuracy on test dataset. print ("Accuracy = ", metrics.accuracy_score(y_test, prediction_test)) from yellowbrick.classifier import ROCAUC print("Classes in the image are: ", np.unique(Y)) #ROC curve for RF roc_auc=ROCAUC(model, classes=[0, 1, 2, 3]) #Create object roc_auc.fit(X_train, y_train) roc_auc.score(X_test, y_test) roc_auc.show() ########################################################## #STEP 6: SAVE MODEL FOR FUTURE USE ########################################################### ##You can store the model for future use. In fact, this is how you do machine elarning ##Train on training images, validate on test images and deploy the model on unknown images. # # ##Save the trained model as pickle string to disk for future use model_name = "sandstone_model" pickle.dump(model, open(model_name, 'wb')) # ##To test the model on future datasets
datasets = DatasetMixin() credit = datasets.load_data('credit') credit_keys = credit.dtype.names datatype = credit.dtype[0] ncols = len(credit_keys) categorical_names = ['edu','married'] y_name = 'default' credit_data = None for j in range(0,ncols): if credit_keys[j] in categorical_names: credit_data = add_categorical(credit_data,credit[credit_keys[j]],datatype) elif credit_keys[j] == y_name: y = credit[y_name].astype(int) else: credit_data = add_column(credit_data,credit[credit_keys[j]]) datashape = credit_data.shape nrows = datashape[0] cmeans = np.mean(credit_data,0) repmeans = numpy.matlib.repmat(cmeans,nrows,1) mydata = credit_data - repmeans sstds = np.std(mydata,0) repstds = numpy.matlib.repmat(sstds,nrows,1) mydata = np.divide(mydata,repstds) visualizer = ROCAUC(LinearSVC()) visualizer.fit(mydata,y) visualizer.score(mydata,y) visualizer.poof()
def train(experiment_id, run_name, xtrain, xtest, ytrain, ytest): np.random.seed(100) with mlflow.start_run(experiment_id=experiment_id, run_name=run_name) as run: tfid_vect =TfidfVectorizer(analyzer='word', tokenizer=nltk.tokenize.word_tokenize, stop_words='english', min_df=5) my_pipeline = Pipeline(steps=[('vectorizer', tfid_vect), ('lr', LogisticRegression(random_state=42))]) my_pipeline.fit(xtrain, ytrain) predictions = my_pipeline.predict(xtest) joblib.dump(my_pipeline, 'pipeline_lr.pkl') accuracy = accuracy_score(ytest, predictions) f1score = f1_score(ytest, predictions) auc_score = roc_auc_score(ytest, predictions) class_report = classification_report(ytest, predictions) print(f'Accuracy : {round(accuracy, 2)}') print(f'f1_score : {round(f1score, 2)}') print(f'auc_score : {round(auc_score, 2)}') print(f'class_report : \n {class_report}') mlflow.log_metric('Accuracy', round(accuracy, 2)) mlflow.log_metric('f1_score', round(f1score, 2)) mlflow.log_metric('auc_score', round(auc_score, 2)) fig, (ax1, ax2, ax3, ax4) = plt.subplots(nrows=4) visualizer = ClassificationReport(my_pipeline, ax=ax1, classes=[0,1]) visualizer.fit(xtrain, ytrain) visualizer.score(xtest, ytest) a=visualizer.poof(outpath="image/classification_report.png") print(' ') mlflow.log_artifact("image/classification_report.png") # The ConfusionMatrix visualizer taxes a model cm = ConfusionMatrix(my_pipeline, ax=ax2, classes=[0,1]) cm.fit(xtrain, ytrain) cm.score(xtest, ytest) b=cm.poof(outpath="image/confusionmatrix.png") mlflow.log_artifact("image/confusionmatrix.png") print(' ') vis = ROCAUC(my_pipeline, ax=ax3, classes=[0,1]) vis.fit(xtrain, ytrain) # Fit the training data to the visualizer vis.score(xtest, ytest) # Evaluate the model on the test data c = vis.poof(outpath="image/rocauc.png") # Draw/show/poof the data print(' ') mlflow.log_artifact("image/rocauc.png") visual = ClassPredictionError(my_pipeline, ax=ax4, classes=[0,1]) visual.fit(xtrain, ytrain) visual.score(xtest, ytest) g = visual.poof(outpath="image/ClassificationError.png") print(' ') mlflow.log_artifact("image/ClassificationError.png") return run.info.run_uuid
# %% classification_report(clf, X, y) # %% visualizer = ROCAUC(clf, classes=class_names) visualizer.score(X, y) visualizer.poof() # %% visualizer = ClassPredictionError(clf, classes=class_names) visualizer.score(X, y) visualizer.poof() # %% visualizer = DiscriminationThreshold(clf) visualizer.fit(X, y) visualizer.poof() # %% keep = [263, 268, 287, 288, 300, 302, 307, 308, 313, 315] # %% seed = 15 test_size = 0.33 Xt, Xv, yt, yv = \ sklearn.model_selection.train_test_split( X[keep], y, test_size=test_size, stratify=y, random_state=seed) # %% explainer = shap.TreeExplainer(clf) shap_values = explainer.shap_values(Xv)
#Note the low accuracy for the important class (201 label) #Right metric is ROC AUC #Starting version 0.23.1 you can report this for multilabel problems. #https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html from sklearn.metrics import roc_auc_score #Version 0.23.1 of sklearn print("ROC_AUC score for imbalanced data is:") print(roc_auc_score(y_test, prediction_test_RF)) #https://www.scikit-yb.org/en/latest/api/classifier/rocauc.html from yellowbrick.classifier import ROCAUC roc_auc = ROCAUC(model_RF) #Create object roc_auc.fit(X_train, y_train) roc_auc.score(X_test, y_test) roc_auc.show() ############################################################################# # Handling Imbalanced data ########################################### # Technique 2 Up-sample minority class from sklearn.utils import resample print(df['Label'].value_counts()) #Separate majority and minority classes df_majority = df[df['Label'] == 1] df_minority = df[df['Label'] == 2]
plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic') plt.legend(loc="lower right") plt.savefig('Log_ROC') plt.show() # Yellowbrick visualization viz = ClassificationReport(model) viz.fit(X_train, y_train) viz.score(X_test, y_test) viz.show() roc = ROCAUC(model) roc.fit(X_train, y_train) roc.score(X_test, y_test) roc.show() # # Perform Gridsearch using various parameters penalty = ['l1', 'l2'] C = [0.1, 0.25, 0.5] solver = ['newton-cg', 'lbfgs', 'liblinear'] class_weight = [{0: 15, 1: 85}, {0: 0.14, 1: 0.86}] miter = [5000] hyperparams = dict(C=C, penalty=penalty, solver=solver, class_weight=class_weight, max_iter=miter) RegLog = LogisticRegression() finalmodel = GridSearchCV(RegLog, hyperparams, verbose=0, cv=5)
classes=classes, label_encoder={ 0: 'benign', 1: 'malignant' }) conf_matrix.fit(X_train, y_train) conf_matrix.score(X_test, y_test) conf_matrix.poof() ### ROC-AUC from yellowbrick.classifier import ROCAUC visualizer = ROCAUC(LogisticRegression(), classes=classes) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) visualizer.poof() ### Class Prediction Error from yellowbrick.classifier import ClassPredictionError visualizer = ClassPredictionError(LogisticRegression(), classes=classes) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) visualizer.poof() ### Discrimination Threshold
test_mean = np.mean(test_scores, axis=1) test_mean test_std = np.std(test_scores, axis=1) test_std plt.plot(train_sizes, train_mean, label='Training Score') plt.plot(train_sizes, test_mean, label='Cross-Validation Score') plt.fill_between(train_sizes, train_mean - train_std, train_mean + train_std, color='#DDDDDD') plt.fill_between(train_sizes, test_mean - test_std, test_mean + test_std, color='#DDDDDD') plt.title("Learning Curve") plt.xlabel("Training Size") plt.ylabel("Accuracy Score") plt.legend(loc='best') """**ROC ve AUC**""" from yellowbrick.classifier import ROCAUC fig, ax = plt.subplots(1, 1, figsize=(12, 8)) roc_auc = ROCAUC(clf, ax=ax) roc_auc.fit(pc_train, y_train) roc_auc.score(pc_test, y_test) roc_auc.poof()
knn_result = pd.DataFrame(knn_result.cv_results_) knn_result.to_csv('knn_preprocessed.csv') print('knn process done') """ ################################################################### # Evaluation ################################################################### #Random_Forest Evaluation rf_best = RandomForestClassifier(criterion="entropy", bootstrap=False,max_depth=10) rf_best.fit(X,Y) print(confusion_matrix(Y_test,rf_best.predict(X_test))) plt.figure(figsize=(2, 2)) sns.heatmap(metrics.confusion_matrix(Y_test,rf_best.predict(X_test)), annot=True, fmt='.2f', linewidths=.1, cmap='Blues') visualizer = ROCAUC(rf_best, classes=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], micro=False, macro=True, per_class=False) visualizer.fit(X, Y) visualizer.score(X, Y) visualizer.show() #Bagging Classifier Evaluation bagging_best = BaggingClassifier(bootstrap=True, n_estimators=16) bagging_best.fit(X,Y) print(confusion_matrix(Y_test,bagging_best.predict(X_test))) plt.figure(figsize=(2, 2)) sns.heatmap(metrics.confusion_matrix(Y_test,bagging_best.predict(X_test)), annot=True, fmt='.2f', linewidths=.1, cmap='Blues') visualizer = ROCAUC(bagging_best, classes=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], micro=False, macro=True, per_class=False) visualizer.fit(X, Y) visualizer.score(X, Y) visualizer.show()
def best_model(model): visualizer = ROCAUC(model, classes=['reggae','soca','dancehall','pop']) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) visualizer.show()