def test_data_split_identical():
X_, y_ = X[:40], np.array([0] * 20 + [1] * 20)
for md in [
model.SVM(),
model.KNN(),
model.XGBoost(),
model.LinearModel(),
model.DecisionTree()
]:
a = evaluation.estimate(md, X_train, X_test, y_train, y_test)
b = evaluation.estimate(md, X_train, X_test, y_train, y_test)
assert a == b
a = evaluation.cross_validation(md,
X_,
y_,
scoring='both',
n_splits=2,
n_jobs=1)
b = evaluation.cross_validation(md,
X_,
y_,
scoring='both',
n_splits=2,
n_jobs=1)
assert np.all(a['f1'] == b['f1'])
assert np.all(a['roc_auc'] == b['roc_auc'])
def test_estimate():
for md in [
model.SVM(),
model.KNN(),
model.XGBoost(),
model.LinearModel(),
model.DecisionTree()
]:
evaluation.estimate(md, X_train, X_test, y_train, y_test)
def test_clone():
models = [
model.LinearModel(),
model.SVM(),
model.DecisionTree(),
model.MultiClassesLearner('KNN', {'n_neighbors': 1}),
model.KNN(),
model.XGBoost(),
]
from sklearn.base import clone
for md in models:
clone(md)
def test_cross_validation():
for md in [
model.SVM(),
model.MultiClassesLearner('KNN', {'n_neighbors': 1}),
model.KNN(),
model.XGBoost(),
model.LinearModel(),
model.DecisionTree(),
]:
try:
evaluation.cross_validation(md,
X,
y,
scoring='both',
n_jobs=1,
n_splits=2)
except Exception as e:
print(md.__class__)
raise e
def test_get_params():
models = [
model.LinearModel(),
model.SVM(),
model.DecisionTree(),
model.MultiClassesLearner('KNN', {'n_neighbors': 1}),
model.KNN(),
model.XGBoost(),
]
for md in models:
params = md.get_params()
try:
assert 'normalizer_name' in params
assert 'sample_method' in params
if not isinstance(md, model.KNN):
assert 'balanced_learning' in params
except AssertionError as e:
print(params)
print(md.__class__)
raise e
def main():
utils.local_css("css/styles.css")
st.title("Heart Disease Prediction - Manual Parameter Tuner")
st.sidebar.title("Manual Parameter Tuning")
st.markdown("### Machine Learning is not only about the algorithms you use but also about the different Parameters "
"assigned to each of them. The final model is heavily affected by the parameters used for a specific "
"algorithm. "
"\nThis interactive web app will help you to explore the various parameters of different ML algorithms."
"\nThe different ML models presented here are:"
"\n* Logistic Regression"
"\n* Support Vector Classifier"
"\n* k-Nears Neighbour Classifier"
"\n* Decision Tree Classifier"
"\n* Random Forest Classifier"
"\n* Gradient Boosting Classifier"
"\n* XGBoost Classifier"
"\n### The dataset used here is the **Framingham** Coronary Heart Disease dataset publicly available "
"at [Kaggle](https://www.kaggle.com/amanajmera1/framingham-heart-study-dataset)."
"\n## About the Dataset:"
"\nThe **Framingham** dataset is from an ongoing cardiovascular study"
" on residents of the town of Framingham, Massachusetts. The classification goal is "
"to predict whether the patient has 10-year risk of future coronary heart disease (CHD). The dataset "
"provides the patients’ information. It includes over 4,240 records and 15 attributes."
"\n ### Even after optimizing parameters, the model would only work properly if accurate data is provided to it."
" So, through this web app, the users will be able to get a feel of hyperparameter tuning but only on this specific dataset."
"\n ## Head to the *Manual Parameter Tuning* section to get started!")
st.sidebar.markdown("Manually select the model you want to view and use the interactive text boxes, sliding bars "
"and buttons to tune the respective models. More than one options are provided for each model"
" and you can view and gain insight on how hyper-parameter tuning works. Enjoy exploring!")
data = pd.read_csv("Dataset/framingham.csv")
data = utils.preprocess(data)
st.sidebar.markdown("\n#### Exploratory Data Analysis:")
viz_list = st.sidebar.multiselect("(Be sure to Clear off all the selected options here before moving on for faster response)",
('Categorical Visualisation',
'Numerical Visualisation',
'sysBP and diaBP Visualisation'))
utils.visualize(viz_list, data)
if st.sidebar.checkbox("View raw and preprocessed data", False):
st.subheader("Raw-preprocessed Data")
st.write(data)
st.sidebar.markdown("\n#### Feature Selection:")
feature = st.sidebar.radio("Feature selection using chi-squared test", ("Don't select features",
"Select Features"), key='feature')
if feature == "Don't select features":
st.markdown("### Feature Selection is not done!")
else:
st.markdown("Best 10 features along with their chi-squared score")
score, data = utils.feature_selection(data)
st.write(score)
if st.sidebar.checkbox("Plot Feature Selection", False):
utils.plot_feature_selection(score)
train_x, test_x, train_y, test_y = utils.split_and_scale(data)
class_names = ["Has Heart Disease", "Doesn't have Heart Disease"]
st.sidebar.subheader("Choose Classifier")
classifier = st.sidebar.selectbox("Classifier", ("Logistic Regression",
"Support Vector Classifier",
"k-Nears Neighbour Classifier",
"Decision Tree Classifier",
"Random Forest Classifier",
"Gradient Boosting Classifier",
"XGBoost Classifier"))
if classifier == "Logistic Regression":
st.sidebar.subheader("Model Hyperparameters")
C = st.sidebar.number_input("C (Regularization parameter)", 0.01, 10.0, step=0.01, key='Lr')
max_iter = st.sidebar.slider("Maximum no. of Iterations", 100, 500, key='max_iter')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Logistic Regression Results")
y_pred, accuracy, models = model.LR(train_x, test_x, train_y, test_y, C=C, max_iter=max_iter)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "Support Vector Classifier":
st.sidebar.subheader("Model Hyperparameters")
C = st.sidebar.number_input("C (Regularization parameter)", 0.01, 10.0, step=0.01, key='C')
gamma = st.sidebar.radio("Gamma (for non linear hyperplanes)", ("auto", "scale"), key='gamma')
kernel = st.sidebar.radio("Kernel (type of hyperplane)", ("linear", "rbf", "poly"), key='kernel')
degree = 3
if kernel == 'poly':
degree = st.sidebar.number_input("Degree of the polynomial used to find the hyperplane", 1, 10, step=1,
key='degree')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Support Vector Classification Results")
y_pred, accuracy, models = model.SVM(train_x, test_x, train_y, test_y, C=C, gamma=gamma, kernel=kernel,
degree=degree)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "k-Nears Neighbour Classifier":
st.sidebar.subheader("Model Hyperparameters")
n = st.sidebar.number_input("n_neighbors (Number of nearest neighbors)", 1, 20, step=1, key='n')
leaf_size = st.sidebar.slider("Leaf Size", 10, 200, key='leaf_size')
algorithm = st.sidebar.radio("Algorithm to use", ("ball_tree", "kd_tree", "auto"), key='algorithm')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("kNN Classification Results")
y_pred, accuracy, models = model.KNN(train_x, test_x, train_y, test_y, n=n, leaf_size=leaf_size,
algorithm=algorithm)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "Decision Tree Classifier":
criterion = st.sidebar.radio("Criterion of splitting trees", ("gini", "entropy"), key='criterion')
max_depth = st.sidebar.slider("Max depth of the tree", 1, 50, key='amx_depth')
min_samples_leaf = st.sidebar.number_input("Minimum Leaf Samples", 1, 10, step=1, key='min_samples_leaf')
max_features = st.sidebar.radio("No. of features to consider during best split", ("auto", "sqrt", "log2"),
key='max_features')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Decision Tree Classification Results")
y_pred, accuracy, models = model.DT(train_x, test_x, train_y, test_y, criterion=criterion,
max_depth=max_depth,
leaf=min_samples_leaf, max_features=max_features)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "Random Forest Classifier":
st.sidebar.subheader("Model Hyperparameters")
n_estimators = st.sidebar.slider("Number of Trees in the Random Forest", 100, 4000, key='n_estimators')
max_depth = st.sidebar.number_input("The maximum depth of the tree", 1, 100, step=5, key='max_depth')
bootstrap = st.sidebar.radio("Bootstrap samples when building trees", ("True", "False"), key='bootstrap')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Random Forest Classification Results")
y_pred, accuracy, models = model.RF(train_x, test_x, train_y, test_y, n_estimators=n_estimators,
max_depth=max_depth, bootstrap=bootstrap)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "Gradient Boosting Classifier":
st.sidebar.subheader("Model Hyperparameters")
n_estimators = st.sidebar.slider("Number of Trees in the Gradient Boost ensemble", 100, 4000,
key='n_estimators')
max_depth = st.sidebar.number_input("The maximum depth of the tree", 1, 100, step=5, key='max_depth')
learning_rate = st.sidebar.number_input("Learning Rate", 0.01, 10.0, step=0.01, key='learning_rate')
warm_start = st.sidebar.radio("Reuse previous solution for more ensemble", ("True", "False"), key='warm_start')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Gradient Boosting Classification Results")
y_pred, accuracy, models = model.GBC(train_x, test_x, train_y, test_y, n_estimators=n_estimators,
max_depth=max_depth, learning_rate=learning_rate,
warm_start=warm_start)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "XGBoost Classifier":
st.sidebar.subheader("Model Hyperparameters")
n_estimators = st.sidebar.slider("Number of Trees in the XGBoost ensemble", 100, 4000, key='n_estimators')
max_depth = st.sidebar.number_input("The maximum depth of the tree", 1, 100, step=5, key='max_depth')
eta = st.sidebar.number_input("Learning Rate", 0.01, 10.0, step=0.01, key='eta')
colsample_bytree = st.sidebar.number_input("Percentage of features used per tree", 0.01, 1.0, step=0.01,
key='colsample_bytree')
reg_alpha = st.sidebar.number_input("L1 regularization on leaf weights", 1, 10, step=1, key='reg_alpha')
reg_lambda = st.sidebar.number_input("L2 regularization on leaf weights", 1, 10, step=1, key='reg_lambda')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Extreme Gradient Boosting(XGBoost) Classification Results")
y_pred, accuracy, models = model.XGB(train_x, test_x, train_y, test_y, n_estimators=n_estimators,
max_depth=max_depth, eta=eta, colsample_bytree=colsample_bytree,
reg_alpha=reg_alpha, reg_lambda=reg_lambda)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)