def cross_validation(args):
for iter in range(1, args.num_models + 1):
args.lr = 10**np.random.uniform(-5, -1)
args.weight_decay = 10**np.random.uniform(-5, 1)
if args.modeltype in ['LR', 'MLP']:
args.factor = np.random.randint(90, 111)
model = models.LR(args) if args.modeltype == 'LR'\
else models.MLP(args)
print('{}, Model: {}, lr: {:.5f}, wd: {:.5f}, factor: {}'.format(
iter, args.modeltype, args.lr, args.weight_decay, args.factor))
f = open(args.filepath, 'a')
f.write('%d, Model: %s, lr: %.5f, wd: %.5f, factor: %d\n' %
(iter, args.modeltype, args.lr, args.weight_decay,
args.factor))
f.close()
elif args.modeltype in ['CNN', 'CNNDeep']:
args.embed_dim = np.random.randint(90, 111)
args.kernel_num = np.random.randint(90, 111)
args.dropout = np.random.uniform(0.1, 0.5)
model = models.CNN(args) if args.modeltype == 'CNN'\
else models.CNNDeep(args)
print('{}, Model: {}, lr: {:.5f}, wd: {:.5f}, embed_dim: {},\
kernel_num: {}, dropout: {:.5f}'.format(
iter, args.modeltype, args.lr, args.weight_decay,
args.embed_dim, args.kernel_num, args.dropout))
f = open(args.filepath, 'a')
f.write('%d, Model: %s, lr: %.5f, wd: %.5f, embed_dim: %d,\
kernel_num: %d, dropout: %.5f\n' %
(iter, args.modeltype, args.lr, args.weight_decay,
args.embed_dim, args.kernel_num, args.dropout))
f.close()
train_and_evaluate(args, model)
def final_run(args):
if args.modeltype == 'LR':
model = models.LR(args)
elif args.modeltype == 'MLP':
model = models.MLP(args)
elif args.modeltype == 'CNN':
model = models.CNN(args)
elif args.modeltype == 'CNNDeep':
model = models.CNNDeep(args)
train_and_evaluate(args, model)
def main(options=None):
pass
ft = Feature()
ft.fit()
print(ft.feature_names)
print("==============数据信息==============")
print(ft.describe())
print("--------------训练数据--------------")
print(ft.features_train.describe().transpose())
print("--------------测试数据--------------")
print(ft.features_test.describe().transpose())
ft.fillna()
models = [
# md.POKS(ft),
md.CF(feature=ft, kind='user_base'),
md.CF(feature=ft, kind='item_base'),
md.Irt(ft),
md.LR(ft),
md.AdaBoost(ft),
md.DecisionTree(ft),
md.GradientBoostingTree(ft),
md.RandomForest(ft),
md.Xgboost(ft),
]
for model in models:
model.fit()
model.test()
# model.evaluation.print_evaluate()
# model.evaluation.plot_auc()
ev = EvaluationMany(models)
print("==============模型效果==============")
print(ev.evaluate())
print(ev.evaluate_no())
# ev.plot_allin()
return ev
w = np.array([5, -10])
b = 50
X1 = np.random.normal(loc=0, scale=3.0, size=[100, 1]) # 数据集特征
X2 = np.zeros([len(X1), 1])
for i in range(len(X1)):
X2[i] = X1[i] * (-w[0] / w[1]) + (-b / w[1])
X2 += np.random.normal(loc=0, scale=0.5, size=[100, 1])
X0 = np.concatenate((X1, X2), axis=1)
Y = target_fun(X0) # 数据集的标签
#%%
X = model.add_one(X0)
w = np.random.normal(loc=0.0, scale=1.0, size=[3])
w[2] = 50
lr = model.LR(w)
w, loss_list, w_list = lr.fit(X, Y, silent=False, mask=[1, 1, 0])
print(w)
t = plt.plot(loss_list)
plt.show()
#%%
edge = []
x = []
y = []
z = []
l = len(w_list)
for i in range(0, l - 1):
edge.append([[w_list[i][0], w_list[i][1], loss_list[i] + 100],
[w_list[i + 1][0], w_list[i + 1][1], loss_list[i + 1] + 100]])
x.append(w_list[i][0])
def main():
utils.local_css("css/styles.css")
st.title("Heart Disease Prediction - Manual Parameter Tuner")
st.sidebar.title("Manual Parameter Tuning")
st.markdown("### Machine Learning is not only about the algorithms you use but also about the different Parameters "
"assigned to each of them. The final model is heavily affected by the parameters used for a specific "
"algorithm. "
"\nThis interactive web app will help you to explore the various parameters of different ML algorithms."
"\nThe different ML models presented here are:"
"\n* Logistic Regression"
"\n* Support Vector Classifier"
"\n* k-Nears Neighbour Classifier"
"\n* Decision Tree Classifier"
"\n* Random Forest Classifier"
"\n* Gradient Boosting Classifier"
"\n* XGBoost Classifier"
"\n### The dataset used here is the **Framingham** Coronary Heart Disease dataset publicly available "
"at [Kaggle](https://www.kaggle.com/amanajmera1/framingham-heart-study-dataset)."
"\n## About the Dataset:"
"\nThe **Framingham** dataset is from an ongoing cardiovascular study"
" on residents of the town of Framingham, Massachusetts. The classification goal is "
"to predict whether the patient has 10-year risk of future coronary heart disease (CHD). The dataset "
"provides the patients’ information. It includes over 4,240 records and 15 attributes."
"\n ### Even after optimizing parameters, the model would only work properly if accurate data is provided to it."
" So, through this web app, the users will be able to get a feel of hyperparameter tuning but only on this specific dataset."
"\n ## Head to the *Manual Parameter Tuning* section to get started!")
st.sidebar.markdown("Manually select the model you want to view and use the interactive text boxes, sliding bars "
"and buttons to tune the respective models. More than one options are provided for each model"
" and you can view and gain insight on how hyper-parameter tuning works. Enjoy exploring!")
data = pd.read_csv("Dataset/framingham.csv")
data = utils.preprocess(data)
st.sidebar.markdown("\n#### Exploratory Data Analysis:")
viz_list = st.sidebar.multiselect("(Be sure to Clear off all the selected options here before moving on for faster response)",
('Categorical Visualisation',
'Numerical Visualisation',
'sysBP and diaBP Visualisation'))
utils.visualize(viz_list, data)
if st.sidebar.checkbox("View raw and preprocessed data", False):
st.subheader("Raw-preprocessed Data")
st.write(data)
st.sidebar.markdown("\n#### Feature Selection:")
feature = st.sidebar.radio("Feature selection using chi-squared test", ("Don't select features",
"Select Features"), key='feature')
if feature == "Don't select features":
st.markdown("### Feature Selection is not done!")
else:
st.markdown("Best 10 features along with their chi-squared score")
score, data = utils.feature_selection(data)
st.write(score)
if st.sidebar.checkbox("Plot Feature Selection", False):
utils.plot_feature_selection(score)
train_x, test_x, train_y, test_y = utils.split_and_scale(data)
class_names = ["Has Heart Disease", "Doesn't have Heart Disease"]
st.sidebar.subheader("Choose Classifier")
classifier = st.sidebar.selectbox("Classifier", ("Logistic Regression",
"Support Vector Classifier",
"k-Nears Neighbour Classifier",
"Decision Tree Classifier",
"Random Forest Classifier",
"Gradient Boosting Classifier",
"XGBoost Classifier"))
if classifier == "Logistic Regression":
st.sidebar.subheader("Model Hyperparameters")
C = st.sidebar.number_input("C (Regularization parameter)", 0.01, 10.0, step=0.01, key='Lr')
max_iter = st.sidebar.slider("Maximum no. of Iterations", 100, 500, key='max_iter')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Logistic Regression Results")
y_pred, accuracy, models = model.LR(train_x, test_x, train_y, test_y, C=C, max_iter=max_iter)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "Support Vector Classifier":
st.sidebar.subheader("Model Hyperparameters")
C = st.sidebar.number_input("C (Regularization parameter)", 0.01, 10.0, step=0.01, key='C')
gamma = st.sidebar.radio("Gamma (for non linear hyperplanes)", ("auto", "scale"), key='gamma')
kernel = st.sidebar.radio("Kernel (type of hyperplane)", ("linear", "rbf", "poly"), key='kernel')
degree = 3
if kernel == 'poly':
degree = st.sidebar.number_input("Degree of the polynomial used to find the hyperplane", 1, 10, step=1,
key='degree')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Support Vector Classification Results")
y_pred, accuracy, models = model.SVM(train_x, test_x, train_y, test_y, C=C, gamma=gamma, kernel=kernel,
degree=degree)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "k-Nears Neighbour Classifier":
st.sidebar.subheader("Model Hyperparameters")
n = st.sidebar.number_input("n_neighbors (Number of nearest neighbors)", 1, 20, step=1, key='n')
leaf_size = st.sidebar.slider("Leaf Size", 10, 200, key='leaf_size')
algorithm = st.sidebar.radio("Algorithm to use", ("ball_tree", "kd_tree", "auto"), key='algorithm')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("kNN Classification Results")
y_pred, accuracy, models = model.KNN(train_x, test_x, train_y, test_y, n=n, leaf_size=leaf_size,
algorithm=algorithm)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "Decision Tree Classifier":
criterion = st.sidebar.radio("Criterion of splitting trees", ("gini", "entropy"), key='criterion')
max_depth = st.sidebar.slider("Max depth of the tree", 1, 50, key='amx_depth')
min_samples_leaf = st.sidebar.number_input("Minimum Leaf Samples", 1, 10, step=1, key='min_samples_leaf')
max_features = st.sidebar.radio("No. of features to consider during best split", ("auto", "sqrt", "log2"),
key='max_features')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Decision Tree Classification Results")
y_pred, accuracy, models = model.DT(train_x, test_x, train_y, test_y, criterion=criterion,
max_depth=max_depth,
leaf=min_samples_leaf, max_features=max_features)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "Random Forest Classifier":
st.sidebar.subheader("Model Hyperparameters")
n_estimators = st.sidebar.slider("Number of Trees in the Random Forest", 100, 4000, key='n_estimators')
max_depth = st.sidebar.number_input("The maximum depth of the tree", 1, 100, step=5, key='max_depth')
bootstrap = st.sidebar.radio("Bootstrap samples when building trees", ("True", "False"), key='bootstrap')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Random Forest Classification Results")
y_pred, accuracy, models = model.RF(train_x, test_x, train_y, test_y, n_estimators=n_estimators,
max_depth=max_depth, bootstrap=bootstrap)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "Gradient Boosting Classifier":
st.sidebar.subheader("Model Hyperparameters")
n_estimators = st.sidebar.slider("Number of Trees in the Gradient Boost ensemble", 100, 4000,
key='n_estimators')
max_depth = st.sidebar.number_input("The maximum depth of the tree", 1, 100, step=5, key='max_depth')
learning_rate = st.sidebar.number_input("Learning Rate", 0.01, 10.0, step=0.01, key='learning_rate')
warm_start = st.sidebar.radio("Reuse previous solution for more ensemble", ("True", "False"), key='warm_start')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Gradient Boosting Classification Results")
y_pred, accuracy, models = model.GBC(train_x, test_x, train_y, test_y, n_estimators=n_estimators,
max_depth=max_depth, learning_rate=learning_rate,
warm_start=warm_start)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)
if classifier == "XGBoost Classifier":
st.sidebar.subheader("Model Hyperparameters")
n_estimators = st.sidebar.slider("Number of Trees in the XGBoost ensemble", 100, 4000, key='n_estimators')
max_depth = st.sidebar.number_input("The maximum depth of the tree", 1, 100, step=5, key='max_depth')
eta = st.sidebar.number_input("Learning Rate", 0.01, 10.0, step=0.01, key='eta')
colsample_bytree = st.sidebar.number_input("Percentage of features used per tree", 0.01, 1.0, step=0.01,
key='colsample_bytree')
reg_alpha = st.sidebar.number_input("L1 regularization on leaf weights", 1, 10, step=1, key='reg_alpha')
reg_lambda = st.sidebar.number_input("L2 regularization on leaf weights", 1, 10, step=1, key='reg_lambda')
metrics = st.sidebar.multiselect("What matrix to plot?", ("Confusion Matrix", "ROC Curve",
"Precision-Recall Curve"))
if st.sidebar.button("Classify", key="classify"):
st.subheader("Extreme Gradient Boosting(XGBoost) Classification Results")
y_pred, accuracy, models = model.XGB(train_x, test_x, train_y, test_y, n_estimators=n_estimators,
max_depth=max_depth, eta=eta, colsample_bytree=colsample_bytree,
reg_alpha=reg_alpha, reg_lambda=reg_lambda)
st.write("Accuracy: ", accuracy.round(3))
st.write("Precision: ", precision_score(test_y, y_pred, labels=class_names).round(3))
st.write("Recall: ", recall_score(test_y, y_pred, labels=class_names).round(3))
utils.plot_metrics(metrics, models, test_x, test_y, class_names)