def mini_batch_SGD(eta, batch_size, epochs):
MB_start_time = time.time()
np.random.seed(seed)
beta_init = np.random.randn(X_train.shape[1], 1)
w1 = Weight(X_train, y_train, beta_init, eta, epochs, batch_size=batch_size)
final_betas_MB, _ = w1.train(w1.mini_batch_gradient_descent)
prob_MB, y_pred_MB = classification(X_test, final_betas_MB, y_test)[0:2]
false_pos_MB, true_pos_MB = roc_curve(y_test, prob_MB)[0:2]
AUC_MB = auc(false_pos_MB, true_pos_MB)
print("Area under curve MB%s: " %batch_size, AUC_MB)
MB_time = time.time() - MB_start_time
return AUC_MB, MB_time, false_pos_MB, true_pos_MB
def Best_Parameters(epochs, batch_size, method, etamin, etamax, step, y_val, X_val):
beta_init = np.random.randn(X_train.shape[1], 1)
eta_vals = np.logspace(etamin, etamax, step)
auc_array = np.zeros((2, step))
for i, eta in enumerate(eta_vals):
np.random.seed(seed)
print("Iteration: ",i)
print("eta: ", eta)
w = Weight(X_train, y_train, beta_init, eta, epochs, batch_size=batch_size)
method_ = getattr(w, method)
final_betas, _ = w.train(method_)
prob = sigmoid(X_val, final_betas)
auc_array[0][i] = roc_auc_score(y_val, prob)
auc_array[1][i] = eta
max_auc = np.max(auc_array[0])
best_eta = auc_array[1][np.argmax(auc_array[0])]
return max_auc, best_eta
def Plots(epochs, AUC_time_plot = 0, ROC_plot = 0, Lift_plot_test_NN = 0, Lift_plot_train_NN = 0, GD_plot = 0, MB_GD_plot = 0, Stoch_GD_plot = 0,
Newton_plot = 0, Scatter_GD_plot = 0):
if (ROC_plot == 1 or AUC_time_plot == 1):
GRAD_start_time = time.time()
np.random.seed(seed)
beta_init = np.random.randn(X_train.shape[1],1)
w = Weight(X_train,y_train,beta_init,6.892612104349695e-05, epochs)
final_betas_grad,cost = w.train(w.gradient_descent)
prob_grad, y_pred_grad = classification(X_test, final_betas_grad, y_test)[0:2]
false_pos_grad, true_pos_grad = roc_curve(y_test, prob_grad)[0:2]
AUC_GRAD = auc(false_pos_grad, true_pos_grad)
print("Area under curve gradient: ", AUC_GRAD)
GRAD_time = time.time() - GRAD_start_time
SGD_start_time = time.time()
np.random.seed(seed)
beta_init = np.random.randn(X_train.shape[1], 1)
w2 = Weight(X_train, y_train, beta_init, 0.0007924828983539169, epochs)
final_betas_ST, _ = w2.train(w2.stochastic_gradient_descent)
prob_ST, y_pred_ST = classification(X_test, final_betas_ST, y_test)[0:2] ### HERE
false_pos_ST, true_pos_ST = roc_curve(y_test, prob_ST)[0:2]
AUC_SGD = auc(false_pos_ST, true_pos_ST)
print("Area under curve ST: ", AUC_SGD)
SGD_time = time.time() - SGD_start_time
"""np.random.seed(seed)
beta_init = np.random.randn(X_train.shape[1],1)
w3 = Weight(X_train,y_train,beta_init,0.001, 20)
final_betas_Newton,_ = w3.train(w3.newtons_method)
prob_Newton, y_pred_Newton = classification(X_train,final_betas_Newton, y_test)[0:2]
false_pos_Newton, true_pos_Newton = roc_curve(y_test, prob_Newton)[0:2]
print("Area under curve Newton: ", auc(false_pos_Newton, true_pos_Newton))"""
AUC_MB5 = 0
MB5_time = 0
AUC_MB1000 = 0
MB1000_time = 0
AUC_MB6000 = 0
MB6000_time = 0
AUC_MB = 0
false_pos_MB = 0
true_pos_MB = 0
if(AUC_time_plot != 0):
AUC_MB5, MB5_time, _, _ = mini_batch_SGD(0.0038625017292608175, 5, epochs)
AUC_MB1000, MB1000_time, _, _ = mini_batch_SGD(0.0009501185073181439, 1000, epochs)
AUC_MB6000, MB6000_time, _ ,_ = mini_batch_SGD(0.0001999908383831537, 6000, epochs)
return AUC_SGD, AUC_GRAD, AUC_MB5, AUC_MB1000, AUC_MB6000, SGD_time, GRAD_time, MB5_time, MB1000_time, MB6000_time
else:
AUC_MB, _,false_pos_MB, true_pos_MB = mini_batch_SGD(0.0038625017292608175, 32, epochs)
np.random.seed(seed)
beta_init = np.random.randn(X_train.shape[1], 1)
w4 = Weight(X_train, y_train, beta_init, 0.0007924828983539169, epochs)
final_betas_ST_Skl,_ = w.train(w4.stochastic_gradient_descent_Skl)
prob_ST_Skl, y_pred_ST_Skl = classification(X_test,final_betas_ST_Skl[0], y_test)[0:2]
false_pos_ST_Skl, true_pos_ST_Skl = roc_curve(y_test, prob_ST_Skl)[0:2]
print("Area under curve ST_skl: ", auc(false_pos_ST_Skl, true_pos_ST_Skl))
epochs = 20
batch_size = 25
eta = 0.1
lmbd = 0.01
n_hidden_neurons = 41
####################
# epochs = 20
# batch_size = 26
# eta = 3.14230708e+00
# lmbd = 1.25472709e-02
# n_hidden_neurons = 66
np.random.seed(seed)
n_categories = 1
dnn = NN(X_train, y_train, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
n_hidden_neurons=n_hidden_neurons, n_categories=n_categories,
cost_grad = 'crossentropy', activation = 'sigmoid', activation_out='sigmoid')
dnn.train_and_validate()
y_predict = dnn.predict_probabilities(X_test)
false_pos_NN, true_pos_NN = roc_curve(y_test, y_predict)[0:2]
print("AUC score NN: ", auc(false_pos_NN, true_pos_NN))
plt.plot([0, 1], [0, 1], "k--")
plt.plot(false_pos_grad, true_pos_grad,label="Gradient")
plt.plot(false_pos_ST, true_pos_ST, label="Stoch")
plt.plot(false_pos_ST_Skl, true_pos_ST_Skl, label="Stoch_Skl")
plt.plot(false_pos_MB, true_pos_MB, label="Mini")
# plt.plot(false_pos_Newton, true_pos_Newton, label="Newton")
plt.plot(false_pos_NN, true_pos_NN, label='NeuralNetwork')
plt.legend()
plt.xlabel("False Positive rate")
plt.ylabel("True Positive rate")
plt.title("ROC curve")
plt.show()
"""Creates cumulative gain charts/lift plots for Neural network. The two optimal parameters sets from tuning are listed below"""
if (Lift_plot_test_NN == 1):
np.random.seed(seed)
# epochs = 20
# batch_size = 26
# eta = 3.14230708e+00
# lmbd = 1.25472709e-02
# n_hidden_neurons = 66
epochs = 20
batch_size = 25
eta = 0.1
lmbd = 0.01
n_hidden_neurons = 41
n_categories = 1
dnn = NN(X_train, y_train, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
n_hidden_neurons=n_hidden_neurons, n_categories=n_categories,
cost_grad='crossentropy', activation='sigmoid', activation_out='sigmoid')
dnn.train_and_validate()
y_predict_proba = dnn.predict_probabilities(X_test)
y_predict_proba_tuple = np.concatenate((1 - y_predict_proba, y_predict_proba), axis=1)
pos_true = y_test.sum()
pos_true_perc = pos_true / len(y_test)
x = np.linspace(0, 1, len(y_test))
m = 1 / pos_true_perc
best_line = np.zeros((len(x)))
for i in range(len(x)):
best_line[i] = m * x[i]
if (x[i] > pos_true_perc):
best_line[i] = 1
x_, y_ = skplt.helpers.cumulative_gain_curve(y_test, y_predict_proba_tuple[:, 1])
Score = (np.trapz(y_, x=x_) - 0.5) / (np.trapz(best_line, dx=(1 / len(y_predict_proba))) - 0.5)
print('Area ratio score(test)', Score) # The score Area ratio = 0.49129354889528054 Neural Network test against predicted
perc = np.linspace(0, 100, len(y_test))
plt.plot(x_*100, y_*100)
plt.plot(perc, best_line*100)
plt.plot(perc, perc, "k--")
plt.xlabel("Percentage of clients")
plt.ylabel("Cumulative % of defaults")
plt.title("Cumulative Gain Chart for Test Data")
plt.show()
"""Let's you insert a threshold and classify"""
_, y_predict, y_predict_tot = classification(y_prob_input=y_predict_proba, threshold=0.5)
pos = y_predict.sum()
neg = len(y_predict) - pos
pos_perc = (pos / len(y_predict))
neg_perc = (neg / len(y_predict))
print("default: ", pos_perc)
print("Non-default: ", neg_perc)
if (Lift_plot_train_NN == 1):
np.random.seed(seed)
# epochs = 20
# batch_size = 26
# eta = 3.14230708e+00
# lmbd = 1.25472709e-02
# n_hidden_neurons = 66
epochs = 20
batch_size = 25
eta = 0.1
lmbd = 0.01
n_hidden_neurons = 41
n_categories = 1
dnn = NN(X_train, y_train, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
n_hidden_neurons=n_hidden_neurons, n_categories=n_categories,
cost_grad='crossentropy', activation='sigmoid', activation_out='sigmoid')
dnn.train_and_validate()
y_predict_proba = dnn.predict_probabilities(X_train)
y_predict_proba_tuple = np.concatenate((1 - y_predict_proba, y_predict_proba), axis=1)
pos_true = y_train.sum()
pos_true_perc = pos_true / len(y_train)
x = np.linspace(0, 1, len(y_train))
m = 1 / pos_true_perc
best_line = np.zeros((len(x)))
for i in range(len(x)):
best_line[i] = m * x[i]
if (x[i] > pos_true_perc):
best_line[i] = 1
x_, y_ = skplt.helpers.cumulative_gain_curve(y_train, y_predict_proba_tuple[:, 1])
Score = (np.trapz(y_, x=x_) - 0.5) / (np.trapz(best_line, dx=(1 / len(y_predict_proba))) - 0.5)
print('Area ratio score(train)', Score)
perc = np.linspace(0, 100, len(y_train))
plt.plot(x_ * 100, y_ * 100)
plt.plot(perc, best_line * 100)
plt.plot(perc, perc, "k--")
plt.xlabel("Percentage of clients")
plt.ylabel("Cumulative % of defaults")
plt.title("Cumulative Gain Chart for Train Data")
plt.show()
"""Let's you insert a threshold and classify"""
_, y_predict, y_predict_tot = classification(y_prob_input=y_predict_proba, threshold=0.5)
pos = y_predict.sum()
neg = len(y_predict) - pos
pos_perc = (pos / len(y_predict))
neg_perc = (neg / len(y_predict))
print("default: ", pos_perc)
print("Non-default: ", neg_perc)
beta_init = np.random.randn(X_train.shape[1], 1)
w = Weight(X_train, y_train, beta_init, 0.0007924828983539169, epochs)
if (GD_plot == 1):
_, cost_all = w.train(w.gradient_descent)
epoch = np.arange(len(cost_all))
plt.plot(epoch, cost_all)
plt.show()
if (MB_GD_plot == 1):
_, cost_all = w.train(w.mini_batch_gradient_descent)
batch = np.arange(len(cost_all))
plt.plot(batch, cost_all)
plt.show()
if (Stoch_GD_plot == 1):
_, cost_all = w.train(w.stochastic_gradient_descent)
batch = np.arange(len(cost_all))
plt.plot(batch, cost_all)
plt.show()
if (Newton_plot == 1):
_, cost_all = w.train(w.newtons_method)
epochs = np.arange(len(cost_all))
plt.plot(epochs, cost_all)
plt.show()
if (Scatter_GD_plot == 1):
final_betas, _ = w.train(w.gradient_descent)
prob_train = classification(X_train, final_betas)[0]
x_sigmoid = np.dot(X_train, final_betas)
plt.scatter(x_sigmoid, prob_train)
plt.show()
