def test_on_parameters(n_hiddenLayers, n_epochs, showCost):
# Split data into train and test sections
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
test_size=test_ratio,
random_state=42)
print(X.shape, X_train.shape, X_test.shape)
#Train the network on train data
params = nn.train_nn(X_train,
Y_train,
n_hiddenLayers,
n_epochs,
adaptive_lRate,
showCost=showCost)
#Test neural network on test data
A2, cache = nn.f_propagate(X_test, params)
acc = "{:.4f}".format(nn.accuracy(A2, Y_test))
print('Accuracy on test data: ' + acc + '%')
#Test neural network on train data
A2, cache = nn.f_propagate(X_train, params)
acc = "{:.4f}".format(nn.accuracy(A2, Y_train))
print('Accuracy on train data: ' + acc + '%')
# #Test neural network on all data
A2, cache = nn.f_propagate(X, params)
acc = "{:.4f}".format(nn.accuracy(A2, Y))
print('Accuracy on all available data: ' + acc + '%')
def test_inc_and_dec():
plotX = []
plotY = []
plotZ = []
size = 10
incArr = np.linspace(1, 1.1, size)
decArr = np.linspace(0.5, 1, size)
for inc in range(0, size):
for dec in range(0, size):
acc_arr = []
adaptive_lRate = {
'InitialRate': 0.01,
'DecrementVar': decArr[dec],
'IncrementVar': incArr[inc],
'ErrorRatio': 1.04
}
for i in range(1, 4):
X_train, X_test, Y_train, Y_test = train_test_split(
X, Y, test_size=test_ratio, random_state=randint(1, 100))
params = nn.train_nn(X_train,
Y_train,
8,
500,
adaptive_lRate,
showCost=False)
pred_test, pred_cache = nn.f_propagate(X_test, params)
acc_test = nn.accuracy(pred_test, Y_test)
pred_train, train_cache = nn.f_propagate(X_train, params)
acc_train = nn.accuracy(pred_train, Y_train)
pred_all, all_cache = nn.f_propagate(X, params)
acc_all = nn.accuracy(pred_all, Y)
acc_arr.append((acc_test + acc_train + acc_all) / 3)
plotX.append(incArr[inc])
plotY.append(decArr[dec])
plotZ.append(np.mean(acc_arr)) #plotZ.append(acc_test)
print('Inc: %f - Dec: %f - Average Acc: %f' %
(incArr[inc], decArr[dec], np.mean(acc_arr)))
fig = plt.figure()
ax = Axes3D(fig)
ax.set_xlabel('Increment')
ax.set_ylabel('Decrement')
ax.set_zlabel('Accuracy')
surf = ax.plot_trisurf(plotX, plotY, plotZ, linewidth=0.1, cmap='summer')
plt.savefig('./plots/plot.png')
plt.show()
def gradient_descent_fakedata():
"""
optimizes neural network on fake datasets, used for debugging
:return:
"""
(x_train, y_train, l_train) = dp.generate_x_y()
weights = np.ndarray(conf.LAYERS_NUM - 1, dtype=np.matrix)
bias = np.ndarray(conf.LAYERS_NUM - 1, dtype=np.ndarray)
rand_init_weights(weights, bias)
for n in range(conf.NUM_EPOCHS):
cost, gradients = nn.cost_derivatives(x_train, y_train, weights, bias)
cost, gradients = nn.regularize(weights, cost, gradients)
print("Epoch %d, Cost %f" % (n, cost))
for l in range(conf.LAYERS_NUM - 1):
weights[l] -= conf.LEARNING_RATE * gradients[l][:, 1:]
bias[l] -= conf.LEARNING_RATE * np.squeeze(
np.asarray(gradients[l][:, 0]))
# cost from the last epoch
cost = np.float(0)
activations = nn.feed_forward(x_train, weights, bias)
for i in range(0, x_train.shape[0]):
cost += nn.cross_entropy(activations[conf.LAYERS_NUM - 1][:, i],
y_train[:, i])
cost /= x_train.shape[0]
for n in range(1, conf.LAYERS_NUM):
cost += conf.REG_CONST * np.sum(
np.multiply(weights[n - 1], weights[n - 1])) / 2
print("Epoch %d, Cost %f" % (conf.NUM_EPOCHS, cost))
train_accuracy = nn.accuracy(x_train, l_train, weights, bias)
print("Accuracy: %f" % train_accuracy)
def test_range__hidden_and_epochs(hidden_start, hidden_end, hidden_step,
epochs_start, epochs_end, epochs_step,
iterate_n):
plotX = []
plotY = []
plotZ = []
for hidden in range(hidden_start, hidden_end + 1, hidden_step):
for epoch in range(epochs_start, epochs_end + 1, epochs_step):
acc_arr = []
for i in range(1, iterate_n + 1):
X_train, X_test, Y_train, Y_test = train_test_split(
X, Y, test_size=test_ratio, random_state=randint(1, 100))
params = nn.train_nn(X_train,
Y_train,
hidden,
epoch,
const_lRate,
showCost=False)
pred_test, pred_cache = nn.f_propagate(X_test, params)
acc_test = nn.accuracy(pred_test, Y_test)
pred_train, train_cache = nn.f_propagate(X_train, params)
acc_train = nn.accuracy(pred_train, Y_train)
pred_all, all_cache = nn.f_propagate(X, params)
acc_all = nn.accuracy(pred_all, Y)
acc_arr.append((acc_test + acc_train + acc_all) / 3)
plotX.append(hidden)
plotY.append(epoch)
plotZ.append(np.mean(acc_arr)) #plotZ.append(acc_test)
print('Hidden: %i - Epoch: %i - Average Acc: %f' %
(hidden, epoch, np.mean(acc_arr)))
fig = plt.figure()
ax = Axes3D(fig)
ax.set_xlabel('Neurons in hidden layer')
ax.set_ylabel('Epochs')
ax.set_zlabel('Accuracy')
surf = ax.plot_trisurf(plotX, plotY, plotZ, linewidth=0.1, cmap='winter')
plt.savefig('./plots/plot.png')
plt.show()
def test_adaptive_lRate():
rateAcc = []
singleAcc = []
ratesArr = np.linspace(0.001, 0.2, 50)
for i in range(0, 50):
for j in range(1, 3):
adaptive_lRate = {
'InitialRate': ratesArr[i],
'DecrementVar': 0.7,
'IncrementVar': 1.05,
'ErrorRatio': 1.04
}
X_train, X_test, Y_train, Y_test = train_test_split(
X, Y, test_size=test_ratio, random_state=i * j)
params = nn.train_nn(X_train,
Y_train,
8,
500,
adaptive_lRate,
showCost=False)
pred_test, pred_cache = nn.f_propagate(X_test, params)
acc_test = nn.accuracy(pred_test, Y_test)
pred_train, train_cache = nn.f_propagate(X_train, params)
acc_train = nn.accuracy(pred_train, Y_train)
pred_all, all_cache = nn.f_propagate(X, params)
acc_all = nn.accuracy(pred_all, Y)
singleAcc.append(np.mean([acc_test, acc_train, acc_all]))
rateAcc.append(np.mean(singleAcc))
print('Error rate: %f, Acc: %f' % (ratesArr[i], np.mean(singleAcc)))
singleAcc = []
plt.plot(ratesArr, rateAcc)
plt.xlabel('Initial Rate')
plt.ylabel('Accuracy')
plt.savefig('./plots/plot.png')
plt.show()
def gradient_descent(weights, bias):
"""
optimizes neural network on actual datasets
:param weights: weights
:param bias: bias
:return: optimized weights and bias
"""
x_train, y_train, x_test, y_test, l_train, l_test = dp.load_datasets()
for n in range(conf.NUM_EPOCHS):
cost, gradients = nn.cost_derivatives(x_train, y_train, weights, bias)
cost, gradients = nn.regularize(weights, cost, gradients)
if n % conf.EVAL_INTERVAL == 0:
print("Epoch %d, Cost %f" % (n, cost))
train_accuracy = nn.accuracy(x_train, l_train, weights, bias)
test_accuracy = nn.accuracy(x_test, l_test, weights, bias)
print("Train Accuracy: %f" % train_accuracy)
print("Test Accuracy: %f" % test_accuracy)
for l in range(conf.LAYERS_NUM - 1):
weights[l] -= conf.LEARNING_RATE * gradients[l][:, 1:]
bias[l] -= conf.LEARNING_RATE * np.squeeze(
np.asarray(gradients[l][:, 0]))
# cost from the last epoch
cost = np.float(0)
activations = nn.feed_forward(x_train, weights, bias)
for i in range(0, x_train.shape[0]):
cost += nn.cross_entropy(activations[conf.LAYERS_NUM - 1][:, i],
y_train[:, i])
cost /= x_train.shape[0]
for n in range(1, conf.LAYERS_NUM):
cost += conf.REG_CONST * np.sum(
np.multiply(weights[n - 1], weights[n - 1])) / 2
print("Epoch %d, Cost %f" % (conf.NUM_EPOCHS, cost))
train_accuracy = nn.accuracy(x_train, l_train, weights, bias)
test_accuracy = nn.accuracy(x_test, l_test, weights, bias)
print("Train Accuracy: %f" % train_accuracy)
print("Test Accuracy: %f" % test_accuracy)
return weights, bias
def test_params(n_hidden, n_epochs, n_iterations):
acc_arr = []
lowest_test = lowest_train = lowest_all = 100
incidents = 0
for i in range(1, n_iterations + 1):
X_train, X_test, Y_train, Y_test = train_test_split(
X, Y, test_size=test_ratio, random_state=i)
params = nn.train_nn(X_train,
Y_train,
n_hidden,
n_epochs,
const_lRate,
showCost=False)
pred_test, pred_cache = nn.f_propagate(X_test, params)
acc_test = nn.accuracy(pred_test, Y_test)
pred_train, train_cache = nn.f_propagate(X_train, params)
acc_train = nn.accuracy(pred_train, Y_train)
pred_all, all_cache = nn.f_propagate(X, params)
acc_all = nn.accuracy(pred_all, Y)
if (acc_test < lowest_test): lowest_test = acc_test
if (acc_train < lowest_train): lowest_train = acc_train
if (acc_all < lowest_all): lowest_all = acc_all
if (acc_all < 90 or acc_train < 90 or acc_test < 90): incidents += 1
acc_arr.append((acc_test + acc_train + acc_all) / 3)
print('Iteration: %i, Test: %f, Train: %f, All: %f' %
(i, acc_test, acc_train, acc_all))
print('================================================')
print(
'Average accuracy for all iterations: %f, Number of incidents (<90 acc): %i'
% (np.mean(acc_arr), incidents))
print('Lowest accuracies >>> Test: %f, Train: %f, All: %f' %
(lowest_test, lowest_train, lowest_all))
def test_const_lRate(l_start, l_end):
rateAcc = []
singleAcc = []
ratesArr = np.linspace(l_start, l_end, 100)
for i in range(0, 100):
for j in range(1, 5):
X_train, X_test, Y_train, Y_test = train_test_split(
X, Y, test_size=test_ratio, random_state=i * j)
params = nn.train_nn(X_train,
Y_train,
8,
500,
ratesArr[i],
showCost=False)
pred_test, pred_cache = nn.f_propagate(X_test, params)
acc_test = nn.accuracy(pred_test, Y_test)
pred_train, train_cache = nn.f_propagate(X_train, params)
acc_train = nn.accuracy(pred_train, Y_train)
pred_all, all_cache = nn.f_propagate(X, params)
acc_all = nn.accuracy(pred_all, Y)
singleAcc.append(np.mean([acc_test, acc_train, acc_all]))
rateAcc.append(np.mean(singleAcc))
print('Learning_rate: %f, Acc: %f' % (ratesArr[i], np.mean(singleAcc)))
singleAcc = []
plt.plot(ratesArr, rateAcc)
plt.xlabel('Learning rate')
plt.ylabel('Accuracy')
plt.savefig('./plots/plot.png')
plt.show()
def compare_rates():
plotX = []
plotConst = []
plotAdapt = []
for epoch in range(0, 400, 10):
for j in range(1, 5):
singleConst = []
singleAdapt = []
X_train, X_test, Y_train, Y_test = train_test_split(
X, Y, test_size=test_ratio, random_state=randint(1, 100))
params_const = nn.train_nn(X_train,
Y_train,
8,
epoch,
const_lRate,
showCost=False)
params_adapt = nn.train_nn(X_train,
Y_train,
8,
epoch,
adaptive_lRate,
showCost=False)
pred_test_const, pred_cache = nn.f_propagate(X_test, params_const)
acc_test_const = nn.accuracy(pred_test_const, Y_test)
pred_train_const, train_cache = nn.f_propagate(
X_train, params_const)
acc_train_const = nn.accuracy(pred_train_const, Y_train)
pred_all_const, all_cache = nn.f_propagate(X, params_const)
acc_all_const = nn.accuracy(pred_all_const, Y)
pred_test_adapt, pred_cache = nn.f_propagate(X_test, params_adapt)
acc_test_adapt = nn.accuracy(pred_test_adapt, Y_test)
pred_train_adapt, train_cache = nn.f_propagate(
X_train, params_adapt)
acc_train_adapt = nn.accuracy(pred_train_adapt, Y_train)
pred_all_adapt, all_cache = nn.f_propagate(X, params_adapt)
acc_all_adapt = nn.accuracy(pred_all_adapt, Y)
singleConst.append(
np.mean([acc_test_const, acc_train_const, acc_all_const]))
singleAdapt.append(
np.mean([acc_test_adapt, acc_train_adapt, acc_all_adapt]))
plotX.append(epoch)
plotConst.append(np.mean(singleConst))
plotAdapt.append(np.mean(singleAdapt))
print('Epoch: %f, Const acc: %f, Adaptive acc: %f' %
(epoch, np.mean(singleConst), np.mean(singleAdapt)))
plt.plot(plotX, plotConst, label='Constant Learning Rate')
plt.plot(plotX, plotAdapt, label='Adaptive Learning Rate')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend(bbox_to_anchor=(0., 1.02, 1., .102),
loc='lower left',
ncol=2,
mode="expand",
borderaxespad=0.)
plt.savefig('./plots/plot.png')
plt.show()
num_iterations = 56000
learning_rate = 0.007
layers_dims = [X_train.shape[0], 40, 20, 10, 5, 1
] if args.activate == "sigmoid" else [
X_train.shape[0], 40, 20, 10, 5, 2
]
parameters = L_layer_model(X_train,
Y_train,
X_test,
Y_test,
layers_dims,
learning_rate,
num_iterations,
args.activate,
print_cost=args.verbose)
with open('parameters.pkl', 'wb') as output:
pickle.dump(parameters, output)
elif args.model == "predict":
try:
fp = open("parameters.pkl", "rb")
except Exception as e:
print("run train model on dataset to create parameters.pkl")
sys.exit(print("{}: {}".format(type(e).__name__, e)))
parameters = pickle.load(fp)
accuracy(Y_test, X_test, parameters, args.activate, "TestSet")
accuracy(Y_train, X_train, parameters, args.activate, "TrainSet")
X, Y = Preprocessing_predict(df, args.activate)
accuracy(Y, X, parameters, args.activate, "All DataSet")
X = X[rand,:]
y = y[rand]
# Since the output will be a probability for each class, we will
# put the class labels y into this format as well. y_mat will be a
# 1797 x 10 array where each row contains a one in the appropriate class
# column and zeros in all other columns.
y_mat = neural_network.y_to_mat(y,digits['target_names'].shape[0])
# Put the cutoff between the training set and the test set at 70%.
cutoff = int(.7 * len(y))
# Initialize a neural network with 25 hidden units.
n_network = neural_network.Network(X.shape[1], 25, y_mat.shape[1])
for i in range(10):
# Train the neural network on the training set using
# n_iterations more iterations of conjugate gradient.
n_iterations = 5
n_network.learn(X[:cutoff,:], y_mat[:cutoff,:], n_iterations = n_iterations,
lamb = .1, disp=False)
# Measure the neural network accuracy on the test set.
y_prob = n_network.predict(X[cutoff:,:])
y_pred = neural_network.mat_to_y(y_prob)
prediction_accuracy = neural_network.accuracy(y_pred, y[cutoff:])
# Print out the predication accuracy. Ideally, the accuracy will increase
# on each iteration of this for loop.
print ('{0:.2f}% accuracy on the test set after {1:2d} iterations of '
+ 'conjugate gradient.').format(
prediction_accuracy * 100, n_iterations * (1+i))