def test_neuralNetwork_adam():
from sklearn.neural_network._stochastic_optimizers import AdamOptimizer
np.random.seed(2019)
X = np.random.normal(size=(1, 500))
target = 3.9285985 * X
nn = NeuralNetwork(inputs=1,
neurons=3,
outputs=1,
activations='sigmoid',
silent=True)
nn.addLayer()
nn.addLayer()
nn.addOutputLayer(activations='identity')
learning_rate = 0.001
yhat = nn.forward_pass(X)
nn.backpropagation(yhat.T, target.T)
nn.learning_rate = learning_rate
nn.initializeAdam()
nn.adam()
skl_adam = AdamOptimizer(params=nn.param, learning_rate_init=learning_rate)
upd = skl_adam._get_updates(nn.grad)
for update_nn, update_skl in zip(nn.change, upd):
assert update_nn == pytest.approx(update_skl)
def main():
visualiseTrainingExamples()
nn = NeuralNetwork(config.numLayers, config.numClasses,
config.weightInitialisation, config.activationFn,
config.weightDecay)
nn.initialiseParams(len(x_train[0]) * len(x_train[0]), config.numNeurons)
sample = np.random.randint(3 * len(x_train) / 4)
nn.forwardPropagate(x_train[sample])
if config.optimizer == "sgd":
nn.stochasticGradDesc(x_train, y_train, config.maxIterations,
config.learningRate, config.batchSize)
elif config.optimizer == "momentum":
nn.momentumGradDesc(x_train, y_train, config.maxIterations,
config.learningRate, config.batchSize,
config.gamma)
elif config.optimizer == "nesterov":
nn.nesterovAcceleratedGradDesc(x_train, y_train, config.maxIterations,
config.learningRate, config.batchSize,
config.gamma)
elif config.optimizer == "rmsprop":
nn.rmsprop(x_train, y_train, config.maxIterations, config.learningRate,
config.batchSize, config.eps, config.beta1)
elif config.optimizer == "adam":
nn.adam(x_train, y_train, config.maxIterations, config.learningRate,
config.batchSize, config.eps, config.beta1, config.beta2)
elif config.optimizer == "nadam":
nn.nadam(x_train, y_train, config.maxIterations, config.learningRate,
config.batchSize, config.eps, config.beta1, config.beta2)
else:
print("No such optimizer available.")
predictions = []
predProbs = []
test_acc = 0
test_entropy = 0
test_mse = 0
for i in range(len(x_test)):
nn.forwardPropagate(x_test[i])
predictions.append(nn.predictedClass)
predProbs.append(nn.output[nn.predictedClass])
test_acc = accuracy(y_test, predictions)
test_entropy = crossEntropyLoss(y_test, predProbs)
test_mse = MSEloss(y_test, predictions)
predictions = []
predProbs = []
valid_acc = 0
valid_entropy = 0
valid_mse = 0
for i in range(len(x_valid)):
nn.forwardPropagate(x_valid[i])
predictions.append(nn.predictedClass)
predProbs.append(nn.output[nn.predictedClass])
valid_acc = accuracy(y_valid, predictions)
valid_entropy = crossEntropyLoss(y_valid, predProbs)
valid_mse = MSEloss(y_valid, predictions)
print(
f"Test Set:\nAccuracy = {test_acc}\nLoss = {test_entropy}\nMSE = {test_mse}"
)
print(
f"Validation Set:\nAccuracy = {valid_acc}\nLoss = {valid_entropy}\nMSE = {valid_mse}"
)
# #Log in wandb
metrics = {
'test_acc': test_acc,
# 'test_entropy': test_entropy,
"test_mse": test_mse,
'valid_acc': valid_acc,
# 'valid_entropy': valid_entropy,
"valid_mse": valid_mse
}
wandb.log(metrics)
run.finish()
