def test_neuralNetwork_sgd():
from sklearn.neural_network._stochastic_optimizers import SGDOptimizer
np.random.seed(2019)
X = np.random.normal(size=(1, 500))
target = 3.9285985 * X
nn = NeuralNetwork(inputs=1,
neurons=3,
outputs=1,
activations='sigmoid',
silent=True)
nn.addLayer()
nn.addLayer()
nn.addOutputLayer(activations='identity')
learning_rate = 0.001
yhat = nn.forward_pass(X)
nn.backpropagation(yhat.T, target.T)
nn.learning_rate = learning_rate
initial_params = copy.deepcopy(nn.weights + nn.biases)
nn.sgd()
grad = nn.d_weights + nn.d_biases
params = nn.weights + nn.biases
change = [p - i_p for p, i_p in zip(params, initial_params)]
skl_sgd = SGDOptimizer(params=initial_params,
learning_rate_init=learning_rate,
nesterov=False,
momentum=1.0)
upd = skl_sgd._get_updates(grad)
for update_nn, update_skl in zip(change, upd):
assert update_nn == pytest.approx(update_skl)
def test_neuralNetwork_adam():
from sklearn.neural_network._stochastic_optimizers import AdamOptimizer
np.random.seed(2019)
X = np.random.normal(size=(1, 500))
target = 3.9285985 * X
nn = NeuralNetwork(inputs=1,
neurons=3,
outputs=1,
activations='sigmoid',
silent=True)
nn.addLayer()
nn.addLayer()
nn.addOutputLayer(activations='identity')
learning_rate = 0.001
yhat = nn.forward_pass(X)
nn.backpropagation(yhat.T, target.T)
nn.learning_rate = learning_rate
nn.initializeAdam()
nn.adam()
skl_adam = AdamOptimizer(params=nn.param, learning_rate_init=learning_rate)
upd = skl_adam._get_updates(nn.grad)
for update_nn, update_skl in zip(nn.change, upd):
assert update_nn == pytest.approx(update_skl)
def test_predict(self, ):
target = NeuralNetwork()
target.addLayer(10, 10, lambda x: x**2)
prediction = target.predict(
tf.constant([1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
dtype=tf.dtypes.float32), [tf.zeros([10, 10])],
[tf.zeros([10])])
tf.debugging.assert_equal(prediction, tf.zeros(10))
def test_neuralNetwork_network(silent=False):
# Lets set up a sci-kit learn neural network and copy over the weights
# and biases to our network, verify that the two give the exact same
# result.
from sklearn.neural_network import MLPRegressor
X = [[0.0], [1.0], [2.0], [3.0], [4.0], [5.0]]
y = [0, 2, 4, 6, 8, 10]
mlp = MLPRegressor(solver='sgd',
alpha=0.0,
hidden_layer_sizes=(3, 3),
random_state=1,
activation='relu')
mlp.fit(X, y)
W_skl = mlp.coefs_
b_skl = mlp.intercepts_
nn = NeuralNetwork(inputs=1,
outputs=1,
layers=3,
neurons=3,
activations='relu',
silent=silent)
nn.addLayer()
nn.addLayer()
nn.addOutputLayer(activations='identity')
W_nn = nn.weights
b_nn = nn.biases
for i in range(len(W_nn)):
W_nn[i] = W_skl[i]
for i in range(len(b_nn)):
b_nn[i] = np.expand_dims(b_skl[i], axis=1)
X_test = np.array([[1.2857], [9.2508255], [-5.25255], [3.251095]])
output_skl = mlp.predict(X_test)
output_nn = np.squeeze(nn(X_test.T))
if not silent:
print("%20.15f %20.15f %20.15f %20.15f" % (*output_skl, ))
print("%20.15f %20.15f %20.15f %20.15f" % (*output_nn, ))
assert output_nn == pytest.approx(output_skl)
return nn, mlp
def test_addLayer(self):
activationFunction1 = lambda x: x
activationFunction2 = lambda x: x**2
target = NeuralNetwork()
target.addLayer(10, 20, activationFunction1)
self.assertEqual(len(target._layers), 1)
self.assertEqual(len(target._layers[0]), 10)
tf.debugging.assert_equal(target._layers[0][0]._weights, tf.zeros(20))
tf.debugging.assert_equal(target._layers[0][0]._bias,
tf.Variable(0, dtype=tf.dtypes.float32))
self.assertEqual(target._layers[0][0]._activationFunction,
activationFunction1)
target.addLayer(5, 15, activationFunction2)
self.assertEqual(len(target._layers), 2)
self.assertEqual(len(target._layers[1]), 5)
tf.debugging.assert_equal(target._layers[1][0]._weights, tf.zeros(15))
tf.debugging.assert_equal(target._layers[1][0]._bias,
tf.Variable(0, dtype=tf.dtypes.float32))
self.assertEqual(target._layers[1][0]._activationFunction,
activationFunction2)
def test_neuralNetwork_fit_adam():
np.random.seed(2019)
X = np.random.normal(size=(1, 500))
target = 3.9285985 * X
nn = NeuralNetwork(inputs=1,
neurons=3,
outputs=1,
activations='tanh',
silent=True)
nn.addLayer()
nn.addLayer()
nn.addOutputLayer(activations='identity')
nn.fit(X,
target,
shuffle=True,
batch_size=100,
validation_fraction=0.2,
learning_rate=0.05,
verbose=True,
silent=False,
epochs=100,
optimizer='adam')
loss = nn.loss
nn.fit(X,
target,
shuffle=True,
batch_size=100,
validation_fraction=0.2,
learning_rate=0.05,
verbose=True,
silent=False,
epochs=100,
optimizer='adam')
assert loss > nn.loss
def test_neuralNetwork_fit_sgd():
np.random.seed(2019)
X = np.random.normal(size=(1, 500))
target = 3.9285985 * X
nn = NeuralNetwork(inputs=1,
neurons=3,
outputs=1,
activations='sigmoid',
silent=True)
nn.addLayer()
nn.addLayer()
nn.addOutputLayer(activations='identity')
nn.fit(X,
target,
shuffle=True,
batch_size=100,
validation_fraction=0.2,
learning_rate=0.05,
verbose=False,
silent=True,
epochs=100)
loss_after_100 = nn.loss
nn.fit(X,
target,
shuffle=True,
batch_size=100,
validation_fraction=0.2,
learning_rate=0.05,
verbose=False,
silent=True,
epochs=100)
loss_after_200 = nn.loss
assert loss_after_200 < loss_after_100
def test_getBiasesForLayer(self):
target = NeuralNetwork()
target.addLayer(10, 20, lambda x: x**2)
tf.debugging.assert_equal(target.getBiasesForLayer(0), tf.zeros([10]))
def initNetwork():
# Initialize and build neural network
nn = NeuralNetwork(0.001)
#
# Layer zero, the input layer
# Create neurons: the number of neurons is the same as the input
# List of 29*29=841 pixels, and no weights/connections
#
layer0 = NNLayer("layer0")
for i in range(0, 841):
layer0.addNeuron()
nn.addLayer(layer0)
#
# Layer 1: Convolutional layer
# 6 feature maps. Each feature map is 13x13, and each unit in the feature map is a 5x5 convolutional kernel
# from the input layer.
# So there are 13x13x6 = 1014 neurons, (5x5+1)x6 wights
#
layer1 = NNLayer("layer1")
layer1.setPrevLayer(layer0)
# Add the neurons
for i in range(0, 1014):
layer1.addNeuron()
# Add wights
for i in range(0, 156):
# Uniform random distribution
initWeight = 0.05 * random.uniform(-1, 1)
layer1.addWeight(initWeight)
# interconnections with previous layer: this is difficult
# The previous layer is a top-down bitmap
# image that has been padded to size 29x29
# Each neuron in this layer is connected
# to a 5x5 kernel in its feature map, which
# is also a top-down bitmap of size 13x13.
# We move the kernel by TWO pixels, i.e., we
# skip every other pixel in the input image
kernelTemplate = [
0, 1, 2, 3, 4, 29, 30, 31, 32, 33, 58, 59, 60, 61, 62, 87, 88, 89, 90,
91, 116, 117, 118, 119, 120
]
#Feature maps
for fm in range(0, 6):
for i in range(0, 13):
for j in range(0, 13):
# 26 is the number of weights per featuremaps
iNumWeights = fm * 26
# Bias weight
layer1.neurons[fm * 169 + j + i * 13].addConnection(
-10000, iNumWeights)
iNumWeights += 1
for k in range(0, 25):
layer1.neurons[fm * 169 + j + i * 13].addConnection(
2 * j + 58 * i + kernelTemplate[k], iNumWeights)
iNumWeights += 1
# Add layer to network
nn.addLayer(layer1)
#
# Layer two: This layer is a convolutional layer
# 50 feature maps. Each feature map is 5x5, and each unit in the feature maps is a 5x5 convolutional kernel of
# corresponding areas of all 6 of the previous layers, each of which is a 13x13 feature map.
# So, there are 5x5x50 = 1250 neurons, (5X5+1)x6x50 = 7800 weights
layer2 = NNLayer("layer2")
layer2.setPrevLayer(layer1)
# Add the neurons
for i in range(0, 1250):
layer2.addNeuron()
# Add wights
for i in range(0, 7800):
# Uniform random distribution
initWeight = 0.05 * random.uniform(-1, 1)
layer2.addWeight(initWeight)
# Interconnections with previous layer: this is difficult
# Each feature map in the previous layer
# is a top-down bitmap image whose size
# is 13x13, and there are 6 such feature maps.
# Each neuron in one 5x5 feature map of this
# layer is connected to a 5x5 kernel
# positioned correspondingly in all 6 parent
# feature maps, and there are individual
# weights for the six different 5x5 kernels. As
# before, we move the kernel by TWO pixels, i.e., we
# skip every other pixel in the input image.
# The result is 50 different 5x5 top-down bitmap
# feature maps
kernelTemplate = [
0, 1, 2, 3, 4, 13, 14, 15, 16, 17, 26, 27, 28, 29, 30, 39, 40, 41, 42,
43, 52, 53, 54, 55, 56
]
for fm in range(0, 50):
for i in range(0, 5):
for j in range(0, 5):
# 26 is the number of weights per featuremaps
iNumWeight = fm * 26
# Bias weight
layer2.neurons[fm * 25 + j + i * 5].addConnection(
-10000, iNumWeight)
iNumWeight += 1
for k in range(0, 25):
layer2.neurons[fm * 25 + j + i * 5].addConnection(
2 * j + 26 * i + kernelTemplate[k], iNumWeight)
iNumWeight += 1
layer2.neurons[fm * 25 + j + i * 5].addConnection(
169 + 2 * j + 26 * i + kernelTemplate[k], iNumWeight)
iNumWeight += 1
layer2.neurons[fm * 25 + j + i * 5].addConnection(
338 + 2 * j + 26 * i + kernelTemplate[k], iNumWeight)
iNumWeight += 1
layer2.neurons[fm * 25 + j + i * 5].addConnection(
507 + 2 * j + 26 * i + kernelTemplate[k], iNumWeight)
iNumWeight += 1
layer2.neurons[fm * 25 + j + i * 5].addConnection(
676 + 2 * j + 26 * i + kernelTemplate[k], iNumWeight)
iNumWeight += 1
layer2.neurons[fm * 25 + j + i * 5].addConnection(
845 + 2 * j + 26 * i + kernelTemplate[k], iNumWeight)
iNumWeight += 1
# add layer to network
nn.addLayer(layer2)
#
# layer three:
# This layer is a fully-connected layer
# with 100 units. Since it is fully-connected,
# each of the 100 neurons in the
# layer is connected to all 1250 neurons in
# the previous layer.
# So, there are 100 neurons and 100*(1250+1)=125100 weights
#
layer3 = NNLayer("layer3")
layer3.setPrevLayer(layer2)
# Add the neurons
for i in range(0, 100):
layer3.addNeuron()
# Add wights
for i in range(0, 125100):
# Uniform random distribution
initWeight = 0.05 * random.uniform(-1, 1)
layer3.addWeight(initWeight)
# Interconnections with previous layer: fully-connected
iNumWeight = 0 # Weights are not shared in this layer
for fm in range(0, 100):
layer3.neurons[fm].addConnection(-10000, iNumWeight) #bias
iNumWeight += 1
for i in range(0, 1250):
layer3.neurons[fm].addConnection(i, iNumWeight) #bias
iNumWeight += 1
# Add layer to network
nn.addLayer(layer3)
# layer four, the final (output) layer:
# This layer is a fully-connected layer
# with 10 units. Since it is fully-connected,
# each of the 10 neurons in the layer
# is connected to all 100 neurons in
# the previous layer.
# So, there are 10 neurons and 10*(100+1)=1010 weights
layer4 = NNLayer("layer4")
layer4.setPrevLayer(layer3)
# Add the neurons
for i in range(0, 10):
layer4.addNeuron()
# Add wights
for i in range(0, 1010):
# Uniform random distribution
initWeight = 0.05 * random.uniform(-1, 1)
layer4.addWeight(initWeight)
# Interconnections with previous layer: fully-connected
iNumWeight = 0 # Weights are not shared in this layer
for fm in range(0, 10):
layer4.neurons[fm].addConnection(-10000, iNumWeight) #bias
iNumWeight += 1
for i in range(0, 100):
layer4.neurons[fm].addConnection(i, iNumWeight) #bias
iNumWeight += 1
# Add layer to network
nn.addLayer(layer4)
print "NN structure:"
print "Layer 0:", len(nn.layers[0].neurons)
print "Layer 1:", len(nn.layers[1].neurons)
print "Layer 2:", len(nn.layers[2].neurons)
print "Layer 3:", len(nn.layers[3].neurons)
print "Layer 4:", len(nn.layers[4].neurons)
print "\n"
return nn
def test_neuralNetwork_addLayer():
inputs = 6
outputs = 4
layers = 3
neurons = 87
activations = 'sigmoid'
nn = NeuralNetwork(inputs=inputs,
outputs=outputs,
layers=layers,
neurons=neurons,
activations=activations)
nn.addLayer()
assert nn.weights[-1].shape == (inputs, neurons)
assert nn.biases[-1].shape == (neurons, 1)
assert type(nn.act[-1]) is Activation
assert nn.act[-1].function == nn.act[-1]._sigmoid
new_neurons = 10
new_activations = 'relu'
nn.addLayer(neurons=new_neurons, activations=new_activations)
assert nn.weights[-1].shape == (neurons, new_neurons)
assert nn.biases[-1].shape == (new_neurons, 1)
assert type(nn.act[-1]) is Activation
assert nn.act[-1].function == nn.act[-1]._relu
nn_copy1 = copy.deepcopy(nn)
nn_copy2 = copy.deepcopy(nn)
nn_copy3 = copy.deepcopy(nn)
nn_copy4 = copy.deepcopy(nn)
nn_copy1.addOutputLayer()
assert nn_copy1.weights[-1].shape == (new_neurons, outputs)
assert nn_copy1.biases[-1].shape == (outputs, 1)
nn_copy2.addLayer(output=True)
assert nn_copy2.weights[-1].shape == (new_neurons, outputs)
assert nn_copy2.biases[-1].shape == (outputs, 1)
new_outputs = 24
with warnings.catch_warnings():
warnings.simplefilter("ignore")
nn_copy3.addOutputLayer(outputs=new_outputs)
assert nn_copy3.weights[-1].shape == (new_neurons, new_outputs)
assert nn_copy3.biases[-1].shape == (new_outputs, 1)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
nn_copy4.addLayer(outputs=new_outputs, output=True)
assert nn_copy4.weights[-1].shape == (new_neurons, new_outputs)
assert nn_copy4.biases[-1].shape == (new_outputs, 1)
# Now that we constructed an entire network, check that the matrices line
# up so that the network can be evaluated
nn = copy.deepcopy(nn_copy1)
x = np.random.uniform(-1.0, 1.0, size=(inputs, 1))
assert nn(x).shape == (outputs, 1)
assert nn.network(x).shape == (outputs, 1)
nn = copy.deepcopy(nn_copy3)
assert nn(x).shape == (new_outputs, 1)
assert nn.network(x).shape == (new_outputs, 1)
def test_neuralNetwork_backpropagation_multiple_outputs():
# Similar to the test_neuralNetwork_backpropagation() test, but with
# multiple samples and features, X having dimensions
# (n_samples, n_features) = (3,2), as well as the target having dimensions
# (n_outputs) = (2)
from sklearn.neural_network import MLPRegressor
X = np.array([[0, 1], [1, 2], [2, 3]])
y = np.array([[0, 1], [2, 3], [3, 4]])
mlp = MLPRegressor(solver='sgd',
alpha=0.0,
learning_rate='constant',
learning_rate_init=1e-20,
max_iter=1,
hidden_layer_sizes=(3, 3),
random_state=1,
activation='logistic')
# Force sklearn to set up all the matrices by fitting a data set.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
mlp.fit(X, y)
W_skl = mlp.coefs_
b_skl = mlp.intercepts_
nn = NeuralNetwork(inputs=2,
outputs=2,
layers=3,
neurons=3,
activations='sigmoid',
silent=True)
nn.addLayer()
nn.addLayer()
nn.addOutputLayer(activations='identity')
for i, w in enumerate(W_skl):
nn.weights[i] = w
for i, b in enumerate(b_skl):
nn.biases[i] = np.expand_dims(b, axis=1)
# ========================================================================
n_samples, n_features = X.shape
batch_size = n_samples
hidden_layer_sizes = mlp.hidden_layer_sizes
if not hasattr(hidden_layer_sizes, "__iter__"):
hidden_layer_sizes = [hidden_layer_sizes]
hidden_layer_sizes = list(hidden_layer_sizes)
layer_units = ([n_features] + hidden_layer_sizes + [mlp.n_outputs_])
activations = [X]
activations.extend(
np.empty((batch_size, n_fan_out)) for n_fan_out in layer_units[1:])
deltas = [np.empty_like(a_layer) for a_layer in activations]
coef_grads = [
np.empty((n_fan_in_, n_fan_out_))
for n_fan_in_, n_fan_out_ in zip(layer_units[:-1], layer_units[1:])
]
intercept_grads = [np.empty(n_fan_out_) for n_fan_out_ in layer_units[1:]]
# ========================================================================
activations = mlp._forward_pass(activations)
if y.ndim == 1:
y = y.reshape((-1, 1))
loss, coef_grads, intercept_grads = mlp._backprop(X, y, activations,
deltas, coef_grads,
intercept_grads)
yhat = nn.forward_pass(X.T)
nn.backpropagation(yhat.T, y)
for i, d_bias in enumerate(nn.d_biases):
assert np.squeeze(d_bias) == pytest.approx(
np.squeeze(intercept_grads[i]))
for i, d_weight in enumerate(nn.d_weights):
assert np.squeeze(d_weight) == pytest.approx(np.squeeze(coef_grads[i]))
def test_neuralNetwork_backpropagation():
# We re-use the test_neuralNetwork_network networks and this time check
# that the computed backpropagation derivatives are equal.
from sklearn.neural_network import MLPRegressor
X = [[0.0], [1.0], [2.0], [3.0], [4.0], [5.0]]
y = [0, 2, 4, 6, 8, 10]
mlp = MLPRegressor(solver='sgd',
alpha=0.0,
hidden_layer_sizes=(3, 3),
random_state=1,
activation='logistic')
# Force sklearn to set up all the matrices by fitting a data set.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
mlp.fit(X, y)
# Throw away all the fitted values, randomize W and b matrices.
np.random.seed(18)
for i, coeff in enumerate(mlp.coefs_):
mlp.coefs_[i] = np.random.normal(size=coeff.shape)
for i, bias in enumerate(mlp.intercepts_):
mlp.intercepts_[i] = np.random.normal(size=bias.shape)
W_skl = mlp.coefs_
b_skl = mlp.intercepts_
nn = NeuralNetwork(inputs=1,
outputs=1,
layers=3,
neurons=3,
activations='sigmoid',
silent=False)
nn.addLayer()
nn.addLayer()
nn.addOutputLayer(activations='identity')
nn.weights = W_skl
for i, b in enumerate(b_skl):
nn.biases[i] = np.expand_dims(b, axis=1)
# From the sklearn source, we need to set up some lists to use the _backprop
# function in MLPRegressor, see:
#
# https://github.com/scikit-learn/scikit-learn/blob/bac89c2/sklearn/neural_network/multilayer_perceptron.py#L355
#
# ========================================================================
# Initialize lists
X = np.array([[1.125982598]])
y = np.array([8.29289285])
mlp.predict(X)
n_samples, n_features = X.shape
batch_size = n_samples
hidden_layer_sizes = mlp.hidden_layer_sizes
# Make sure self.hidden_layer_sizes is a list
if not hasattr(hidden_layer_sizes, "__iter__"):
hidden_layer_sizes = [hidden_layer_sizes]
hidden_layer_sizes = list(hidden_layer_sizes)
layer_units = ([n_features] + hidden_layer_sizes + [mlp.n_outputs_])
activations = [X]
activations.extend(
np.empty((batch_size, n_fan_out)) for n_fan_out in layer_units[1:])
deltas = [np.empty_like(a_layer) for a_layer in activations]
coef_grads = [
np.empty((n_fan_in_, n_fan_out_))
for n_fan_in_, n_fan_out_ in zip(layer_units[:-1], layer_units[1:])
]
intercept_grads = [np.empty(n_fan_out_) for n_fan_out_ in layer_units[1:]]
# ========================================================================
activations = mlp._forward_pass(activations)
loss, coef_grads, intercept_grads = mlp._backprop(X, y, activations,
deltas, coef_grads,
intercept_grads)
yhat = nn(X)
nn.backpropagation(yhat, y)
for i, d_bias in enumerate(nn.d_biases):
assert np.squeeze(d_bias) == pytest.approx(
np.squeeze(intercept_grads[i]))
for i, d_weight in enumerate(nn.d_weights):
assert np.squeeze(d_weight) == pytest.approx(np.squeeze(coef_grads[i]))
def train_net_predict_energy(L=10, N=5000):
ising = Ising(L, N)
X, y = ising.generateTrainingData1D()
y /= L
n_samples, n_features = X.shape
nn = NeuralNetwork(inputs=L,
neurons=L * L,
outputs=1,
activations='sigmoid',
cost='mse',
silent=False)
nn.addLayer(neurons=L * L)
nn.addLayer(neurons=L * L)
nn.addOutputLayer(activations='identity')
validation_skip = 10
epochs = 1000
nn.fit(X.T,
y,
shuffle=True,
batch_size=1000,
validation_fraction=0.2,
learning_rate=0.001,
verbose=False,
silent=False,
epochs=epochs,
validation_skip=validation_skip,
optimizer='adam')
# Use the net to predict the energies for the validation set.
x_validation = nn.x_validation
y_validation = nn.predict(x_validation)
target_validation = nn.target_validation
# Sort the targets for better visualization of the network output.
ind = np.argsort(target_validation)
y_validation = np.squeeze(y_validation.T[ind])
target_validation = np.squeeze(target_validation.T[ind])
# We dont want to plot the discontinuities in the target.
target_validation[np.where(
np.abs(np.diff(target_validation)) > 1e-5)] = np.nan
plt.rc('text', usetex=True)
plt.figure()
plt.plot(target_validation, 'k--', label=r'Target')
plt.plot(y_validation, 'r.', markersize=0.5, label=r'NN output')
plt.legend(fontsize=10)
plt.xlabel(r'Validation sample', fontsize=10)
plt.ylabel(r'$E / L$', fontsize=10)
#plt.savefig(os.path.join(os.path.dirname(__file__), 'figures', 'nn_1d_energy_predict' + str(L) + '.png'), transparent=True, bbox_inches='tight')
# Plot the training / validation loss during training.
training_loss = nn.training_loss
validation_loss = nn.validation_loss
# There are more training loss values than validation loss values, lets
# align them so the plot makes sense.
xaxis_validation_loss = np.zeros_like(validation_loss)
xaxis_validation_loss[0] = 0
xaxis_validation_loss[1:-1] = np.arange(validation_skip,
len(training_loss),
validation_skip)
xaxis_validation_loss[-1] = len(training_loss)
plt.figure()
plt.semilogy(training_loss, 'r-', label=r'Training loss')
plt.semilogy(xaxis_validation_loss,
validation_loss,
'k--',
label=r'Validation loss')
plt.legend(fontsize=10)
plt.xlabel(r'Epoch', fontsize=10)
plt.ylabel(r'Cost $C(\theta)$', fontsize=10)
#plt.savefig(os.path.join(os.path.dirname(__file__), 'figures', 'nn_1d_loss' + str(L) + '.png'), transparent=True, bbox_inches='tight')
plt.show()
def R2_versus_lasso():
L = 3
N = 10000
training_fraction = 0.4
ising = Ising(L, N)
D, ry = ising.generateDesignMatrix1D()
X, y = ising.generateTrainingData1D()
y /= L
D_train = D[int(training_fraction * N):, :]
ry_train = ry[int(training_fraction * N):]
D_validation = D[:int(training_fraction * N), :]
ry_validation = ry[:int(training_fraction * N)]
lasso = LeastSquares(method='lasso', backend='skl')
lasso.setLambda(1e-2)
lasso.fit(D_train, ry_train)
lasso.y = ry_validation
lasso_R2 = sklearn.metrics.mean_squared_error(
ry_validation / L,
lasso.predict(D_validation) / L)
n_samples, n_features = X.shape
nn = NeuralNetwork(inputs=L * L,
neurons=L,
outputs=1,
activations='identity',
cost='mse',
silent=False)
nn.addLayer(neurons=1)
nn.addOutputLayer(activations='identity')
validation_skip = 100
epochs = 50000
nn.fit(D.T,
ry,
shuffle=True,
batch_size=2000,
validation_fraction=1 - training_fraction,
learning_rate=0.0001,
verbose=False,
silent=False,
epochs=epochs,
validation_skip=validation_skip,
optimizer='adam')
plt.rc('text', usetex=True)
validation_loss = nn.validation_loss_improving
validation_ep = np.linspace(0, epochs, len(nn.validation_loss_improving))
plt.semilogy(validation_ep, validation_loss, 'r-', label=r'NN')
plt.semilogy([0, epochs],
np.array([lasso_R2, lasso_R2]),
'k--',
label=r'Lasso')
plt.xlabel(r'Epoch', fontsize=10)
plt.ylabel(r'Mean squared error', fontsize=10)
plt.legend(fontsize=10)
plt.xlim((0, epochs))
ax = plt.gca()
ymin, ymax = ax.get_ylim()
if ymin > pow(10, -5):
ymin = pow(10, -5)
#plt.ylim((ymin,ymax))
plt.savefig(os.path.join(os.path.dirname(__file__), 'figures',
'NN_compare_lasso.png'),
transparent=True,
bbox_inches='tight')
import numpy as np
import tensorflow as tf
from dataset import Dataset
from neuralNetwork import NeuralNetwork
from softMaxClassifier import SoftMaxClassifier
if __name__ == "__main__":
dataset = Dataset()
neuralNetwork = NeuralNetwork()
neuralNetwork.addLayer(10, len(dataset.XTrain[0]))
softMaxClassifier = SoftMaxClassifier(neuralNetwork, 0.1, 50)
softMaxClassifier.gradientDescent(dataset.XTrain, dataset.yTrain)
