def test_neuralNetwork_sgd(): from sklearn.neural_network._stochastic_optimizers import SGDOptimizer np.random.seed(2019) X = np.random.normal(size=(1, 500)) target = 3.9285985 * X nn = NeuralNetwork(inputs=1, neurons=3, outputs=1, activations='sigmoid', silent=True) nn.addLayer() nn.addLayer() nn.addOutputLayer(activations='identity') learning_rate = 0.001 yhat = nn.forward_pass(X) nn.backpropagation(yhat.T, target.T) nn.learning_rate = learning_rate initial_params = copy.deepcopy(nn.weights + nn.biases) nn.sgd() grad = nn.d_weights + nn.d_biases params = nn.weights + nn.biases change = [p - i_p for p, i_p in zip(params, initial_params)] skl_sgd = SGDOptimizer(params=initial_params, learning_rate_init=learning_rate, nesterov=False, momentum=1.0) upd = skl_sgd._get_updates(grad) for update_nn, update_skl in zip(change, upd): assert update_nn == pytest.approx(update_skl)
def test_neuralNetwork_adam(): from sklearn.neural_network._stochastic_optimizers import AdamOptimizer np.random.seed(2019) X = np.random.normal(size=(1, 500)) target = 3.9285985 * X nn = NeuralNetwork(inputs=1, neurons=3, outputs=1, activations='sigmoid', silent=True) nn.addLayer() nn.addLayer() nn.addOutputLayer(activations='identity') learning_rate = 0.001 yhat = nn.forward_pass(X) nn.backpropagation(yhat.T, target.T) nn.learning_rate = learning_rate nn.initializeAdam() nn.adam() skl_adam = AdamOptimizer(params=nn.param, learning_rate_init=learning_rate) upd = skl_adam._get_updates(nn.grad) for update_nn, update_skl in zip(nn.change, upd): assert update_nn == pytest.approx(update_skl)
def test_predict(self, ): target = NeuralNetwork() target.addLayer(10, 10, lambda x: x**2) prediction = target.predict( tf.constant([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=tf.dtypes.float32), [tf.zeros([10, 10])], [tf.zeros([10])]) tf.debugging.assert_equal(prediction, tf.zeros(10))
def test_neuralNetwork_network(silent=False): # Lets set up a sci-kit learn neural network and copy over the weights # and biases to our network, verify that the two give the exact same # result. from sklearn.neural_network import MLPRegressor X = [[0.0], [1.0], [2.0], [3.0], [4.0], [5.0]] y = [0, 2, 4, 6, 8, 10] mlp = MLPRegressor(solver='sgd', alpha=0.0, hidden_layer_sizes=(3, 3), random_state=1, activation='relu') mlp.fit(X, y) W_skl = mlp.coefs_ b_skl = mlp.intercepts_ nn = NeuralNetwork(inputs=1, outputs=1, layers=3, neurons=3, activations='relu', silent=silent) nn.addLayer() nn.addLayer() nn.addOutputLayer(activations='identity') W_nn = nn.weights b_nn = nn.biases for i in range(len(W_nn)): W_nn[i] = W_skl[i] for i in range(len(b_nn)): b_nn[i] = np.expand_dims(b_skl[i], axis=1) X_test = np.array([[1.2857], [9.2508255], [-5.25255], [3.251095]]) output_skl = mlp.predict(X_test) output_nn = np.squeeze(nn(X_test.T)) if not silent: print("%20.15f %20.15f %20.15f %20.15f" % (*output_skl, )) print("%20.15f %20.15f %20.15f %20.15f" % (*output_nn, )) assert output_nn == pytest.approx(output_skl) return nn, mlp
def test_addLayer(self): activationFunction1 = lambda x: x activationFunction2 = lambda x: x**2 target = NeuralNetwork() target.addLayer(10, 20, activationFunction1) self.assertEqual(len(target._layers), 1) self.assertEqual(len(target._layers[0]), 10) tf.debugging.assert_equal(target._layers[0][0]._weights, tf.zeros(20)) tf.debugging.assert_equal(target._layers[0][0]._bias, tf.Variable(0, dtype=tf.dtypes.float32)) self.assertEqual(target._layers[0][0]._activationFunction, activationFunction1) target.addLayer(5, 15, activationFunction2) self.assertEqual(len(target._layers), 2) self.assertEqual(len(target._layers[1]), 5) tf.debugging.assert_equal(target._layers[1][0]._weights, tf.zeros(15)) tf.debugging.assert_equal(target._layers[1][0]._bias, tf.Variable(0, dtype=tf.dtypes.float32)) self.assertEqual(target._layers[1][0]._activationFunction, activationFunction2)
def test_neuralNetwork_fit_adam(): np.random.seed(2019) X = np.random.normal(size=(1, 500)) target = 3.9285985 * X nn = NeuralNetwork(inputs=1, neurons=3, outputs=1, activations='tanh', silent=True) nn.addLayer() nn.addLayer() nn.addOutputLayer(activations='identity') nn.fit(X, target, shuffle=True, batch_size=100, validation_fraction=0.2, learning_rate=0.05, verbose=True, silent=False, epochs=100, optimizer='adam') loss = nn.loss nn.fit(X, target, shuffle=True, batch_size=100, validation_fraction=0.2, learning_rate=0.05, verbose=True, silent=False, epochs=100, optimizer='adam') assert loss > nn.loss
def test_neuralNetwork_fit_sgd(): np.random.seed(2019) X = np.random.normal(size=(1, 500)) target = 3.9285985 * X nn = NeuralNetwork(inputs=1, neurons=3, outputs=1, activations='sigmoid', silent=True) nn.addLayer() nn.addLayer() nn.addOutputLayer(activations='identity') nn.fit(X, target, shuffle=True, batch_size=100, validation_fraction=0.2, learning_rate=0.05, verbose=False, silent=True, epochs=100) loss_after_100 = nn.loss nn.fit(X, target, shuffle=True, batch_size=100, validation_fraction=0.2, learning_rate=0.05, verbose=False, silent=True, epochs=100) loss_after_200 = nn.loss assert loss_after_200 < loss_after_100
def test_getBiasesForLayer(self): target = NeuralNetwork() target.addLayer(10, 20, lambda x: x**2) tf.debugging.assert_equal(target.getBiasesForLayer(0), tf.zeros([10]))
def initNetwork(): # Initialize and build neural network nn = NeuralNetwork(0.001) # # Layer zero, the input layer # Create neurons: the number of neurons is the same as the input # List of 29*29=841 pixels, and no weights/connections # layer0 = NNLayer("layer0") for i in range(0, 841): layer0.addNeuron() nn.addLayer(layer0) # # Layer 1: Convolutional layer # 6 feature maps. Each feature map is 13x13, and each unit in the feature map is a 5x5 convolutional kernel # from the input layer. # So there are 13x13x6 = 1014 neurons, (5x5+1)x6 wights # layer1 = NNLayer("layer1") layer1.setPrevLayer(layer0) # Add the neurons for i in range(0, 1014): layer1.addNeuron() # Add wights for i in range(0, 156): # Uniform random distribution initWeight = 0.05 * random.uniform(-1, 1) layer1.addWeight(initWeight) # interconnections with previous layer: this is difficult # The previous layer is a top-down bitmap # image that has been padded to size 29x29 # Each neuron in this layer is connected # to a 5x5 kernel in its feature map, which # is also a top-down bitmap of size 13x13. # We move the kernel by TWO pixels, i.e., we # skip every other pixel in the input image kernelTemplate = [ 0, 1, 2, 3, 4, 29, 30, 31, 32, 33, 58, 59, 60, 61, 62, 87, 88, 89, 90, 91, 116, 117, 118, 119, 120 ] #Feature maps for fm in range(0, 6): for i in range(0, 13): for j in range(0, 13): # 26 is the number of weights per featuremaps iNumWeights = fm * 26 # Bias weight layer1.neurons[fm * 169 + j + i * 13].addConnection( -10000, iNumWeights) iNumWeights += 1 for k in range(0, 25): layer1.neurons[fm * 169 + j + i * 13].addConnection( 2 * j + 58 * i + kernelTemplate[k], iNumWeights) iNumWeights += 1 # Add layer to network nn.addLayer(layer1) # # Layer two: This layer is a convolutional layer # 50 feature maps. Each feature map is 5x5, and each unit in the feature maps is a 5x5 convolutional kernel of # corresponding areas of all 6 of the previous layers, each of which is a 13x13 feature map. # So, there are 5x5x50 = 1250 neurons, (5X5+1)x6x50 = 7800 weights layer2 = NNLayer("layer2") layer2.setPrevLayer(layer1) # Add the neurons for i in range(0, 1250): layer2.addNeuron() # Add wights for i in range(0, 7800): # Uniform random distribution initWeight = 0.05 * random.uniform(-1, 1) layer2.addWeight(initWeight) # Interconnections with previous layer: this is difficult # Each feature map in the previous layer # is a top-down bitmap image whose size # is 13x13, and there are 6 such feature maps. # Each neuron in one 5x5 feature map of this # layer is connected to a 5x5 kernel # positioned correspondingly in all 6 parent # feature maps, and there are individual # weights for the six different 5x5 kernels. As # before, we move the kernel by TWO pixels, i.e., we # skip every other pixel in the input image. # The result is 50 different 5x5 top-down bitmap # feature maps kernelTemplate = [ 0, 1, 2, 3, 4, 13, 14, 15, 16, 17, 26, 27, 28, 29, 30, 39, 40, 41, 42, 43, 52, 53, 54, 55, 56 ] for fm in range(0, 50): for i in range(0, 5): for j in range(0, 5): # 26 is the number of weights per featuremaps iNumWeight = fm * 26 # Bias weight layer2.neurons[fm * 25 + j + i * 5].addConnection( -10000, iNumWeight) iNumWeight += 1 for k in range(0, 25): layer2.neurons[fm * 25 + j + i * 5].addConnection( 2 * j + 26 * i + kernelTemplate[k], iNumWeight) iNumWeight += 1 layer2.neurons[fm * 25 + j + i * 5].addConnection( 169 + 2 * j + 26 * i + kernelTemplate[k], iNumWeight) iNumWeight += 1 layer2.neurons[fm * 25 + j + i * 5].addConnection( 338 + 2 * j + 26 * i + kernelTemplate[k], iNumWeight) iNumWeight += 1 layer2.neurons[fm * 25 + j + i * 5].addConnection( 507 + 2 * j + 26 * i + kernelTemplate[k], iNumWeight) iNumWeight += 1 layer2.neurons[fm * 25 + j + i * 5].addConnection( 676 + 2 * j + 26 * i + kernelTemplate[k], iNumWeight) iNumWeight += 1 layer2.neurons[fm * 25 + j + i * 5].addConnection( 845 + 2 * j + 26 * i + kernelTemplate[k], iNumWeight) iNumWeight += 1 # add layer to network nn.addLayer(layer2) # # layer three: # This layer is a fully-connected layer # with 100 units. Since it is fully-connected, # each of the 100 neurons in the # layer is connected to all 1250 neurons in # the previous layer. # So, there are 100 neurons and 100*(1250+1)=125100 weights # layer3 = NNLayer("layer3") layer3.setPrevLayer(layer2) # Add the neurons for i in range(0, 100): layer3.addNeuron() # Add wights for i in range(0, 125100): # Uniform random distribution initWeight = 0.05 * random.uniform(-1, 1) layer3.addWeight(initWeight) # Interconnections with previous layer: fully-connected iNumWeight = 0 # Weights are not shared in this layer for fm in range(0, 100): layer3.neurons[fm].addConnection(-10000, iNumWeight) #bias iNumWeight += 1 for i in range(0, 1250): layer3.neurons[fm].addConnection(i, iNumWeight) #bias iNumWeight += 1 # Add layer to network nn.addLayer(layer3) # layer four, the final (output) layer: # This layer is a fully-connected layer # with 10 units. Since it is fully-connected, # each of the 10 neurons in the layer # is connected to all 100 neurons in # the previous layer. # So, there are 10 neurons and 10*(100+1)=1010 weights layer4 = NNLayer("layer4") layer4.setPrevLayer(layer3) # Add the neurons for i in range(0, 10): layer4.addNeuron() # Add wights for i in range(0, 1010): # Uniform random distribution initWeight = 0.05 * random.uniform(-1, 1) layer4.addWeight(initWeight) # Interconnections with previous layer: fully-connected iNumWeight = 0 # Weights are not shared in this layer for fm in range(0, 10): layer4.neurons[fm].addConnection(-10000, iNumWeight) #bias iNumWeight += 1 for i in range(0, 100): layer4.neurons[fm].addConnection(i, iNumWeight) #bias iNumWeight += 1 # Add layer to network nn.addLayer(layer4) print "NN structure:" print "Layer 0:", len(nn.layers[0].neurons) print "Layer 1:", len(nn.layers[1].neurons) print "Layer 2:", len(nn.layers[2].neurons) print "Layer 3:", len(nn.layers[3].neurons) print "Layer 4:", len(nn.layers[4].neurons) print "\n" return nn
def test_neuralNetwork_addLayer(): inputs = 6 outputs = 4 layers = 3 neurons = 87 activations = 'sigmoid' nn = NeuralNetwork(inputs=inputs, outputs=outputs, layers=layers, neurons=neurons, activations=activations) nn.addLayer() assert nn.weights[-1].shape == (inputs, neurons) assert nn.biases[-1].shape == (neurons, 1) assert type(nn.act[-1]) is Activation assert nn.act[-1].function == nn.act[-1]._sigmoid new_neurons = 10 new_activations = 'relu' nn.addLayer(neurons=new_neurons, activations=new_activations) assert nn.weights[-1].shape == (neurons, new_neurons) assert nn.biases[-1].shape == (new_neurons, 1) assert type(nn.act[-1]) is Activation assert nn.act[-1].function == nn.act[-1]._relu nn_copy1 = copy.deepcopy(nn) nn_copy2 = copy.deepcopy(nn) nn_copy3 = copy.deepcopy(nn) nn_copy4 = copy.deepcopy(nn) nn_copy1.addOutputLayer() assert nn_copy1.weights[-1].shape == (new_neurons, outputs) assert nn_copy1.biases[-1].shape == (outputs, 1) nn_copy2.addLayer(output=True) assert nn_copy2.weights[-1].shape == (new_neurons, outputs) assert nn_copy2.biases[-1].shape == (outputs, 1) new_outputs = 24 with warnings.catch_warnings(): warnings.simplefilter("ignore") nn_copy3.addOutputLayer(outputs=new_outputs) assert nn_copy3.weights[-1].shape == (new_neurons, new_outputs) assert nn_copy3.biases[-1].shape == (new_outputs, 1) with warnings.catch_warnings(): warnings.simplefilter("ignore") nn_copy4.addLayer(outputs=new_outputs, output=True) assert nn_copy4.weights[-1].shape == (new_neurons, new_outputs) assert nn_copy4.biases[-1].shape == (new_outputs, 1) # Now that we constructed an entire network, check that the matrices line # up so that the network can be evaluated nn = copy.deepcopy(nn_copy1) x = np.random.uniform(-1.0, 1.0, size=(inputs, 1)) assert nn(x).shape == (outputs, 1) assert nn.network(x).shape == (outputs, 1) nn = copy.deepcopy(nn_copy3) assert nn(x).shape == (new_outputs, 1) assert nn.network(x).shape == (new_outputs, 1)
def test_neuralNetwork_backpropagation_multiple_outputs(): # Similar to the test_neuralNetwork_backpropagation() test, but with # multiple samples and features, X having dimensions # (n_samples, n_features) = (3,2), as well as the target having dimensions # (n_outputs) = (2) from sklearn.neural_network import MLPRegressor X = np.array([[0, 1], [1, 2], [2, 3]]) y = np.array([[0, 1], [2, 3], [3, 4]]) mlp = MLPRegressor(solver='sgd', alpha=0.0, learning_rate='constant', learning_rate_init=1e-20, max_iter=1, hidden_layer_sizes=(3, 3), random_state=1, activation='logistic') # Force sklearn to set up all the matrices by fitting a data set. with warnings.catch_warnings(): warnings.simplefilter("ignore") mlp.fit(X, y) W_skl = mlp.coefs_ b_skl = mlp.intercepts_ nn = NeuralNetwork(inputs=2, outputs=2, layers=3, neurons=3, activations='sigmoid', silent=True) nn.addLayer() nn.addLayer() nn.addOutputLayer(activations='identity') for i, w in enumerate(W_skl): nn.weights[i] = w for i, b in enumerate(b_skl): nn.biases[i] = np.expand_dims(b, axis=1) # ======================================================================== n_samples, n_features = X.shape batch_size = n_samples hidden_layer_sizes = mlp.hidden_layer_sizes if not hasattr(hidden_layer_sizes, "__iter__"): hidden_layer_sizes = [hidden_layer_sizes] hidden_layer_sizes = list(hidden_layer_sizes) layer_units = ([n_features] + hidden_layer_sizes + [mlp.n_outputs_]) activations = [X] activations.extend( np.empty((batch_size, n_fan_out)) for n_fan_out in layer_units[1:]) deltas = [np.empty_like(a_layer) for a_layer in activations] coef_grads = [ np.empty((n_fan_in_, n_fan_out_)) for n_fan_in_, n_fan_out_ in zip(layer_units[:-1], layer_units[1:]) ] intercept_grads = [np.empty(n_fan_out_) for n_fan_out_ in layer_units[1:]] # ======================================================================== activations = mlp._forward_pass(activations) if y.ndim == 1: y = y.reshape((-1, 1)) loss, coef_grads, intercept_grads = mlp._backprop(X, y, activations, deltas, coef_grads, intercept_grads) yhat = nn.forward_pass(X.T) nn.backpropagation(yhat.T, y) for i, d_bias in enumerate(nn.d_biases): assert np.squeeze(d_bias) == pytest.approx( np.squeeze(intercept_grads[i])) for i, d_weight in enumerate(nn.d_weights): assert np.squeeze(d_weight) == pytest.approx(np.squeeze(coef_grads[i]))
def test_neuralNetwork_backpropagation(): # We re-use the test_neuralNetwork_network networks and this time check # that the computed backpropagation derivatives are equal. from sklearn.neural_network import MLPRegressor X = [[0.0], [1.0], [2.0], [3.0], [4.0], [5.0]] y = [0, 2, 4, 6, 8, 10] mlp = MLPRegressor(solver='sgd', alpha=0.0, hidden_layer_sizes=(3, 3), random_state=1, activation='logistic') # Force sklearn to set up all the matrices by fitting a data set. with warnings.catch_warnings(): warnings.simplefilter("ignore") mlp.fit(X, y) # Throw away all the fitted values, randomize W and b matrices. np.random.seed(18) for i, coeff in enumerate(mlp.coefs_): mlp.coefs_[i] = np.random.normal(size=coeff.shape) for i, bias in enumerate(mlp.intercepts_): mlp.intercepts_[i] = np.random.normal(size=bias.shape) W_skl = mlp.coefs_ b_skl = mlp.intercepts_ nn = NeuralNetwork(inputs=1, outputs=1, layers=3, neurons=3, activations='sigmoid', silent=False) nn.addLayer() nn.addLayer() nn.addOutputLayer(activations='identity') nn.weights = W_skl for i, b in enumerate(b_skl): nn.biases[i] = np.expand_dims(b, axis=1) # From the sklearn source, we need to set up some lists to use the _backprop # function in MLPRegressor, see: # # https://github.com/scikit-learn/scikit-learn/blob/bac89c2/sklearn/neural_network/multilayer_perceptron.py#L355 # # ======================================================================== # Initialize lists X = np.array([[1.125982598]]) y = np.array([8.29289285]) mlp.predict(X) n_samples, n_features = X.shape batch_size = n_samples hidden_layer_sizes = mlp.hidden_layer_sizes # Make sure self.hidden_layer_sizes is a list if not hasattr(hidden_layer_sizes, "__iter__"): hidden_layer_sizes = [hidden_layer_sizes] hidden_layer_sizes = list(hidden_layer_sizes) layer_units = ([n_features] + hidden_layer_sizes + [mlp.n_outputs_]) activations = [X] activations.extend( np.empty((batch_size, n_fan_out)) for n_fan_out in layer_units[1:]) deltas = [np.empty_like(a_layer) for a_layer in activations] coef_grads = [ np.empty((n_fan_in_, n_fan_out_)) for n_fan_in_, n_fan_out_ in zip(layer_units[:-1], layer_units[1:]) ] intercept_grads = [np.empty(n_fan_out_) for n_fan_out_ in layer_units[1:]] # ======================================================================== activations = mlp._forward_pass(activations) loss, coef_grads, intercept_grads = mlp._backprop(X, y, activations, deltas, coef_grads, intercept_grads) yhat = nn(X) nn.backpropagation(yhat, y) for i, d_bias in enumerate(nn.d_biases): assert np.squeeze(d_bias) == pytest.approx( np.squeeze(intercept_grads[i])) for i, d_weight in enumerate(nn.d_weights): assert np.squeeze(d_weight) == pytest.approx(np.squeeze(coef_grads[i]))
def train_net_predict_energy(L=10, N=5000): ising = Ising(L, N) X, y = ising.generateTrainingData1D() y /= L n_samples, n_features = X.shape nn = NeuralNetwork(inputs=L, neurons=L * L, outputs=1, activations='sigmoid', cost='mse', silent=False) nn.addLayer(neurons=L * L) nn.addLayer(neurons=L * L) nn.addOutputLayer(activations='identity') validation_skip = 10 epochs = 1000 nn.fit(X.T, y, shuffle=True, batch_size=1000, validation_fraction=0.2, learning_rate=0.001, verbose=False, silent=False, epochs=epochs, validation_skip=validation_skip, optimizer='adam') # Use the net to predict the energies for the validation set. x_validation = nn.x_validation y_validation = nn.predict(x_validation) target_validation = nn.target_validation # Sort the targets for better visualization of the network output. ind = np.argsort(target_validation) y_validation = np.squeeze(y_validation.T[ind]) target_validation = np.squeeze(target_validation.T[ind]) # We dont want to plot the discontinuities in the target. target_validation[np.where( np.abs(np.diff(target_validation)) > 1e-5)] = np.nan plt.rc('text', usetex=True) plt.figure() plt.plot(target_validation, 'k--', label=r'Target') plt.plot(y_validation, 'r.', markersize=0.5, label=r'NN output') plt.legend(fontsize=10) plt.xlabel(r'Validation sample', fontsize=10) plt.ylabel(r'$E / L$', fontsize=10) #plt.savefig(os.path.join(os.path.dirname(__file__), 'figures', 'nn_1d_energy_predict' + str(L) + '.png'), transparent=True, bbox_inches='tight') # Plot the training / validation loss during training. training_loss = nn.training_loss validation_loss = nn.validation_loss # There are more training loss values than validation loss values, lets # align them so the plot makes sense. xaxis_validation_loss = np.zeros_like(validation_loss) xaxis_validation_loss[0] = 0 xaxis_validation_loss[1:-1] = np.arange(validation_skip, len(training_loss), validation_skip) xaxis_validation_loss[-1] = len(training_loss) plt.figure() plt.semilogy(training_loss, 'r-', label=r'Training loss') plt.semilogy(xaxis_validation_loss, validation_loss, 'k--', label=r'Validation loss') plt.legend(fontsize=10) plt.xlabel(r'Epoch', fontsize=10) plt.ylabel(r'Cost $C(\theta)$', fontsize=10) #plt.savefig(os.path.join(os.path.dirname(__file__), 'figures', 'nn_1d_loss' + str(L) + '.png'), transparent=True, bbox_inches='tight') plt.show()
def R2_versus_lasso(): L = 3 N = 10000 training_fraction = 0.4 ising = Ising(L, N) D, ry = ising.generateDesignMatrix1D() X, y = ising.generateTrainingData1D() y /= L D_train = D[int(training_fraction * N):, :] ry_train = ry[int(training_fraction * N):] D_validation = D[:int(training_fraction * N), :] ry_validation = ry[:int(training_fraction * N)] lasso = LeastSquares(method='lasso', backend='skl') lasso.setLambda(1e-2) lasso.fit(D_train, ry_train) lasso.y = ry_validation lasso_R2 = sklearn.metrics.mean_squared_error( ry_validation / L, lasso.predict(D_validation) / L) n_samples, n_features = X.shape nn = NeuralNetwork(inputs=L * L, neurons=L, outputs=1, activations='identity', cost='mse', silent=False) nn.addLayer(neurons=1) nn.addOutputLayer(activations='identity') validation_skip = 100 epochs = 50000 nn.fit(D.T, ry, shuffle=True, batch_size=2000, validation_fraction=1 - training_fraction, learning_rate=0.0001, verbose=False, silent=False, epochs=epochs, validation_skip=validation_skip, optimizer='adam') plt.rc('text', usetex=True) validation_loss = nn.validation_loss_improving validation_ep = np.linspace(0, epochs, len(nn.validation_loss_improving)) plt.semilogy(validation_ep, validation_loss, 'r-', label=r'NN') plt.semilogy([0, epochs], np.array([lasso_R2, lasso_R2]), 'k--', label=r'Lasso') plt.xlabel(r'Epoch', fontsize=10) plt.ylabel(r'Mean squared error', fontsize=10) plt.legend(fontsize=10) plt.xlim((0, epochs)) ax = plt.gca() ymin, ymax = ax.get_ylim() if ymin > pow(10, -5): ymin = pow(10, -5) #plt.ylim((ymin,ymax)) plt.savefig(os.path.join(os.path.dirname(__file__), 'figures', 'NN_compare_lasso.png'), transparent=True, bbox_inches='tight')
import numpy as np import tensorflow as tf from dataset import Dataset from neuralNetwork import NeuralNetwork from softMaxClassifier import SoftMaxClassifier if __name__ == "__main__": dataset = Dataset() neuralNetwork = NeuralNetwork() neuralNetwork.addLayer(10, len(dataset.XTrain[0])) softMaxClassifier = SoftMaxClassifier(neuralNetwork, 0.1, 50) softMaxClassifier.gradientDescent(dataset.XTrain, dataset.yTrain)