def fizzbuzz_accuracy(low: int, hi: int, net: Layer) -> float: num_correct = 0 for n in range(low, hi): x = binary_encode(n) predicted = argmax(net.forward(x)) actual = argmax(fizz_buzz_encode(n)) if predicted == actual: num_correct += 1 return num_correct / (hi - low)
def fizzbuzz_accuracy(low: int, hi: int, net: Layer) -> float: num_correct = 0 for n in range(low, hi): x = binary_encode(n) predicted = argmax(net.forward(x)) actual = argmax(fizz_buzz_encode(n)) if predicted == actual: num_correct += 1 return num_correct / (hi - low)
def main(): # XOR revisited # training data xs = [[0., 0], [0., 1], [1., 0], [1., 1]] ys = [[0.], [1.], [1.], [0.]] random.seed(0) net = Sequential([ Linear(input_dim=2, output_dim=2), Sigmoid(), Linear(input_dim=2, output_dim=1) ]) import tqdm optimizer = GradientDescent(learning_rate=0.1) loss = SSE() with tqdm.trange(3000) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) t.set_description(f"xor loss {epoch_loss:.3f}") for param in net.params(): print(param) # FizzBuzz Revisited from scratch.neural_networks import binary_encode, fizz_buzz_encode, argmax xs = [binary_encode(n) for n in range(101, 1024)] ys = [fizz_buzz_encode(n) for n in range(101, 1024)] NUM_HIDDEN = 25 random.seed(0) net = Sequential([ Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'), Tanh(), Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform'), Sigmoid() ]) def fizzbuzz_accuracy(low: int, hi: int, net: Layer) -> float: num_correct = 0 for n in range(low, hi): x = binary_encode(n) predicted = argmax(net.forward(x)) actual = argmax(fizz_buzz_encode(n)) if predicted == actual: num_correct += 1 return num_correct / (hi - low) optimizer = Momentum(learning_rate=0.1, momentum=0.9) loss = SSE() with tqdm.trange(1000) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) accuracy = fizzbuzz_accuracy(101, 1024, net) t.set_description(f"fb loss: {epoch_loss:.2f} acc: {accuracy:.2f}") # Now check results on the test set print("test results", fizzbuzz_accuracy(1, 101, net)) random.seed(0) net = Sequential([ Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'), Tanh(), Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform') # No final sigmoid layer now ]) optimizer = Momentum(learning_rate=0.1, momentum=0.9) loss = SoftmaxCrossEntropy() with tqdm.trange(100) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) accuracy = fizzbuzz_accuracy(101, 1024, net) t.set_description(f"fb loss: {epoch_loss:.3f} acc: {accuracy:.2f}") # Again check results on the test set print("test results", fizzbuzz_accuracy(1, 101, net)) # Load the MNIST data import mnist # This will download the data, change this to where you want it. # (Yes, it's a 0-argument function, that's what the library expects.) # (Yes, I'm assigning a lambda to a variable, like I said never to do.) mnist.temporary_dir = lambda: '/tmp' # Each of these functions first downloads the data and returns a numpy array. # We call .tolist() because our "tensors" are just lists. train_images = mnist.train_images().tolist() train_labels = mnist.train_labels().tolist() assert shape(train_images) == [60000, 28, 28] assert shape(train_labels) == [60000] import matplotlib.pyplot as plt fig, ax = plt.subplots(10, 10) for i in range(10): for j in range(10): # Plot each image in black and white and hide the axes. ax[i][j].imshow(train_images[10 * i + j], cmap='Greys') ax[i][j].xaxis.set_visible(False) ax[i][j].yaxis.set_visible(False) # plt.show() # Load the MNIST test data test_images = mnist.test_images().tolist() test_labels = mnist.test_labels().tolist() assert shape(test_images) == [10000, 28, 28] assert shape(test_labels) == [10000] # Recenter the images # Compute the average pixel value avg = tensor_sum(train_images) / 60000 / 28 / 28 # Recenter, rescale, and flatten train_images = [[(pixel - avg) / 256 for row in image for pixel in row] for image in train_images] test_images = [[(pixel - avg) / 256 for row in image for pixel in row] for image in test_images] assert shape(train_images) == [60000, 784], "images should be flattened" assert shape(test_images) == [10000, 784], "images should be flattened" # After centering, average pixel should be very close to 0 assert -0.0001 < tensor_sum(train_images) < 0.0001 # One-hot encode the test data train_labels = [one_hot_encode(label) for label in train_labels] test_labels = [one_hot_encode(label) for label in test_labels] assert shape(train_labels) == [60000, 10] assert shape(test_labels) == [10000, 10] # Training loop import tqdm def loop(model: Layer, images: List[Tensor], labels: List[Tensor], loss: Loss, optimizer: Optimizer = None) -> None: correct = 0 # Track number of correct predictions. total_loss = 0.0 # Track total loss. with tqdm.trange(len(images)) as t: for i in t: predicted = model.forward(images[i]) # Predict. if argmax(predicted) == argmax(labels[i]): # Check for correct += 1 # correctness. total_loss += loss.loss(predicted, labels[i]) # Compute loss. # If we're training, backpropagate gradient and update weights. if optimizer is not None: gradient = loss.gradient(predicted, labels[i]) model.backward(gradient) optimizer.step(model) # And update our metrics in the progress bar. avg_loss = total_loss / (i + 1) acc = correct / (i + 1) t.set_description(f"mnist loss: {avg_loss:.3f} acc: {acc:.3f}") # The logistic regression model for MNIST random.seed(0) # Logistic regression is just a linear layer followed by softmax model = Linear(784, 10) loss = SoftmaxCrossEntropy() # This optimizer seems to work optimizer = Momentum(learning_rate=0.01, momentum=0.99) # Train on the training data loop(model, train_images, train_labels, loss, optimizer) # Test on the test data (no optimizer means just evaluate) loop(model, test_images, test_labels, loss) # A deep neural network for MNIST random.seed(0) # Name them so we can turn train on and off dropout1 = Dropout(0.1) dropout2 = Dropout(0.1) model = Sequential([ Linear(784, 30), # Hidden layer 1: size 30 dropout1, Tanh(), Linear(30, 10), # Hidden layer 2: size 10 dropout2, Tanh(), Linear(10, 10) # Output layer: size 10 ]) # Training the deep model for MNIST optimizer = Momentum(learning_rate=0.01, momentum=0.99) loss = SoftmaxCrossEntropy() # Enable dropout and train (takes > 20 minutes on my laptop!) dropout1.train = dropout2.train = True loop(model, train_images, train_labels, loss, optimizer) # Disable dropout and evaluate dropout1.train = dropout2.train = False loop(model, test_images, test_labels, loss)
def main(): # XOR revisited # training data xs = [[0., 0], [0., 1], [1., 0], [1., 1]] ys = [[0.], [1.], [1.], [0.]] random.seed(0) net = Sequential([ Linear(input_dim=2, output_dim=2), Sigmoid(), Linear(input_dim=2, output_dim=1) ]) import tqdm optimizer = GradientDescent(learning_rate=0.1) loss = SSE() with tqdm.trange(3000) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) t.set_description(f"xor loss {epoch_loss:.3f}") for param in net.params(): print(param) # FizzBuzz Revisited from scratch.neural_networks import binary_encode, fizz_buzz_encode, argmax xs = [binary_encode(n) for n in range(101, 1024)] ys = [fizz_buzz_encode(n) for n in range(101, 1024)] NUM_HIDDEN = 25 random.seed(0) net = Sequential([ Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'), Tanh(), Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform'), Sigmoid() ]) def fizzbuzz_accuracy(low: int, hi: int, net: Layer) -> float: num_correct = 0 for n in range(low, hi): x = binary_encode(n) predicted = argmax(net.forward(x)) actual = argmax(fizz_buzz_encode(n)) if predicted == actual: num_correct += 1 return num_correct / (hi - low) optimizer = Momentum(learning_rate=0.1, momentum=0.9) loss = SSE() with tqdm.trange(1000) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) accuracy = fizzbuzz_accuracy(101, 1024, net) t.set_description(f"fb loss: {epoch_loss:.2f} acc: {accuracy:.2f}") # Now check results on the test set print("test results", fizzbuzz_accuracy(1, 101, net)) random.seed(0) net = Sequential([ Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'), Tanh(), Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform') # No final sigmoid layer now ]) optimizer = Momentum(learning_rate=0.1, momentum=0.9) loss = SoftmaxCrossEntropy() with tqdm.trange(100) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) accuracy = fizzbuzz_accuracy(101, 1024, net) t.set_description(f"fb loss: {epoch_loss:.3f} acc: {accuracy:.2f}") # Again check results on the test set print("test results", fizzbuzz_accuracy(1, 101, net)) # Load the MNIST data import mnist # This will download the data, change this to where you want it. # (Yes, it's a 0-argument function, that's what the library expects.) # (Yes, I'm assigning a lambda to a variable, like I said never to do.) mnist.temporary_dir = lambda: '/tmp' # Each of these functions first downloads the data and returns a numpy array. # We call .tolist() because our "tensors" are just lists. train_images = mnist.train_images().tolist() train_labels = mnist.train_labels().tolist() assert shape(train_images) == [60000, 28, 28] assert shape(train_labels) == [60000] import matplotlib.pyplot as plt fig, ax = plt.subplots(10, 10) for i in range(10): for j in range(10): # Plot each image in black and white and hide the axes. ax[i][j].imshow(train_images[10 * i + j], cmap='Greys') ax[i][j].xaxis.set_visible(False) ax[i][j].yaxis.set_visible(False) # plt.show() # Load the MNIST test data test_images = mnist.test_images().tolist() test_labels = mnist.test_labels().tolist() assert shape(test_images) == [10000, 28, 28] assert shape(test_labels) == [10000] # Recenter the images # Compute the average pixel value avg = tensor_sum(train_images) / 60000 / 28 / 28 # Recenter, rescale, and flatten train_images = [[(pixel - avg) / 256 for row in image for pixel in row] for image in train_images] test_images = [[(pixel - avg) / 256 for row in image for pixel in row] for image in test_images] assert shape(train_images) == [60000, 784], "images should be flattened" assert shape(test_images) == [10000, 784], "images should be flattened" # After centering, average pixel should be very close to 0 assert -0.0001 < tensor_sum(train_images) < 0.0001 # One-hot encode the test data train_labels = [one_hot_encode(label) for label in train_labels] test_labels = [one_hot_encode(label) for label in test_labels] assert shape(train_labels) == [60000, 10] assert shape(test_labels) == [10000, 10] # Training loop import tqdm def loop(model: Layer, images: List[Tensor], labels: List[Tensor], loss: Loss, optimizer: Optimizer = None) -> None: correct = 0 # Track number of correct predictions. total_loss = 0.0 # Track total loss. with tqdm.trange(len(images)) as t: for i in t: predicted = model.forward(images[i]) # Predict. if argmax(predicted) == argmax(labels[i]): # Check for correct += 1 # correctness. total_loss += loss.loss(predicted, labels[i]) # Compute loss. # If we're training, backpropagate gradient and update weights. if optimizer is not None: gradient = loss.gradient(predicted, labels[i]) model.backward(gradient) optimizer.step(model) # And update our metrics in the progress bar. avg_loss = total_loss / (i + 1) acc = correct / (i + 1) t.set_description(f"mnist loss: {avg_loss:.3f} acc: {acc:.3f}") # The logistic regression model for MNIST random.seed(0) # Logistic regression is just a linear layer followed by softmax model = Linear(784, 10) loss = SoftmaxCrossEntropy() # This optimizer seems to work optimizer = Momentum(learning_rate=0.01, momentum=0.99) # Train on the training data loop(model, train_images, train_labels, loss, optimizer) # Test on the test data (no optimizer means just evaluate) loop(model, test_images, test_labels, loss) # A deep neural network for MNIST random.seed(0) # Name them so we can turn train on and off dropout1 = Dropout(0.1) dropout2 = Dropout(0.1) model = Sequential([ Linear(784, 30), # Hidden layer 1: size 30 dropout1, Tanh(), Linear(30, 10), # Hidden layer 2: size 10 dropout2, Tanh(), Linear(10, 10) # Output layer: size 10 ]) # Training the deep model for MNIST optimizer = Momentum(learning_rate=0.01, momentum=0.99) loss = SoftmaxCrossEntropy() # Enable dropout and train (takes > 20 minutes on my laptop!) dropout1.train = dropout2.train = True loop(model, train_images, train_labels, loss, optimizer) # Disable dropout and evaluate dropout1.train = dropout2.train = False loop(model, test_images, test_labels, loss)
def main(): # kolejne podejście do bramki XOR # dane treningowe xs = [[0., 0], [0., 1], [1., 0], [1., 1]] ys = [[0.], [1.], [1.], [0.]] random.seed(0) net = Sequential([ Linear(input_dim=2, output_dim=2), Sigmoid(), Linear(input_dim=2, output_dim=1) ]) import tqdm optimizer = GradientDescent(learning_rate=0.1) loss = SSE() with tqdm.trange(3000) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) t.set_description(f"xor loss {epoch_loss:.3f}") for param in net.params(): print(param) # kolejne podejście do gry Fizz Buzz from scratch.neural_networks import binary_encode, fizz_buzz_encode, argmax xs = [binary_encode(n) for n in range(101, 1024)] ys = [fizz_buzz_encode(n) for n in range(101, 1024)] NUM_HIDDEN = 25 random.seed(0) net = Sequential([ Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'), Tanh(), Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform'), Sigmoid() ]) def fizzbuzz_accuracy(low: int, hi: int, net: Layer) -> float: num_correct = 0 for n in range(low, hi): x = binary_encode(n) predicted = argmax(net.forward(x)) actual = argmax(fizz_buzz_encode(n)) if predicted == actual: num_correct += 1 return num_correct / (hi - low) optimizer = Momentum(learning_rate=0.1, momentum=0.9) loss = SSE() with tqdm.trange(1000) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) accuracy = fizzbuzz_accuracy(101, 1024, net) t.set_description(f"fb loss: {epoch_loss:.2f} acc: {accuracy:.2f}") # teraz sprawdźmy wyniki na zbiorze testowym print("test results", fizzbuzz_accuracy(1, 101, net)) random.seed(0) net = Sequential([ Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'), Tanh(), Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform') # teraz bez końcowej warstwy sigmoid ]) optimizer = Momentum(learning_rate=0.1, momentum=0.9) loss = SoftmaxCrossEntropy() with tqdm.trange(100) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) accuracy = fizzbuzz_accuracy(101, 1024, net) t.set_description(f"fb loss: {epoch_loss:.3f} acc: {accuracy:.2f}") # ponownie sprawdzamy wyniki na zbiorze testowym print("test results", fizzbuzz_accuracy(1, 101, net)) # Pobranie danych MNIST import mnist # Ten fragment pobiera dane. Zmień ścieżkę na taką, jaką chcesz. # (Tak, to jest funkcja, która nie ma argumentów, tego właśnie oczekuje ta biblioteka). # (Tak, przypisuję lambdę do zmiennej, mimo że mówiłem, że nie należy tak robić). mnist.temporary_dir = lambda: '/tmp' # Każda z tych funkcji pobiera dane i zwraca tablicę numpy. # Wywołujemy .tolist(), ponieważ nasze "tensory" są tak naprawdę listami. train_images = mnist.train_images().tolist() train_labels = mnist.train_labels().tolist() assert shape(train_images) == [60000, 28, 28] assert shape(train_labels) == [60000] import matplotlib.pyplot as plt fig, ax = plt.subplots(10, 10) for i in range(10): for j in range(10): # Wyświetl każdy obraz jako czarno-biały i ukryj osie. ax[i][j].imshow(train_images[10 * i + j], cmap='Greys') ax[i][j].xaxis.set_visible(False) ax[i][j].yaxis.set_visible(False) plt.show() # Ładowanie danych testowych MNIST test_images = mnist.test_images().tolist() test_labels = mnist.test_labels().tolist() assert shape(test_images) == [10000, 28, 28] assert shape(test_labels) == [10000] # Centrowanie obrazów # Policz średnią wartość piksela avg = tensor_sum(train_images) / 60000 / 28 / 28 # Centrowanie, skalowanie i spłaszczanie train_images = [[(pixel - avg) / 256 for row in image for pixel in row] for image in train_images] test_images = [[(pixel - avg) / 256 for row in image for pixel in row] for image in test_images] assert shape(train_images) == [60000, 784], "obrazy powinny być spłaszczone" assert shape(test_images) == [10000, 784], "obrazy powinny być spłaszczone" # po centrowaniu wartość każdego piksela powinna być bliska 0 assert -0.0001 < tensor_sum(train_images) < 0.0001 # Kodowanie 1 z n, dane testowe train_labels = [one_hot_encode(label) for label in train_labels] test_labels = [one_hot_encode(label) for label in test_labels] assert shape(train_labels) == [60000, 10] assert shape(test_labels) == [10000, 10] # Pętla trenująca import tqdm def loop(model: Layer, images: List[Tensor], labels: List[Tensor], loss: Loss, optimizer: Optimizer = None) -> None: correct = 0 # Przechowuje liczbę poprawnych przewidywań. total_loss = 0.0 # Przechowuje całkowitą stratę. with tqdm.trange(len(images)) as t: for i in t: predicted = model.forward( images[i]) # Określ wartości przewidywane. if argmax(predicted) == argmax( labels[i]): # Sprawdź poprawność. correct += 1 total_loss += loss.loss(predicted, labels[i]) # Oblicz stratę. # Podczas treningu propaguj wstecznie gradient i zaktualizuj wagi. if optimizer is not None: gradient = loss.gradient(predicted, labels[i]) model.backward(gradient) optimizer.step(model) # Zaktualizuj metryki na pasku postępu. avg_loss = total_loss / (i + 1) acc = correct / (i + 1) t.set_description(f"mnist loss: {avg_loss:.3f} acc: {acc:.3f}") # Model regresji logistycznej na danych MNIST random.seed(0) # Regresja logistyczna to po prostu warstwa liniowa wraz z funkcją softmax model = Linear(784, 10) loss = SoftmaxCrossEntropy() # Ten optymalizator wydaje się działać poprawnie optimizer = Momentum(learning_rate=0.01, momentum=0.99) # Trenowanie modelu na danych treningowych loop(model, train_images, train_labels, loss, optimizer) # Testowanie na danych testowych (dlatego brak optymalizatora) loop(model, test_images, test_labels, loss) # Głęboka sieć neuronowa na danych MNIST random.seed(0) # Nazywamy je, żeby mieć możliwość włączania i wyłączania. dropout1 = Dropout(0.1) dropout2 = Dropout(0.1) model = Sequential([ Linear(784, 30), # Ukryta warstwa 1: rozmiar 30 dropout1, Tanh(), Linear(30, 10), # Ukryta warstwa 2: rozmiar 10 dropout2, Tanh(), Linear(10, 10) # Warstwa wyjściowa: rozmiar 10 ]) # Trenowanie modelu głębokiego na danych MNIST optimizer = Momentum(learning_rate=0.01, momentum=0.99) loss = SoftmaxCrossEntropy() # Włącz warstwę dropout i rozpocznij trening (na moim laptopie zajęło to ponad 20 minut!) dropout1.train = dropout2.train = True loop(model, train_images, train_labels, loss, optimizer) # Wyłącz warstwę dropout i przetestuj dropout1.train = dropout2.train = False loop(model, test_images, test_labels, loss)