def build_dataset(filename, max_lines=-1):
"""Loads a text file, and turns each line into an encoded sequence."""
encodings = dict(map(reversed, enumerate(string.printable)))
digitize = lambda char: encodings[char] if char in encodings else len(encodings)
encode_line = lambda line: np.array(list(map(digitize, line)))
nonblank_line = lambda line: len(line) > 2
with open(filename) as f:
lines = f.readlines()
encoded_lines = map(encode_line, filter(nonblank_line, lines)[:max_lines])
num_outputs = len(encodings) + 1
return encoded_lines, num_outputs
def load_mnist():
print("Loading training data...")
import imp, urllib
partial_flatten = lambda x : np.reshape(x, (x.shape[0], np.prod(x.shape[1:])))
one_hot = lambda x, K: np.array(x[:,None] == np.arange(K)[None, :], dtype=int)
source, _ = urllib.urlretrieve(
'https://raw.githubusercontent.com/HIPS/Kayak/master/examples/data.py')
data = imp.load_source('data', source).mnist()
train_images, train_labels, test_images, test_labels = data
train_images = partial_flatten(train_images) / 255.0
test_images = partial_flatten(test_images) / 255.0
train_labels = one_hot(train_labels, 10)
test_labels = one_hot(test_labels, 10)
N_data = train_images.shape[0]
return N_data, train_images, train_labels, test_images, test_labels
def unary_nd(f, x, eps=EPS):
if isinstance(x, array_types):
if np.iscomplexobj(x):
nd_grad = np.zeros(x.shape) + 0j
elif isinstance(x, garray_obj):
nd_grad = np.array(np.zeros(x.shape), dtype=np.gpu_float32)
else:
nd_grad = np.zeros(x.shape)
for dims in it.product(*list(map(range, x.shape))):
nd_grad[dims] = unary_nd(indexed_function(f, x, dims), x[dims])
return nd_grad
elif isinstance(x, tuple):
return tuple([unary_nd(indexed_function(f, tuple(x), i), x[i])
for i in range(len(x))])
elif isinstance(x, dict):
return {k : unary_nd(indexed_function(f, x, k), v) for k, v in iteritems(x)}
elif isinstance(x, list):
return [unary_nd(indexed_function(f, x, i), v) for i, v in enumerate(x)]
elif np.iscomplexobj(x):
result = (f(x + eps/2) - f(x - eps/2)) / eps \
- 1j*(f(x + 1j*eps/2) - f(x - 1j*eps/2)) / eps
return type(safe_type(x))(result)
else:
return type(safe_type(x))((f(x + eps/2) - f(x - eps/2)) / eps)
def sigmoid(x):
return 0.5*(np.tanh(x) + 1)
def logistic_predictions(weights, inputs):
# Outputs probability of a label being true according to logistic model.
return sigmoid(np.dot(inputs, weights))
def training_loss(weights):
# Training loss is the negative log-likelihood of the training labels.
preds = logistic_predictions(weights, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
return -np.sum(np.log(label_probabilities))
# Build a toy dataset.
inputs = np.array([[0.52, 1.12, 0.77],
[0.88, -1.08, 0.15],
[0.52, 0.06, -1.30],
[0.74, -2.49, 1.39]])
targets = np.array([True, True, False, True])
# Build a function that returns gradients of training loss using autogradwithbay.
training_gradient_fun = grad(training_loss)
# Check the gradients numerically, just to be safe.
weights = np.array([0.0, 0.0, 0.0])
quick_grad_check(training_loss, weights)
# Optimize weights using gradient descent.
print("Initial loss:", training_loss(weights))
for i in range(100):
weights -= training_gradient_fun(weights) * 0.01
tanh_layer(120),
tanh_layer(84),
softmax_layer(10)]
# Training parameters
param_scale = 0.1
learning_rate = 1e-3
momentum = 0.9
batch_size = 256
num_epochs = 50
# Load and process MNIST data (borrowing from Kayak)
print("Loading training data...")
import imp, urllib
add_color_channel = lambda x : x.reshape((x.shape[0], 1, x.shape[1], x.shape[2]))
one_hot = lambda x, K : np.array(x[:,None] == np.arange(K)[None, :], dtype=int)
source, _ = urllib.urlretrieve(
'https://raw.githubusercontent.com/HIPS/Kayak/master/examples/data.py')
data = imp.load_source('data', source).mnist()
train_images, train_labels, test_images, test_labels = data
train_images = add_color_channel(train_images) / 255.0
test_images = add_color_channel(test_images) / 255.0
train_labels = one_hot(train_labels, 10)
test_labels = one_hot(test_labels, 10)
N_data = train_images.shape[0]
# Make neural net functions
N_weights, pred_fun, loss_fun, frac_err = make_nn_funs(input_shape, layer_specs, L2_reg)
loss_grad = grad(loss_fun)
# Initialize weights
def string_to_one_hot(string, maxchar):
"""Converts an ASCII string to a one-of-k encoding."""
ascii = np.array([ord(c) for c in string]).T
return np.array(ascii[:,None] == np.arange(maxchar)[None, :], dtype=int)
from __future__ import absolute_import
from __future__ import print_function
import autogradwithbay.numpy as np
from autogradwithbay import value_and_grad
from scipy.optimize import minimize
def rosenbrock(x):
return 100*(x[1] - x[0]**2)**2 + (1 - x[0])**2
# Build a function that also returns gradients using autogradwithbay.
rosenbrock_with_grad = value_and_grad(rosenbrock)
# Optimize using conjugate gradients.
result = minimize(rosenbrock_with_grad, x0=np.array([0.0, 0.0]), jac=True, method='CG')
print("Found minimum at {0}".format(result.x))
