def load_dataset():
# We first define some helper functions for supporting both Python 2 and 3.
if sys.version_info[0] == 2:
from urllib import urlretrieve
import cPickle as pickle
def pickle_load(f, encoding):
return pickle.load(f)
else:
from urllib.request import urlretrieve
import pickle
def pickle_load(f, encoding):
return pickle.load(f, encoding=encoding)
# We'll now download the MNIST dataset if it is not yet available.
url = 'http://deeplearning.net/data/mnist/mnist.pkl.gz'
filename = 'mnist.pkl.gz'
if not os.path.exists(filename):
print("Downloading MNIST dataset...")
urlretrieve(url, filename)
# We'll then load and unpickle the file.
import gzip
with gzip.open(filename, 'rb') as f:
data = pickle_load(f, encoding='latin-1')
# The MNIST dataset we have here consists of six numpy arrays:
# Inputs and targets for the training set, validation set and test set.
X_train, y_train = data[0]
X_val, y_val = data[1]
X_test, y_test = data[2]
# The inputs come as vectors, we reshape them to monochrome 2D images,
# according to the shape convention: (examples, channels, rows, columns)
X_train = X_train.reshape((-1, 1, 28, 28))
X_val = X_val.reshape((-1, 1, 28, 28))
X_test = X_test.reshape((-1, 1, 28, 28))
# The targets are int64, we cast them to int8 for GPU compatibility.
y_train = y_train.astype(np.uint8)
y_val = y_val.astype(np.uint8)
y_test = y_test.astype(np.uint8)
y_train = numpy_one_hot(y_train, 10)
y_val = numpy_one_hot(y_val, 10)
y_test = numpy_one_hot(y_test, 10)
# We just return all the arrays in order, as expected in main().
# (It doesn't matter how we do this as long as we can read them again.)
dataset = dict(
X_train=X_train,
y_train=y_train,
X_valid=X_val,
y_valid=y_val,
X_test=X_test,
y_test=y_test
)
return dataset
def load_dataset():
# We first define some helper functions for supporting both Python 2 and 3.
if sys.version_info[0] == 2:
from urllib import urlretrieve
import cPickle as pickle
def pickle_load(f, encoding):
return pickle.load(f)
else:
from urllib.request import urlretrieve
import pickle
def pickle_load(f, encoding):
return pickle.load(f, encoding=encoding)
# We'll now download the MNIST dataset if it is not yet available.
url = 'http://deeplearning.net/data/mnist/mnist.pkl.gz'
filename = 'mnist.pkl.gz'
if not os.path.exists(filename):
print("Downloading MNIST dataset...")
urlretrieve(url, filename)
# We'll then load and unpickle the file.
import gzip
with gzip.open(filename, 'rb') as f:
data = pickle_load(f, encoding='latin-1')
# The MNIST dataset we have here consists of six numpy arrays:
# Inputs and targets for the training set, validation set and test set.
X_train, y_train = data[0]
X_val, y_val = data[1]
X_test, y_test = data[2]
# The inputs come as vectors, we reshape them to monochrome 2D images,
# according to the shape convention: (examples, channels, rows, columns)
X_train = X_train.reshape((-1, 1, 28, 28))
X_val = X_val.reshape((-1, 1, 28, 28))
X_test = X_test.reshape((-1, 1, 28, 28))
# The targets are int64, we cast them to int8 for GPU compatibility.
y_train = y_train.astype(np.uint8)
y_val = y_val.astype(np.uint8)
y_test = y_test.astype(np.uint8)
y_train = numpy_one_hot(y_train, 10)
y_val = numpy_one_hot(y_val, 10)
y_test = numpy_one_hot(y_test, 10)
# We just return all the arrays in order, as expected in main().
# (It doesn't matter how we do this as long as we can read them again.)
dataset = dict(X_train=X_train,
y_train=y_train,
X_valid=X_val,
y_valid=y_val,
X_test=X_test,
y_test=y_test)
return dataset
def test_one_hot(): a = [0,1,3,9,-1,6,2,2,10] expected_out = [[ 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],[ 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],[ 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],[ 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.],[ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],[ 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],[ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],[ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],[ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]] actual_out = utils.numpy_one_hot(a) assert np.all(actual_out == expected_out)
