def test_in_memory():
skip_if_not_available(datasets=['mnist.hdf5'])
# Load MNIST and get two batches
mnist = MNIST('train', load_in_memory=True)
data_stream = DataStream(mnist,
iteration_scheme=SequentialScheme(
examples=mnist.num_examples, batch_size=256))
epoch = data_stream.get_epoch_iterator()
for i, (features, targets) in enumerate(epoch):
if i == 1:
break
handle = mnist.open()
known_features, _ = mnist.get_data(handle, slice(256, 512))
mnist.close(handle)
assert numpy.all(features == known_features)
# Pickle the epoch and make sure that the data wasn't dumped
with tempfile.NamedTemporaryFile(delete=False) as f:
filename = f.name
cPickle.dump(epoch, f)
assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB
# Reload the epoch and make sure that the state was maintained
del epoch
with open(filename, 'rb') as f:
epoch = cPickle.load(f)
features, targets = next(epoch)
handle = mnist.open()
known_features, _ = mnist.get_data(handle, slice(512, 768))
mnist.close(handle)
assert numpy.all(features == known_features)
def test_in_memory():
skip_if_not_available(datasets=['mnist.hdf5'])
# Load MNIST and get two batches
mnist = MNIST(('train',), load_in_memory=True)
data_stream = DataStream(mnist, iteration_scheme=SequentialScheme(
examples=mnist.num_examples, batch_size=256))
epoch = data_stream.get_epoch_iterator()
for i, (features, targets) in enumerate(epoch):
if i == 1:
break
handle = mnist.open()
known_features, _ = mnist.get_data(handle, slice(256, 512))
mnist.close(handle)
assert numpy.all(features == known_features)
# Pickle the epoch and make sure that the data wasn't dumped
with tempfile.NamedTemporaryFile(delete=False) as f:
filename = f.name
cPickle.dump(epoch, f)
assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB
# Reload the epoch and make sure that the state was maintained
del epoch
with open(filename, 'rb') as f:
epoch = cPickle.load(f)
features, targets = next(epoch)
handle = mnist.open()
known_features, _ = mnist.get_data(handle, slice(512, 768))
mnist.close(handle)
assert numpy.all(features == known_features)
def test_mnist_test():
skip_if_not_available(datasets=["mnist.hdf5"])
dataset = MNIST(("test",), load_in_memory=False)
handle = dataset.open()
data, labels = dataset.get_data(handle, slice(0, 10))
assert data.dtype == "uint8"
assert data.shape == (10, 1, 28, 28)
assert labels.shape == (10, 1)
known = numpy.array([0, 0, 0, 0, 0, 0, 84, 185, 159, 151, 60, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
assert_allclose(data[0][0][7], known)
assert labels[0][0] == 7
assert dataset.num_examples == 10000
dataset.close(handle)
stream = DataStream.default_stream(dataset, iteration_scheme=SequentialScheme(10, 10))
data = next(stream.get_epoch_iterator())[0]
assert data.min() >= 0.0 and data.max() <= 1.0
assert data.dtype == config.floatX
def test_mnist_train():
skip_if_not_available(datasets=['mnist.hdf5'])
dataset = MNIST('train', load_in_memory=False)
handle = dataset.open()
data, labels = dataset.get_data(handle, slice(0, 10))
assert data.dtype == 'uint8'
assert data.shape == (10, 1, 28, 28)
assert labels.shape == (10, 1)
known = numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 30, 36, 94, 154, 170, 253,
253, 253, 253, 253, 225, 172, 253, 242, 195, 64, 0,
0, 0, 0])
assert_allclose(data[0][0][6], known)
assert labels[0][0] == 5
assert dataset.num_examples == 60000
dataset.close(handle)
stream = DataStream.default_stream(
dataset, iteration_scheme=SequentialScheme(10, 10))
data = next(stream.get_epoch_iterator())[0]
assert data.min() >= 0.0 and data.max() <= 1.0
assert data.dtype == config.floatX
def test_mnist_test():
skip_if_not_available(datasets=['mnist.hdf5'])
dataset = MNIST('test', load_in_memory=False)
handle = dataset.open()
data, labels = dataset.get_data(handle, slice(0, 10))
assert data.dtype == 'uint8'
assert data.shape == (10, 1, 28, 28)
assert labels.shape == (10, 1)
known = numpy.array([
0, 0, 0, 0, 0, 0, 84, 185, 159, 151, 60, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0
])
assert_allclose(data[0][0][7], known)
assert labels[0][0] == 7
assert dataset.num_examples == 10000
dataset.close(handle)
stream = DataStream.default_stream(dataset,
iteration_scheme=SequentialScheme(
10, 10))
data = next(stream.get_epoch_iterator())[0]
assert data.min() >= 0.0 and data.max() <= 1.0
assert data.dtype == config.floatX
#https://raw.githubusercontent.com/Newmu/Theano-Tutorials/master/2_logistic_regression.py
import theano
from theano import tensor as T
import numpy as np
from fuel.datasets import MNIST
from matplotlib import pyplot, cm
dataset = MNIST(('train',), sources=('features',))
state = dataset.open()
image, = dataset.get_data(state=state, request=[1234])
pyplot.imshow(image.reshape((28, 28)), cmap=cm.Greys_r, interpolation='nearest')
pyplot.show()
dataset.close(state)
def floatX(X):
return np.asarray(X, dtype=theano.config.floatX)
def init_weights(shape):
return theano.shared(floatX(np.random.randn(*shape) * 0.01))
def model(X, w):
return T.nnet.softmax(T.dot(X, w))
trX, teX, trY, teY = mnist(onehot=True)
X = T.fmatrix()
Y = T.fmatrix()
