def test_in_memory(): skip_if_not_available(datasets=['mnist.hdf5']) # Load MNIST and get two batches mnist = MNIST('train', load_in_memory=True) data_stream = DataStream(mnist, iteration_scheme=SequentialScheme( examples=mnist.num_examples, batch_size=256)) epoch = data_stream.get_epoch_iterator() for i, (features, targets) in enumerate(epoch): if i == 1: break handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(256, 512)) mnist.close(handle) assert numpy.all(features == known_features) # Pickle the epoch and make sure that the data wasn't dumped with tempfile.NamedTemporaryFile(delete=False) as f: filename = f.name cPickle.dump(epoch, f) assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB # Reload the epoch and make sure that the state was maintained del epoch with open(filename, 'rb') as f: epoch = cPickle.load(f) features, targets = next(epoch) handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(512, 768)) mnist.close(handle) assert numpy.all(features == known_features)
def test_in_memory(): skip_if_not_available(datasets=['mnist.hdf5']) # Load MNIST and get two batches mnist = MNIST(('train',), load_in_memory=True) data_stream = DataStream(mnist, iteration_scheme=SequentialScheme( examples=mnist.num_examples, batch_size=256)) epoch = data_stream.get_epoch_iterator() for i, (features, targets) in enumerate(epoch): if i == 1: break handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(256, 512)) mnist.close(handle) assert numpy.all(features == known_features) # Pickle the epoch and make sure that the data wasn't dumped with tempfile.NamedTemporaryFile(delete=False) as f: filename = f.name cPickle.dump(epoch, f) assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB # Reload the epoch and make sure that the state was maintained del epoch with open(filename, 'rb') as f: epoch = cPickle.load(f) features, targets = next(epoch) handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(512, 768)) mnist.close(handle) assert numpy.all(features == known_features)
def test_mnist_test(): skip_if_not_available(datasets=["mnist.hdf5"]) dataset = MNIST(("test",), load_in_memory=False) handle = dataset.open() data, labels = dataset.get_data(handle, slice(0, 10)) assert data.dtype == "uint8" assert data.shape == (10, 1, 28, 28) assert labels.shape == (10, 1) known = numpy.array([0, 0, 0, 0, 0, 0, 84, 185, 159, 151, 60, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) assert_allclose(data[0][0][7], known) assert labels[0][0] == 7 assert dataset.num_examples == 10000 dataset.close(handle) stream = DataStream.default_stream(dataset, iteration_scheme=SequentialScheme(10, 10)) data = next(stream.get_epoch_iterator())[0] assert data.min() >= 0.0 and data.max() <= 1.0 assert data.dtype == config.floatX
def test_mnist_train(): skip_if_not_available(datasets=['mnist.hdf5']) dataset = MNIST('train', load_in_memory=False) handle = dataset.open() data, labels = dataset.get_data(handle, slice(0, 10)) assert data.dtype == 'uint8' assert data.shape == (10, 1, 28, 28) assert labels.shape == (10, 1) known = numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 30, 36, 94, 154, 170, 253, 253, 253, 253, 253, 225, 172, 253, 242, 195, 64, 0, 0, 0, 0]) assert_allclose(data[0][0][6], known) assert labels[0][0] == 5 assert dataset.num_examples == 60000 dataset.close(handle) stream = DataStream.default_stream( dataset, iteration_scheme=SequentialScheme(10, 10)) data = next(stream.get_epoch_iterator())[0] assert data.min() >= 0.0 and data.max() <= 1.0 assert data.dtype == config.floatX
def test_mnist_test(): skip_if_not_available(datasets=['mnist.hdf5']) dataset = MNIST('test', load_in_memory=False) handle = dataset.open() data, labels = dataset.get_data(handle, slice(0, 10)) assert data.dtype == 'uint8' assert data.shape == (10, 1, 28, 28) assert labels.shape == (10, 1) known = numpy.array([ 0, 0, 0, 0, 0, 0, 84, 185, 159, 151, 60, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) assert_allclose(data[0][0][7], known) assert labels[0][0] == 7 assert dataset.num_examples == 10000 dataset.close(handle) stream = DataStream.default_stream(dataset, iteration_scheme=SequentialScheme( 10, 10)) data = next(stream.get_epoch_iterator())[0] assert data.min() >= 0.0 and data.max() <= 1.0 assert data.dtype == config.floatX
#https://raw.githubusercontent.com/Newmu/Theano-Tutorials/master/2_logistic_regression.py import theano from theano import tensor as T import numpy as np from fuel.datasets import MNIST from matplotlib import pyplot, cm dataset = MNIST(('train',), sources=('features',)) state = dataset.open() image, = dataset.get_data(state=state, request=[1234]) pyplot.imshow(image.reshape((28, 28)), cmap=cm.Greys_r, interpolation='nearest') pyplot.show() dataset.close(state) def floatX(X): return np.asarray(X, dtype=theano.config.floatX) def init_weights(shape): return theano.shared(floatX(np.random.randn(*shape) * 0.01)) def model(X, w): return T.nnet.softmax(T.dot(X, w)) trX, teX, trY, teY = mnist(onehot=True) X = T.fmatrix() Y = T.fmatrix()