def test_in_memory(): skip_if_not_available(datasets=['mnist.hdf5']) # Load MNIST and get two batches mnist = MNIST('train', load_in_memory=True) data_stream = DataStream(mnist, iteration_scheme=SequentialScheme( examples=mnist.num_examples, batch_size=256)) epoch = data_stream.get_epoch_iterator() for i, (features, targets) in enumerate(epoch): if i == 1: break handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(256, 512)) mnist.close(handle) assert numpy.all(features == known_features) # Pickle the epoch and make sure that the data wasn't dumped with tempfile.NamedTemporaryFile(delete=False) as f: filename = f.name cPickle.dump(epoch, f) assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB # Reload the epoch and make sure that the state was maintained del epoch with open(filename, 'rb') as f: epoch = cPickle.load(f) features, targets = next(epoch) handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(512, 768)) mnist.close(handle) assert numpy.all(features == known_features)
def test_in_memory(): skip_if_not_available(datasets=['mnist.hdf5']) # Load MNIST and get two batches mnist = MNIST(('train',), load_in_memory=True) data_stream = DataStream(mnist, iteration_scheme=SequentialScheme( examples=mnist.num_examples, batch_size=256)) epoch = data_stream.get_epoch_iterator() for i, (features, targets) in enumerate(epoch): if i == 1: break handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(256, 512)) mnist.close(handle) assert numpy.all(features == known_features) # Pickle the epoch and make sure that the data wasn't dumped with tempfile.NamedTemporaryFile(delete=False) as f: filename = f.name cPickle.dump(epoch, f) assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB # Reload the epoch and make sure that the state was maintained del epoch with open(filename, 'rb') as f: epoch = cPickle.load(f) features, targets = next(epoch) handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(512, 768)) mnist.close(handle) assert numpy.all(features == known_features)
def test_mnist_test(): skip_if_not_available(datasets=["mnist.hdf5"]) dataset = MNIST(("test",), load_in_memory=False) handle = dataset.open() data, labels = dataset.get_data(handle, slice(0, 10)) assert data.dtype == "uint8" assert data.shape == (10, 1, 28, 28) assert labels.shape == (10, 1) known = numpy.array([0, 0, 0, 0, 0, 0, 84, 185, 159, 151, 60, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) assert_allclose(data[0][0][7], known) assert labels[0][0] == 7 assert dataset.num_examples == 10000 dataset.close(handle) stream = DataStream.default_stream(dataset, iteration_scheme=SequentialScheme(10, 10)) data = next(stream.get_epoch_iterator())[0] assert data.min() >= 0.0 and data.max() <= 1.0 assert data.dtype == config.floatX
def test_mnist_train(): skip_if_not_available(datasets=['mnist.hdf5']) dataset = MNIST('train', load_in_memory=False) handle = dataset.open() data, labels = dataset.get_data(handle, slice(0, 10)) assert data.dtype == 'uint8' assert data.shape == (10, 1, 28, 28) assert labels.shape == (10, 1) known = numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 30, 36, 94, 154, 170, 253, 253, 253, 253, 253, 225, 172, 253, 242, 195, 64, 0, 0, 0, 0]) assert_allclose(data[0][0][6], known) assert labels[0][0] == 5 assert dataset.num_examples == 60000 dataset.close(handle) stream = DataStream.default_stream( dataset, iteration_scheme=SequentialScheme(10, 10)) data = next(stream.get_epoch_iterator())[0] assert data.min() >= 0.0 and data.max() <= 1.0 assert data.dtype == config.floatX
def test_mnist_test(): skip_if_not_available(datasets=['mnist.hdf5']) dataset = MNIST('test', load_in_memory=False) handle = dataset.open() data, labels = dataset.get_data(handle, slice(0, 10)) assert data.dtype == 'uint8' assert data.shape == (10, 1, 28, 28) assert labels.shape == (10, 1) known = numpy.array([ 0, 0, 0, 0, 0, 0, 84, 185, 159, 151, 60, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) assert_allclose(data[0][0][7], known) assert labels[0][0] == 7 assert dataset.num_examples == 10000 dataset.close(handle) stream = DataStream.default_stream(dataset, iteration_scheme=SequentialScheme( 10, 10)) data = next(stream.get_epoch_iterator())[0] assert data.min() >= 0.0 and data.max() <= 1.0 assert data.dtype == config.floatX
from fuel.datasets import MNIST from fuel.schemes import SequentialScheme from fuel.transformers import Mapping, Flatten from blocks.graph import ComputationGraph from blocks.monitoring import aggregation from blocks.extensions import FinishAfter, Timing, Printing from blocks.extensions.monitoring import (DataStreamMonitoring, TrainingDataMonitoring) from blocks.main_loop import MainLoop from blocks_contrib.extensions import DataStreamMonitoringAndSaving floatX = theano.config.floatX mnist = MNIST('train', sources=['features']) handle = mnist.open() data = mnist.get_data(handle, slice(0, 50000))[0] means = data.reshape((50000, 784)).mean(axis=0) def autocorrentropy2(X, ksize=np.inf): b, t, d = X.shape V = np.zeros((b, t, d)) for i in range(b): for j in range(t): if ksize in (np.inf, 0., np.nan): V[i, j, :] = (X[i, :(t-j), :] * X[i, j:, :]).sum(axis=0) / (t-j) else: V[i, j, :] = np.exp((-ksize * (X[i, :(t-j), :]-X[i, j:, :])**2)).sum(axis=0) / (t-j) return V
for x, y in zip(trX, trY): train(x, y) print(w.get_value()) #something around 2 #https://raw.githubusercontent.com/Newmu/Theano-Tutorials/master/2_logistic_regression.py import theano from theano import tensor as T import numpy as np from fuel.datasets import MNIST from matplotlib import pyplot, cm dataset = MNIST(('train',), sources=('features',)) state = dataset.open() image, = dataset.get_data(state=state, request=[1234]) pyplot.imshow(image.reshape((28, 28)), cmap=cm.Greys_r, interpolation='nearest') pyplot.show() dataset.close(state) def floatX(X): return np.asarray(X, dtype=theano.config.floatX) def init_weights(shape): return theano.shared(floatX(np.random.randn(*shape) * 0.01)) def model(X, w): return T.nnet.softmax(T.dot(X, w)) trX, teX, trY, teY = mnist(onehot=True)