Ejemplo n.º 1
0
hyperparameters = dict(
    # "eigh" on covariance matrix or "svd" on data matrix
    decomposition="svd",
    # whether to remix after whitening
    zca=True,
    # compute fisher based on supervised "loss" or model "output"
    objective="output",
    # eigenvalue bias
    eigenvalue_bias=1e-3,
    variance_bias=1e-8,
    batch_normalize=False,
    whiten_inputs=False,
    share_parameters=True)

datasets = mnist.get_data()

features = T.matrix("features")
targets = T.ivector("targets")

#theano.config.compute_test_value = "warn"
#features.tag.test_value = datasets["valid"]["features"]
#targets.tag.test_value = datasets["valid"]["targets"]


# compilation helpers
compile_memo = dict()
def compile(variables=(), updates=()):
    key = (util.tupelo(variables),
           tuple(OrderedDict(updates).items()))
    try:
Ejemplo n.º 2
0
import sys
from collections import OrderedDict
import numpy as np
import theano
import theano.tensor as T
import util, activation, initialization, steprules, whitening, mnist

learning_rate = 1e-3
# use batch normalization in addition to PRONG (i.e. PRONG+)
batch_normalize = False

data = mnist.get_data()
n_outputs = 10

dims = [784, 500, 300, 100, n_outputs]
layers = [
    dict(f=activation.tanh,
         c=util.shared_floatx((m,),   initialization.constant(0)),   # input mean
         U=util.shared_floatx((m, m), initialization.identity()),    # input whitening matrix
         W=util.shared_floatx((m, n), initialization.orthogonal()),  # weight matrix
         g=util.shared_floatx((n,),   initialization.constant(1)),   # gammas (for batch normalization)
         b=util.shared_floatx((n,),   initialization.constant(0)))   # bias
    for m, n in util.safezip(dims[:-1], dims[1:])]
layers[-1]["f"] = activation.logsoftmax

features, targets = T.matrix("features"), T.ivector("targets")

#theano.config.compute_test_value = "warn"
#features.tag.test_value = data["valid"]["features"][:11]
#targets.tag.test_value = data["valid"]["targets"][:11]