# construct theano graph
h = x
for i in xrange(len(dims) - 1):
layer_parameters = []
if hyperparameters["share_parameters"]:
# for the compatible layers
if Ws[i].shape == Ws[1].shape:
i = 1
c, U, W, gamma, b, f = cs[i], Us[i], Ws[i], gammas[i], bs[i], fs[i]
if hyperparameters["whiten_inputs"]:
wupdates, wchecks = whitening.get_updates(
h, c, U, V=W, d=b,
decomposition=hyperparameters["decomposition"],
zca=hyperparameters["zca"],
bias=hyperparameters["eigenvalue_bias"])
updates.extend(wupdates)
checks.extend(wchecks)
h = T.dot(h - c, U)
layer_parameters.extend([c, U])
h = T.dot(h, W)
layer_parameters.append(W)
if hyperparameters["batch_normalize"]:
mean = h.mean(axis=0, keepdims=True)
var = h.var (axis=0, keepdims=True)
h = (h - mean) / T.sqrt(var + hyperparameters["variance_bias"])
h *= gammas[i]
#targets.tag.test_value = data["valid"]["targets"][:11]
# reparametrization updates
reparameterization_updates = []
# theano graphs with assertions & breakpoints, to be evaluated after
# performing the updates
reparameterization_checks = []
# construct theano graph
h = features
for i, layer in enumerate(layers):
f, c, U, W, g, b = [layer[k] for k in "fcUWgb"]
# construct reparameterization graph
updates, checks = whitening.get_updates(
h, c, U, V=W, d=b,
decomposition="svd", zca=True, bias=1e-3)
reparameterization_updates.extend(updates)
reparameterization_checks.extend(checks)
# whiten input
h = T.dot(h - c, U)
# compute layer as usual
h = T.dot(h, W)
if batch_normalize:
h -= h.mean(axis=0, keepdims=True)
h /= T.sqrt(h.var(axis=0, keepdims=True) + 1e-8)
h *= g
h += b
h = f(h)