# construct theano graph h = x for i in xrange(len(dims) - 1): layer_parameters = [] if hyperparameters["share_parameters"]: # for the compatible layers if Ws[i].shape == Ws[1].shape: i = 1 c, U, W, gamma, b, f = cs[i], Us[i], Ws[i], gammas[i], bs[i], fs[i] if hyperparameters["whiten_inputs"]: wupdates, wchecks = whitening.get_updates( h, c, U, V=W, d=b, decomposition=hyperparameters["decomposition"], zca=hyperparameters["zca"], bias=hyperparameters["eigenvalue_bias"]) updates.extend(wupdates) checks.extend(wchecks) h = T.dot(h - c, U) layer_parameters.extend([c, U]) h = T.dot(h, W) layer_parameters.append(W) if hyperparameters["batch_normalize"]: mean = h.mean(axis=0, keepdims=True) var = h.var (axis=0, keepdims=True) h = (h - mean) / T.sqrt(var + hyperparameters["variance_bias"]) h *= gammas[i]
#targets.tag.test_value = data["valid"]["targets"][:11] # reparametrization updates reparameterization_updates = [] # theano graphs with assertions & breakpoints, to be evaluated after # performing the updates reparameterization_checks = [] # construct theano graph h = features for i, layer in enumerate(layers): f, c, U, W, g, b = [layer[k] for k in "fcUWgb"] # construct reparameterization graph updates, checks = whitening.get_updates( h, c, U, V=W, d=b, decomposition="svd", zca=True, bias=1e-3) reparameterization_updates.extend(updates) reparameterization_checks.extend(checks) # whiten input h = T.dot(h - c, U) # compute layer as usual h = T.dot(h, W) if batch_normalize: h -= h.mean(axis=0, keepdims=True) h /= T.sqrt(h.var(axis=0, keepdims=True) + 1e-8) h *= g h += b h = f(h)