Exemplo n.º 1
0
# construct theano graph
h = x
for i in xrange(len(dims) - 1):
    layer_parameters = []

    if hyperparameters["share_parameters"]:
        # for the compatible layers
        if Ws[i].shape == Ws[1].shape:
            i = 1

    c, U, W, gamma, b, f = cs[i], Us[i], Ws[i], gammas[i], bs[i], fs[i]

    if hyperparameters["whiten_inputs"]:
        wupdates, wchecks = whitening.get_updates(
            h, c, U, V=W, d=b,
            decomposition=hyperparameters["decomposition"],
            zca=hyperparameters["zca"],
            bias=hyperparameters["eigenvalue_bias"])
        updates.extend(wupdates)
        checks.extend(wchecks)
        h = T.dot(h - c, U)
        layer_parameters.extend([c, U])

    h = T.dot(h, W)
    layer_parameters.append(W)

    if hyperparameters["batch_normalize"]:
        mean = h.mean(axis=0, keepdims=True)
        var  = h.var (axis=0, keepdims=True)
        h = (h - mean) / T.sqrt(var + hyperparameters["variance_bias"])
        h *= gammas[i]
Exemplo n.º 2
0
#targets.tag.test_value = data["valid"]["targets"][:11]

# reparametrization updates
reparameterization_updates = []
# theano graphs with assertions & breakpoints, to be evaluated after
# performing the updates
reparameterization_checks = []

# construct theano graph
h = features
for i, layer in enumerate(layers):
    f, c, U, W, g, b = [layer[k] for k in "fcUWgb"]

    # construct reparameterization graph
    updates, checks = whitening.get_updates(
        h, c, U, V=W, d=b,
        decomposition="svd", zca=True, bias=1e-3)
    reparameterization_updates.extend(updates)
    reparameterization_checks.extend(checks)

    # whiten input
    h = T.dot(h - c, U)
    # compute layer as usual
    h = T.dot(h, W)
    if batch_normalize:
        h -= h.mean(axis=0, keepdims=True)
        h /= T.sqrt(h.var(axis=0, keepdims=True) + 1e-8)
        h *= g
    h += b
    h = f(h)