Example #1
0
def run():
    (train_images, train_labels),\
    (tests_images, tests_labels) = load_data_subset(N_train, N_tests)
    parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes, L2_per_dpt)
    N_param = len(parser.vect)

    print "Running experiment..."
    results = defaultdict(list)
    for i in xrange(N_samples):
        x_init_scale = np.full(N_param, init_scale)

        def indexed_loss_fun(w, i_iter):
            rs = RandomState((seed, i, i_iter))
            idxs = rs.randint(N_train, size=batch_size)
            return nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
        gradfun = grad(indexed_loss_fun)

        def callback(x, t, v, entropy):
            results[("entropy", i)].append(entropy / N_train)
            results[("v_norm", i)].append(norm(v) / np.sqrt(N_param))
            results[("minibatch_likelihood", i)].append(-indexed_loss_fun(x, t))
            if t % thin != 0 and t != N_iter and t != 0: return
            results[('iterations', i)].append(t)
            results[("train_likelihood", i)].append(-nllfun(x, train_images, train_labels))
            results[("tests_likelihood", i)].append(-nllfun(x, tests_images, tests_labels))
            results[("tests_error", i)].append(frac_err(x, tests_images, tests_labels))
            print "Iteration {0:5} Train likelihood {1:2.4f}  Test likelihood {2:2.4f}" \
                  "  Test Err {3:2.4f}".format(t, results[("train_likelihood", i)][-1],
                                                  results[("tests_likelihood", i)][-1],
                                                  results[("tests_error",      i)][-1])
        rs = RandomState((seed, i))
        entropic_descent2(gradfun, callback=callback, x_scale=x_init_scale,
                          epsilon=epsilon, gamma=gamma, alpha=alpha,
                          annealing_schedule=annealing_schedule, rs=rs)
    return results
def run():
    all_results = []
    for width in widths:

        init_scale = np.min(0.5 / np.sqrt(width), 0.5 / np.sqrt(784))

        def neg_log_prior(w):
            D = len(w)
            return 0.5 * D * np.log(2 * np.pi) + 0.5 * np.dot(w, w) / init_scale ** 2 + D * np.log(init_scale)

        (train_images, train_labels), (tests_images, tests_labels) = load_data_subset(N_train, N_tests)
        layer_sizes = [784, 100, 10]
        parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
        N_param = len(parser.vect)
        print "Number of parameters in model: ", N_param

        def indexed_loss_fun(w, i_iter):
            rs = RandomState((seed, i_iter))
            idxs = rs.randint(N_train, size=batch_size)
            nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
            # nlp = neg_log_prior(w)
            return nll  # + nlp

        gradfun = grad(indexed_loss_fun)

        def callback(x, t, entropy):
            results["entropy_per_dpt"].append(entropy / N_train)
            results["x_rms"].append(np.sqrt(np.mean(x * x)))
            results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
            results["log_prior_per_dpt"].append(-neg_log_prior(x) / N_train)
            if t % thin != 0 and t != N_iter and t != 0:
                return
            results["iterations"].append(t)
            results["train_likelihood"].append(-nllfun(x, train_images, train_labels))
            results["tests_likelihood"].append(-nllfun(x, tests_images, tests_labels))
            results["tests_error"].append(frac_err(x, tests_images, tests_labels))
            results["marg_likelihood"].append(
                estimate_marginal_likelihood(
                    results["log_prior_per_dpt"][-1], results["train_likelihood"][-1], results["entropy_per_dpt"][-1]
                )
            )

            print "Iteration {0:5} Train lik {1:2.4f}  Test lik {2:2.4f}" "  Marg lik {3:2.4f}  Test err {4:2.4f}".format(
                t,
                results["train_likelihood"][-1],
                results["tests_likelihood"][-1],
                results["marg_likelihood"][-1],
                results["tests_error"][-1],
            )

        results = defaultdict(list)
        rs = RandomState((seed))
        sgd_entropic_damped(gradfun, np.full(N_param, init_scale), N_iter, alpha, rs, callback, width=width)
        all_results.append(results)

    return all_results
def run():
    (train_images, train_labels),\
    (tests_images, tests_labels) = load_data_subset(N_train, N_tests)
    parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
    N_param = len(parser.vect)

    def indexed_loss_fun(w, i_iter):
        rs = RandomState((seed, i, i_iter))
        idxs = rs.randint(N_train, size=batch_size)
        nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
        nlp = neg_log_prior(w)
        return nll + nlp
    gradfun = grad(indexed_loss_fun)

    def callback(x, t, entropy):
        results["entropy_per_dpt"     ].append(entropy / N_train)
        results["x_rms"               ].append(np.sqrt(np.mean(x * x)))
        results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
        results["log_prior_per_dpt"   ].append(-neg_log_prior(x) / N_train)
        if t % thin != 0 and t != N_iter and t != 0: return
        results["iterations"      ].append(t)
        results["train_likelihood"].append(-nllfun(x, train_images, train_labels))
        results["tests_likelihood"].append(-nllfun(x, tests_images, tests_labels))
        results["tests_error"     ].append(frac_err(x, tests_images, tests_labels))
        results["marg_likelihood" ].append(estimate_marginal_likelihood(
            results["train_likelihood"][-1], results["entropy_per_dpt"][-1]))
        preds[i].append(pred_fun(x, tests_images))
                                           
        print "Iteration {0:5} Train lik {1:2.4f}  Test lik {2:2.4f}" \
              "  Marg lik {3:2.4f}  Test err {4:2.4f}".format(
                  t, results["train_likelihood"][-1],
                  results["tests_likelihood"][-1],
                  results["marg_likelihood" ][-1],
                  results["tests_error"     ][-1])

    all_results = []
    preds = defaultdict(list)
    for i in xrange(N_samples):
        results = defaultdict(list)
        rs = RandomState((seed, i))
        sgd_entropic_damped(gradfun, np.full(N_param, init_scale), N_iter, alpha, rs, callback, width=100)
        all_results.append(results)

    # Make ensemble prediction by averaging predicted class-conditional probabilities.
    ensemble_frac_err = []
    ensemble_loglike = []
    for t in xrange(len(all_results[0]["iterations"])):
        cur_probs = [preds[i][t] for i in xrange(N_samples)]
        avg_probs_unn = np.mean(np.exp(cur_probs), axis=0)
        avg_probs = avg_probs_unn / np.sum(avg_probs_unn, axis=1, keepdims=True)
        ensemble_preds = np.argmax(avg_probs, axis=1)
        ensemble_frac_err.append(np.mean(np.argmax(tests_labels, axis=1) != ensemble_preds))
        ensemble_loglike.append(np.sum(np.log(avg_probs) * tests_labels)/tests_images.shape[0])

    return all_results, ensemble_frac_err, ensemble_loglike
Example #4
0
def run():
    (train_images, train_labels),\
    (tests_images, tests_labels) = load_data_subset(N_train, N_tests)
    parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
    N_param = len(parser.vect)
    print "Number of parameters in model: ", N_param

    def indexed_loss_fun(w, i_iter):
        rs = RandomState((seed, i_iter))
        idxs = rs.randint(N_train, size=batch_size)
        nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
        nlp = neg_log_prior(w)
        return nll + nlp

    gradfun = grad(indexed_loss_fun)

    def callback(x, t, entropy):
        results["entropy_per_dpt"].append(entropy / N_train)
        results["x_rms"].append(np.sqrt(np.mean(x * x)))
        results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
        results["log_prior_per_dpt"].append(-neg_log_prior(x) / N_train)
        if t % thin != 0 and t != N_iter and t != 0: return
        results["iterations"].append(t)
        results["train_likelihood"].append(
            -nllfun(x, train_images, train_labels))
        results["tests_likelihood"].append(
            -nllfun(x, tests_images, tests_labels))
        results["tests_error"].append(frac_err(x, tests_images, tests_labels))
        results["marg_likelihood"].append(
            estimate_marginal_likelihood(results["train_likelihood"][-1],
                                         results["entropy_per_dpt"][-1]))

        print "Iteration {0:5} Train lik {1:2.4f}  Test lik {2:2.4f}" \
              "  Marg lik {3:2.4f}  Test err {4:2.4f}".format(
                  t, results["train_likelihood"][-1],
                  results["tests_likelihood"][-1],
                  results["marg_likelihood" ][-1],
                  results["tests_error"     ][-1])

    all_results = []
    preds = defaultdict(list)
    for width in widths:
        results = defaultdict(list)
        rs = RandomState((seed))
        sgd_entropic_damped(gradfun,
                            np.full(N_param, init_scale),
                            N_iter,
                            alpha,
                            rs,
                            callback,
                            width=width)
        all_results.append(results)

    return all_results
Example #5
0
def run():
    (train_images, train_labels),\
    (tests_images, tests_labels) = load_data_subset(N_train, N_tests)
    parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
    N_param = len(parser.vect)

    print "Running experiment..."
    results = defaultdict(list)
    for i in xrange(N_samples):

        def indexed_loss_fun(w, i_iter):
            rs = RandomState((seed, i, i_iter))
            idxs = rs.randint(N_train, size=batch_size)
            nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
            nlp = neg_log_prior(w)
            return nll + nlp

        gradfun = grad(indexed_loss_fun)

        def callback(x, t, v, entropy):
            results[("entropy", i)].append(entropy / N_train)
            results[("v_norm", i)].append(norm(v) / np.sqrt(N_param))
            results[("minibatch_likelihood",
                     i)].append(-indexed_loss_fun(x, t))
            results[("log_prior_per_dpt",
                     i)].append(-neg_log_prior(x) / N_train)
            if t % thin != 0 and t != N_iter and t != 0: return
            results[('iterations', i)].append(t)
            results[("train_likelihood",
                     i)].append(-nllfun(x, train_images, train_labels))
            results[("tests_likelihood",
                     i)].append(-nllfun(x, tests_images, tests_labels))
            results[("tests_error",
                     i)].append(frac_err(x, tests_images, tests_labels))
            print "Iteration {0:5} Train likelihood {1:2.4f}  Test likelihood {2:2.4f}" \
                  "  Test Err {3:2.4f}".format(t, results[("train_likelihood", i)][-1],
                                                  results[("tests_likelihood", i)][-1],
                                                  results[("tests_error",      i)][-1])

        rs = RandomState((seed, i))
        x0 = rs.randn(N_param) * init_scale
        v0 = rs.randn(N_param)  # TODO: account for entropy of init.
        #entropy = 0.5 * D * (1 + np.log(2*np.pi)) + np.sum(np.log(x_scale)) + 0.5 * (D - norm(v) **2)
        aed3(gradfun,
             callback=callback,
             x=x0,
             v=v0,
             learn_rate=epsilon,
             iters=N_iter,
             init_log_decay=init_log_decay,
             decay_learn_rate=decay_learn_rate)
    return results
def run():
    (train_images, train_labels), (tests_images, tests_labels) = load_data_subset(N_train, N_tests)
    parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
    N_param = len(parser.vect)

    print "Running experiment..."
    results = defaultdict(list)
    for i in xrange(N_samples):

        def indexed_loss_fun(w, i_iter):
            rs = RandomState((seed, i, i_iter))
            idxs = rs.randint(N_train, size=batch_size)
            nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
            nlp = neg_log_prior(w)
            return nll + nlp

        gradfun = grad(indexed_loss_fun)

        def callback(x, t, v, entropy):
            results[("entropy", i)].append(entropy / N_train)
            results[("v_norm", i)].append(norm(v) / np.sqrt(N_param))
            results[("minibatch_likelihood", i)].append(-indexed_loss_fun(x, t))
            results[("log_prior_per_dpt", i)].append(-neg_log_prior(x) / N_train)
            if t % thin != 0 and t != N_iter and t != 0:
                return
            results[("iterations", i)].append(t)
            results[("train_likelihood", i)].append(-nllfun(x, train_images, train_labels))
            results[("tests_likelihood", i)].append(-nllfun(x, tests_images, tests_labels))
            results[("tests_error", i)].append(frac_err(x, tests_images, tests_labels))
            print "Iteration {0:5} Train likelihood {1:2.4f}  Test likelihood {2:2.4f}" "  Test Err {3:2.4f}".format(
                t,
                results[("train_likelihood", i)][-1],
                results[("tests_likelihood", i)][-1],
                results[("tests_error", i)][-1],
            )

        rs = RandomState((seed, i))
        x0 = rs.randn(N_param) * init_scale
        v0 = rs.randn(N_param)  # TODO: account for entropy of init.
        # entropy = 0.5 * D * (1 + np.log(2*np.pi)) + np.sum(np.log(x_scale)) + 0.5 * (D - norm(v) **2)
        aed3(
            gradfun,
            callback=callback,
            x=x0,
            v=v0,
            learn_rate=epsilon,
            iters=N_iter,
            init_log_decay=init_log_decay,
            decay_learn_rate=decay_learn_rate,
        )
    return results
def run():
    (train_images, train_labels),\
    (tests_images, tests_labels) = load_data_subset(N_train, N_tests)
    parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
    N_param = len(parser.vect)

    def indexed_loss_fun(w, i_iter):
        rs = RandomState((seed, i, i_iter))
        idxs = rs.randint(N_train, size=batch_size)
        nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
        nlp = neg_log_prior(w)
        return nll + nlp
    gradfun = grad(indexed_loss_fun)

    def callback(x, t, entropy):
        results["entropy_per_dpt"     ].append(entropy / N_train)
        results["x_rms"               ].append(np.sqrt(np.mean(x * x)))
        results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
        results["log_prior_per_dpt"   ].append(-neg_log_prior(x) / N_train)
        if t % thin != 0 and t != N_iter and t != 0: return
        results["iterations"      ].append(t)
        results["train_likelihood"].append(-nllfun(x, train_images, train_labels))
        results["tests_likelihood"].append(-nllfun(x, tests_images, tests_labels))
        results["tests_error"     ].append(frac_err(x, tests_images, tests_labels))
        results["marg_likelihood" ].append(estimate_marginal_likelihood(
            results["train_likelihood"][-1], results["entropy_per_dpt"][-1]))
                                           
        print "Iteration {0:5} Train lik {1:2.4f}  Test lik {2:2.4f}" \
              "  Marg lik {3:2.4f}  Test err {4:2.4f}".format(
                  t, results["train_likelihood"][-1],
                  results["tests_likelihood"][-1],
                  results["marg_likelihood" ][-1],
                  results["tests_error"     ][-1])

    all_results = []
    for i in xrange(N_samples):
        results = defaultdict(list)
        rs = RandomState((seed, i))
        sgd_entropic(gradfun, np.full(N_param, init_scale), N_iter, alpha, rs, callback)
        all_results.append(results)

    return all_results
Example #8
0
def run():
    (train_images, train_labels),\
    (tests_images, tests_labels) = load_data_subset(N_train, N_tests)
    parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
    N_param = len(parser.vect)

    def indexed_loss_fun(w, i_iter):
        rs = RandomState((seed, i, i_iter))
        idxs = rs.randint(N_train, size=batch_size)
        nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
        nlp = neg_log_prior(w)
        return nll + nlp

    gradfun = grad(indexed_loss_fun)

    def callback(x, t, entropy):
        results["entropy_per_dpt"].append(entropy / N_train)
        results["x_rms"].append(np.sqrt(np.mean(x * x)))
        results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
        results["log_prior_per_dpt"].append(-neg_log_prior(x) / N_train)
        if t % thin != 0 and t != N_iter and t != 0: return
        results["iterations"].append(t)
        results["train_likelihood"].append(
            -nllfun(x, train_images, train_labels))
        results["tests_likelihood"].append(
            -nllfun(x, tests_images, tests_labels))
        results["tests_error"].append(frac_err(x, tests_images, tests_labels))
        results["marg_likelihood"].append(
            estimate_marginal_likelihood(results["train_likelihood"][-1],
                                         results["entropy_per_dpt"][-1]))
        preds[i].append(pred_fun(x, tests_images))

        print "Iteration {0:5} Train lik {1:2.4f}  Test lik {2:2.4f}" \
              "  Marg lik {3:2.4f}  Test err {4:2.4f}".format(
                  t, results["train_likelihood"][-1],
                  results["tests_likelihood"][-1],
                  results["marg_likelihood" ][-1],
                  results["tests_error"     ][-1])

    all_results = []
    preds = defaultdict(list)
    for i in xrange(N_samples):
        results = defaultdict(list)
        rs = RandomState((seed, i))
        sgd_entropic_damped(gradfun,
                            np.full(N_param, init_scale),
                            N_iter,
                            alpha,
                            rs,
                            callback,
                            width=100)
        all_results.append(results)

    # Make ensemble prediction by averaging predicted class-conditional probabilities.
    ensemble_frac_err = []
    ensemble_loglike = []
    for t in xrange(len(all_results[0]["iterations"])):
        cur_probs = [preds[i][t] for i in xrange(N_samples)]
        avg_probs_unn = np.mean(np.exp(cur_probs), axis=0)
        avg_probs = avg_probs_unn / np.sum(
            avg_probs_unn, axis=1, keepdims=True)
        ensemble_preds = np.argmax(avg_probs, axis=1)
        ensemble_frac_err.append(
            np.mean(np.argmax(tests_labels, axis=1) != ensemble_preds))
        ensemble_loglike.append(
            np.sum(np.log(avg_probs) * tests_labels) / tests_images.shape[0])

    return all_results, ensemble_frac_err, ensemble_loglike