Ejemplo n.º 1
0
def profile_tree(dataset_name, model_name, tag=None, sgp=None, n=None, test_n=None, burnin=10, cutoff_rule=2):

    X_test, y_test = test_data(dataset_name)
    gp = trained_gp(dataset_name, model_name, n=n, tag=tag, build_tree=True, leaf_bin_width=0.5, build_dense_Kinv_hack=True)

    if test_n is None:
        test_n = min(len(X_test), 5000)
    print "loaded GP, evaluating timings on %d test points..." % test_n

    eps_abs = 0.001 * gp.noise_var

    tree_covar = np.zeros((test_n,))
    tree_covar_terms = np.zeros((test_n,))
    tree_covar_distevals = np.zeros((test_n,))
    tree_covar_times = np.zeros((test_n,))
    tree_covar_qftimes = np.zeros((test_n,))
    tree_covar_nonqftimes = np.zeros((test_n,))



    gp.nonqf_time = 0
    for i in range(test_n):
        t6 = time.time()
        tree_covar[i] = gp.covariance_double_tree(X_test[i:i+1,:], eps_abs=eps_abs, cutoff_rule=cutoff_rule)
        t7 = time.time()

        tree_covar_terms[i] = gp.double_tree.fcalls
        tree_covar_distevals[i] = gp.double_tree.dfn_evals
        tree_covar_times[i] = t7-t6
        tree_covar_qftimes[i] = gp.qf_time
        tree_covar_nonqftimes[i] = gp.nonqf_time

    print "tree: eps_abs %f times: %s" % (eps_abs, strstats(tree_covar_times[burnin:]))
    print "tree: eps_abs %f qftimes: %s" % (eps_abs, strstats(tree_covar_qftimes[burnin:]))
    print "tree: eps_abs %f nqftimes: %s" % (eps_abs, strstats(tree_covar_nonqftimes[burnin:]))
    print "tree: eps_abs %f terms: %s" % (eps_abs, strstats(tree_covar_terms[burnin:]))
    print "tree: eps_abs %f dfnevals: %s" % (eps_abs, strstats(tree_covar_distevals[burnin:]))
Ejemplo n.º 2
0
def eval_gp(dataset_name, model_name, tag=None, sgp=None, n=None, test_n=None, burnin=10, cutoff_rule=2):
    X_test, y_test = test_data(dataset_name)
    gp = trained_gp(dataset_name, model_name, n=n, tag=tag, build_tree=True, leaf_bin_width=None, build_dense_Kinv_hack=True, compile_tree=compiled_tree_fname(dataset_name, model_name, tag))
    if test_n is None:
        test_n = min(len(X_test), 5000)


    print "loaded GP, evaluating timings on %d test points..." % test_n

    resultfile = timing_results_fname(dataset_name, model_name, tag)
    errorfile = timing_errors_fname(dataset_name, model_name, tag)




    naive_predict = np.zeros(test_n)
    naive_predict_times = np.zeros(test_n)

    tree_predict = np.zeros(test_n)
    tree_predict_terms = np.zeros(test_n, dtype=int)
    tree_predict_times = np.zeros(test_n)

    dense_covar = np.zeros(test_n)
    dense_covar_times = np.zeros(test_n)

    sparse_covar = np.zeros(test_n)
    sparse_covar_times = np.zeros(test_n)
    sparse_covar_qftimes = np.zeros(test_n)
    sparse_covar_nonqftimes = np.zeros(test_n)

    sparse_covar_spkernel = np.zeros(test_n)
    sparse_covar_spkernel_times = np.zeros(test_n)
    sparse_covar_spkernel_qftimes = np.zeros(test_n)
    sparse_covar_spkernel_nonqftimes = np.zeros(test_n)

    sparse_covar_local = np.zeros(test_n)
    sparse_covar_local_times = np.zeros(test_n)
    sparse_covar_local_tree_times = np.zeros(test_n)
    sparse_covar_local_math_times = np.zeros(test_n)
    sparse_covar_local_terms = np.zeros(test_n)
    sparse_covar_local_dfn_evals = np.zeros(test_n)
    sparse_covar_local_wfn_evals = np.zeros(test_n)
    sparse_covar_local_qftimes = np.zeros(test_n)
    sparse_covar_local_nonqftimes = np.zeros(test_n)


    sparse_covar_spkernel_solve = np.zeros(test_n)
    sparse_covar_spkernel_solve_times = np.zeros(test_n)

    print "naive predict"
    for i in range(test_n):
        t0 = time.time()
        naive_predict[i] = gp.predict_naive(X_test[i:i+1,:])
        t1 = time.time()
        naive_predict_times[i] = t1-t0

    print "tree predict"
    for i in range(test_n):
        t2 = time.time()
        tree_predict[i] = gp.predict(X_test[i:i+1,:], eps=1e-4)
        t3 = time.time()
        tree_predict_times[i] = t3-t2
        tree_predict_terms[i] = gp.predict_tree.terms

    print "treedense/local covar"
    gp.nonqf_time = 0
    for i in range(test_n):
        t4 = time.time()
        sparse_covar_local[i] = gp.covariance_treedense(X_test[i:i+1,:])
        t5 = time.time()
        sparse_covar_local_times[i] = t5-t4
        sparse_covar_local_tree_times[i] = gp.predict_tree.dense_hack_tree_s
        sparse_covar_local_math_times[i] = gp.predict_tree.dense_hack_math_s
        sparse_covar_local_terms[i] = gp.qf_terms
        sparse_covar_local_dfn_evals[i] = gp.qf_dfn_evals
        sparse_covar_local_wfn_evals[i] = gp.qf_wfn_evals
        sparse_covar_local_qftimes[i] = gp.qf_time
        sparse_covar_local_nonqftimes[i] = gp.nonqf_time

    print "sparse covar"
    gp.nonqf_time = 0
    for i in range(test_n):
        t4 = time.time()
        sparse_covar[i] = gp.covariance(X_test[i:i+1,:])
        t5 = time.time()
        sparse_covar_times[i] = t5-t4
        sparse_covar_qftimes[i] = gp.qf_time
        sparse_covar_nonqftimes[i] = gp.nonqf_time



    print "spkernel"
    for i in range(test_n):
        t41 = time.time()
        sparse_covar_spkernel[i] = gp.covariance_spkernel(X_test[i:i+1,:])
        t51 = time.time()
        sparse_covar_spkernel_times[i] = t51-t41
        sparse_covar_spkernel_qftimes[i] = gp.qf_time
        sparse_covar_spkernel_nonqftimes[i] = gp.nonqf_time

    """
    for i in range(test_n):
        t41 = time.time()
        sparse_covar_spkernel_solve[i] = gp.covariance_spkernel_solve(X_test[i:i+1,:])
        t51 = time.time()
        sparse_covar_spkernel_solve_times[i] = t51-t41
    """

    has_dense = True
    try:
        for i in range(100):
            t42 = time.time()
            dense_covar[i] = gp.covariance_dense(X_test[i:i+1,:])
            t52 = time.time()
            dense_covar_times[i] = t52-t42
    except:
        has_dense=False

    f = open(resultfile, 'w')

    eps_abs = 0.001 * gp.noise_var

    tree_covar = np.zeros((test_n,))
    tree_covar_terms = np.zeros((test_n,))
    tree_covar_zeroterms = np.zeros((test_n,))
    tree_covar_nodes_touched = np.zeros((test_n,))
    tree_covar_dfn_evals = np.zeros((test_n,))
    tree_covar_wfn_evals = np.zeros((test_n,))
    tree_covar_dfn_misses = np.zeros((test_n,))
    tree_covar_wfn_misses = np.zeros((test_n,))
    tree_covar_times = np.zeros((test_n,))
    tree_covar_qftimes = np.zeros((test_n,))
    tree_covar_nonqftimes = np.zeros((test_n,))
    for i in range(test_n):
        t6 = time.time()
        tree_covar[i] = gp.covariance_double_tree(X_test[i:i+1,:], eps_abs=eps_abs, cutoff_rule=cutoff_rule)
        t7 = time.time()

        tree_covar_terms[i] = gp.qf_terms
        tree_covar_zeroterms[i] = gp.qf_zeroterms
        tree_covar_nodes_touched[i] = gp.qf_nodes_touched
        tree_covar_dfn_evals[i] = gp.qf_dfn_evals
        tree_covar_wfn_evals[i] = gp.qf_wfn_evals
        tree_covar_dfn_misses[i] = gp.qf_dfn_misses
        tree_covar_wfn_misses[i] = gp.qf_wfn_misses


        tree_covar_times[i] = t7-t6
        tree_covar_qftimes[i] = gp.qf_time
        tree_covar_nonqftimes[i] = gp.nonqf_time

    tree_compiled = np.zeros((test_n,))
    tree_compiled_terms = np.zeros((test_n,))
    tree_compiled_zeroterms = np.zeros((test_n,))
    tree_compiled_nodes_touched = np.zeros((test_n,))
    tree_compiled_dfn_evals = np.zeros((test_n,))
    tree_compiled_times = np.zeros((test_n,))
    tree_compiled_qftimes = np.zeros((test_n,))
    tree_compiled_nonqftimes = np.zeros((test_n,))
    for i in range(test_n):
        t6 = time.time()
        tree_compiled[i] = gp.covariance_compiled(X_test[i:i+1,:], eps_abs=eps_abs)
        t7 = time.time()

        tree_compiled_terms[i] = gp.qf_terms
        tree_compiled_zeroterms[i] = gp.qf_zeroterms
        tree_compiled_nodes_touched[i] = gp.qf_nodes_touched
        tree_compiled_dfn_evals[i] = gp.qf_dfn_evals

        tree_compiled_times[i] = t7-t6
        tree_compiled_qftimes[i] = gp.qf_time
        tree_compiled_nonqftimes[i] = gp.nonqf_time


    f.write("naive predict times: %s\n" % strstats(naive_predict_times[burnin:]))
    f.write("\n")
    f.write("tree predict times: %s\n" % strstats(tree_predict_times[burnin:]))
    f.write("tree predict terms:  %s\n" % strstats(tree_predict_terms[burnin:]))
    f.write("tree predict errors: %s\n" % strstats(np.abs(tree_predict[burnin:] - naive_predict[burnin:])))
    f.write("\n")

    if has_dense:
        f.write("dense covar times: %s\n" % strstats(dense_covar_times[burnin:]))
    f.write("sparse covar times: %s\n" % strstats(sparse_covar_times[burnin:]))
    f.write("sparse covar qftimes: %s\n" % strstats(sparse_covar_qftimes[burnin:]))
    f.write("sparse covar nqftimes: %s\n" % strstats(sparse_covar_nonqftimes[burnin:]))
    f.write("\n")
    f.write("sparse covar spkernel times: %s\n" % strstats(sparse_covar_spkernel_times[burnin:]))
    f.write("sparse covar spkernel qftimes: %s\n" % strstats(sparse_covar_spkernel_qftimes[burnin:]))
    f.write("sparse covar spkernel nqftimes: %s\n" % strstats(sparse_covar_spkernel_nonqftimes[burnin:]))
    f.write("sparse covar spkernel error: %f\n" % np.mean(np.abs(sparse_covar_spkernel[burnin:] - sparse_covar[burnin:])))
    f.write("\n")

    f.write("sparse covar local times: %s\n" % strstats(sparse_covar_local_times[burnin:]))
    f.write("sparse covar local tree time: %s\n" % strstats(sparse_covar_local_tree_times[burnin:]))
    f.write("sparse covar local math time: %s\n" % strstats(sparse_covar_local_math_times[burnin:]))
    f.write("sparse covar local tree/math time ratio: %s\n" % strstats(sparse_covar_local_tree_times[burnin:]/sparse_covar_local_math_times[burnin:]))
    f.write("sparse covar local terms: %s\n" % strstats(sparse_covar_local_terms[burnin:]))
    f.write("sparse covar local dfns: %s\n" % strstats(sparse_covar_local_dfn_evals[burnin:]))
    f.write("sparse covar local wfns: %s\n" % strstats(sparse_covar_local_wfn_evals[burnin:]))
    f.write("sparse covar local qftimes: %s\n" % strstats(sparse_covar_local_qftimes[burnin:]))
    f.write("sparse covar local nqftimes: %s\n" % strstats(sparse_covar_local_nonqftimes[burnin:]))
    f.write("sparse covar local error: %s\n" % strstats(np.abs(sparse_covar_local[burnin:] - sparse_covar[burnin:])))
    f.write("\n")


    """
    f.write("sparse covar spkernel_solve times: %s\n" % strstats(sparse_covar_spkernel_solve_times))
    f.write("sparse covar spkernel_solve error: %f\n" % np.mean(np.abs(sparse_covar_spkernel_solve - sparse_covar)))
    f.write("\n")
    """

    f.write("actual vars: %s\n" % strstats(sparse_covar[burnin:]))
    f.write("actual qfs: %s\n" % strstats(1-sparse_covar[burnin:]))
    f.write("\n")

    f.write("tree: eps_abs %f times: %s\n" % (eps_abs, strstats(tree_covar_times[burnin:])))
    f.write("tree: eps_abs %f terms: %s\n" % (eps_abs, strstats(tree_covar_terms[burnin:])))
    f.write("tree: eps_abs %f zeroterms: %s\n" % (eps_abs, strstats(tree_covar_zeroterms[burnin:])))
    f.write("tree: eps_abs %f nodes touched: %s\n" % (eps_abs, strstats(tree_covar_nodes_touched[burnin:])))
    f.write("tree: eps_abs %f dfn calls: %s\n" % (eps_abs, strstats(tree_covar_dfn_evals[burnin:])))
    f.write("tree: eps_abs %f wfn calls: %s\n" % (eps_abs, strstats(tree_covar_wfn_evals[burnin:])))
    f.write("tree: eps_abs %f dfn cache misses: %s\n" % (eps_abs, strstats(tree_covar_dfn_misses[burnin:])))
    f.write("tree: eps_abs %f wfn cache misses: %s\n" % (eps_abs, strstats(tree_covar_wfn_misses[burnin:])))

    f.write("tree: eps_abs %f qftimes: %s\n" % (eps_abs, strstats(tree_covar_qftimes[burnin:])))
    f.write("tree: eps_abs %f nqftimes: %s\n" % (eps_abs, strstats(tree_covar_nonqftimes[burnin:])))
    f.write("tree: eps_abs %f rel errors: %s \n" %  (eps_abs, strstats(np.abs((tree_covar[burnin:] - sparse_covar[burnin:])/(1-sparse_covar[burnin:])))))
    f.write("tree: eps_abs %f var-rel errors: %s \n" %  (eps_abs, strstats(np.abs((tree_covar[burnin:] - sparse_covar[burnin:])/sparse_covar[burnin:]))))
    f.write("tree: eps_abs %f abs errors: %s \n" %  (eps_abs, strstats(np.abs(tree_covar[burnin:] - sparse_covar[burnin:]))))
    f.write("\n")

    f.write("compiled tree: eps_abs %f times: %s\n" % (eps_abs, strstats(tree_compiled_times[burnin:])))
    f.write("compiled tree: eps_abs %f terms: %s\n" % (eps_abs, strstats(tree_compiled_terms[burnin:])))
    f.write("compiled tree: eps_abs %f zeroterms: %s\n" % (eps_abs, strstats(tree_compiled_zeroterms[burnin:])))
    f.write("compiled tree: eps_abs %f nodes touched: %s\n" % (eps_abs, strstats(tree_compiled_nodes_touched[burnin:])))
    f.write("compiled tree: eps_abs %f dfn calls: %s\n" % (eps_abs, strstats(tree_compiled_dfn_evals[burnin:])))
    f.write("compiled tree: eps_abs %f qftimes: %s\n" % (eps_abs, strstats(tree_compiled_qftimes[burnin:])))
    f.write("compiled tree: eps_abs %f nqftimes: %s\n" % (eps_abs, strstats(tree_compiled_nonqftimes[burnin:])))
    f.write("compiled tree: eps_abs %f rel errors: %s \n" %  (eps_abs, strstats(np.abs((tree_compiled[burnin:] - sparse_covar[burnin:])/(1-sparse_covar[burnin:])))))
    f.write("compiled tree: eps_abs %f var-rel errors: %s \n" %  (eps_abs, strstats(np.abs((tree_compiled[burnin:] - sparse_covar[burnin:])/sparse_covar[burnin:]))))
    f.write("compiled tree: eps_abs %f abs errors: %s \n" %  (eps_abs, strstats(np.abs(tree_compiled[burnin:] - sparse_covar[burnin:]))))
    f.write("\n")


    max_i = np.argmax(tree_covar_times[burnin:]) + burnin
    f.write("most expensive variance point was %s at cost %f\n" % (X_test[max_i,:], tree_covar_times[max_i]))

    f.close()

    np.savez(errorfile, tree_compiled=tree_compiled, tree_covar=tree_covar, sparse_covar=sparse_covar, tree_predict=tree_predict, naive_predict=naive_predict, sparse_covar_spkernel=sparse_covar_spkernel, burnin=burnin)

    print "wrote results to", resultfile