def profile_tree(dataset_name, model_name, tag=None, sgp=None, n=None, test_n=None, burnin=10, cutoff_rule=2): X_test, y_test = test_data(dataset_name) gp = trained_gp(dataset_name, model_name, n=n, tag=tag, build_tree=True, leaf_bin_width=0.5, build_dense_Kinv_hack=True) if test_n is None: test_n = min(len(X_test), 5000) print "loaded GP, evaluating timings on %d test points..." % test_n eps_abs = 0.001 * gp.noise_var tree_covar = np.zeros((test_n,)) tree_covar_terms = np.zeros((test_n,)) tree_covar_distevals = np.zeros((test_n,)) tree_covar_times = np.zeros((test_n,)) tree_covar_qftimes = np.zeros((test_n,)) tree_covar_nonqftimes = np.zeros((test_n,)) gp.nonqf_time = 0 for i in range(test_n): t6 = time.time() tree_covar[i] = gp.covariance_double_tree(X_test[i:i+1,:], eps_abs=eps_abs, cutoff_rule=cutoff_rule) t7 = time.time() tree_covar_terms[i] = gp.double_tree.fcalls tree_covar_distevals[i] = gp.double_tree.dfn_evals tree_covar_times[i] = t7-t6 tree_covar_qftimes[i] = gp.qf_time tree_covar_nonqftimes[i] = gp.nonqf_time print "tree: eps_abs %f times: %s" % (eps_abs, strstats(tree_covar_times[burnin:])) print "tree: eps_abs %f qftimes: %s" % (eps_abs, strstats(tree_covar_qftimes[burnin:])) print "tree: eps_abs %f nqftimes: %s" % (eps_abs, strstats(tree_covar_nonqftimes[burnin:])) print "tree: eps_abs %f terms: %s" % (eps_abs, strstats(tree_covar_terms[burnin:])) print "tree: eps_abs %f dfnevals: %s" % (eps_abs, strstats(tree_covar_distevals[burnin:]))
def eval_gp(dataset_name, model_name, tag=None, sgp=None, n=None, test_n=None, burnin=10, cutoff_rule=2): X_test, y_test = test_data(dataset_name) gp = trained_gp(dataset_name, model_name, n=n, tag=tag, build_tree=True, leaf_bin_width=None, build_dense_Kinv_hack=True, compile_tree=compiled_tree_fname(dataset_name, model_name, tag)) if test_n is None: test_n = min(len(X_test), 5000) print "loaded GP, evaluating timings on %d test points..." % test_n resultfile = timing_results_fname(dataset_name, model_name, tag) errorfile = timing_errors_fname(dataset_name, model_name, tag) naive_predict = np.zeros(test_n) naive_predict_times = np.zeros(test_n) tree_predict = np.zeros(test_n) tree_predict_terms = np.zeros(test_n, dtype=int) tree_predict_times = np.zeros(test_n) dense_covar = np.zeros(test_n) dense_covar_times = np.zeros(test_n) sparse_covar = np.zeros(test_n) sparse_covar_times = np.zeros(test_n) sparse_covar_qftimes = np.zeros(test_n) sparse_covar_nonqftimes = np.zeros(test_n) sparse_covar_spkernel = np.zeros(test_n) sparse_covar_spkernel_times = np.zeros(test_n) sparse_covar_spkernel_qftimes = np.zeros(test_n) sparse_covar_spkernel_nonqftimes = np.zeros(test_n) sparse_covar_local = np.zeros(test_n) sparse_covar_local_times = np.zeros(test_n) sparse_covar_local_tree_times = np.zeros(test_n) sparse_covar_local_math_times = np.zeros(test_n) sparse_covar_local_terms = np.zeros(test_n) sparse_covar_local_dfn_evals = np.zeros(test_n) sparse_covar_local_wfn_evals = np.zeros(test_n) sparse_covar_local_qftimes = np.zeros(test_n) sparse_covar_local_nonqftimes = np.zeros(test_n) sparse_covar_spkernel_solve = np.zeros(test_n) sparse_covar_spkernel_solve_times = np.zeros(test_n) print "naive predict" for i in range(test_n): t0 = time.time() naive_predict[i] = gp.predict_naive(X_test[i:i+1,:]) t1 = time.time() naive_predict_times[i] = t1-t0 print "tree predict" for i in range(test_n): t2 = time.time() tree_predict[i] = gp.predict(X_test[i:i+1,:], eps=1e-4) t3 = time.time() tree_predict_times[i] = t3-t2 tree_predict_terms[i] = gp.predict_tree.terms print "treedense/local covar" gp.nonqf_time = 0 for i in range(test_n): t4 = time.time() sparse_covar_local[i] = gp.covariance_treedense(X_test[i:i+1,:]) t5 = time.time() sparse_covar_local_times[i] = t5-t4 sparse_covar_local_tree_times[i] = gp.predict_tree.dense_hack_tree_s sparse_covar_local_math_times[i] = gp.predict_tree.dense_hack_math_s sparse_covar_local_terms[i] = gp.qf_terms sparse_covar_local_dfn_evals[i] = gp.qf_dfn_evals sparse_covar_local_wfn_evals[i] = gp.qf_wfn_evals sparse_covar_local_qftimes[i] = gp.qf_time sparse_covar_local_nonqftimes[i] = gp.nonqf_time print "sparse covar" gp.nonqf_time = 0 for i in range(test_n): t4 = time.time() sparse_covar[i] = gp.covariance(X_test[i:i+1,:]) t5 = time.time() sparse_covar_times[i] = t5-t4 sparse_covar_qftimes[i] = gp.qf_time sparse_covar_nonqftimes[i] = gp.nonqf_time print "spkernel" for i in range(test_n): t41 = time.time() sparse_covar_spkernel[i] = gp.covariance_spkernel(X_test[i:i+1,:]) t51 = time.time() sparse_covar_spkernel_times[i] = t51-t41 sparse_covar_spkernel_qftimes[i] = gp.qf_time sparse_covar_spkernel_nonqftimes[i] = gp.nonqf_time """ for i in range(test_n): t41 = time.time() sparse_covar_spkernel_solve[i] = gp.covariance_spkernel_solve(X_test[i:i+1,:]) t51 = time.time() sparse_covar_spkernel_solve_times[i] = t51-t41 """ has_dense = True try: for i in range(100): t42 = time.time() dense_covar[i] = gp.covariance_dense(X_test[i:i+1,:]) t52 = time.time() dense_covar_times[i] = t52-t42 except: has_dense=False f = open(resultfile, 'w') eps_abs = 0.001 * gp.noise_var tree_covar = np.zeros((test_n,)) tree_covar_terms = np.zeros((test_n,)) tree_covar_zeroterms = np.zeros((test_n,)) tree_covar_nodes_touched = np.zeros((test_n,)) tree_covar_dfn_evals = np.zeros((test_n,)) tree_covar_wfn_evals = np.zeros((test_n,)) tree_covar_dfn_misses = np.zeros((test_n,)) tree_covar_wfn_misses = np.zeros((test_n,)) tree_covar_times = np.zeros((test_n,)) tree_covar_qftimes = np.zeros((test_n,)) tree_covar_nonqftimes = np.zeros((test_n,)) for i in range(test_n): t6 = time.time() tree_covar[i] = gp.covariance_double_tree(X_test[i:i+1,:], eps_abs=eps_abs, cutoff_rule=cutoff_rule) t7 = time.time() tree_covar_terms[i] = gp.qf_terms tree_covar_zeroterms[i] = gp.qf_zeroterms tree_covar_nodes_touched[i] = gp.qf_nodes_touched tree_covar_dfn_evals[i] = gp.qf_dfn_evals tree_covar_wfn_evals[i] = gp.qf_wfn_evals tree_covar_dfn_misses[i] = gp.qf_dfn_misses tree_covar_wfn_misses[i] = gp.qf_wfn_misses tree_covar_times[i] = t7-t6 tree_covar_qftimes[i] = gp.qf_time tree_covar_nonqftimes[i] = gp.nonqf_time tree_compiled = np.zeros((test_n,)) tree_compiled_terms = np.zeros((test_n,)) tree_compiled_zeroterms = np.zeros((test_n,)) tree_compiled_nodes_touched = np.zeros((test_n,)) tree_compiled_dfn_evals = np.zeros((test_n,)) tree_compiled_times = np.zeros((test_n,)) tree_compiled_qftimes = np.zeros((test_n,)) tree_compiled_nonqftimes = np.zeros((test_n,)) for i in range(test_n): t6 = time.time() tree_compiled[i] = gp.covariance_compiled(X_test[i:i+1,:], eps_abs=eps_abs) t7 = time.time() tree_compiled_terms[i] = gp.qf_terms tree_compiled_zeroterms[i] = gp.qf_zeroterms tree_compiled_nodes_touched[i] = gp.qf_nodes_touched tree_compiled_dfn_evals[i] = gp.qf_dfn_evals tree_compiled_times[i] = t7-t6 tree_compiled_qftimes[i] = gp.qf_time tree_compiled_nonqftimes[i] = gp.nonqf_time f.write("naive predict times: %s\n" % strstats(naive_predict_times[burnin:])) f.write("\n") f.write("tree predict times: %s\n" % strstats(tree_predict_times[burnin:])) f.write("tree predict terms: %s\n" % strstats(tree_predict_terms[burnin:])) f.write("tree predict errors: %s\n" % strstats(np.abs(tree_predict[burnin:] - naive_predict[burnin:]))) f.write("\n") if has_dense: f.write("dense covar times: %s\n" % strstats(dense_covar_times[burnin:])) f.write("sparse covar times: %s\n" % strstats(sparse_covar_times[burnin:])) f.write("sparse covar qftimes: %s\n" % strstats(sparse_covar_qftimes[burnin:])) f.write("sparse covar nqftimes: %s\n" % strstats(sparse_covar_nonqftimes[burnin:])) f.write("\n") f.write("sparse covar spkernel times: %s\n" % strstats(sparse_covar_spkernel_times[burnin:])) f.write("sparse covar spkernel qftimes: %s\n" % strstats(sparse_covar_spkernel_qftimes[burnin:])) f.write("sparse covar spkernel nqftimes: %s\n" % strstats(sparse_covar_spkernel_nonqftimes[burnin:])) f.write("sparse covar spkernel error: %f\n" % np.mean(np.abs(sparse_covar_spkernel[burnin:] - sparse_covar[burnin:]))) f.write("\n") f.write("sparse covar local times: %s\n" % strstats(sparse_covar_local_times[burnin:])) f.write("sparse covar local tree time: %s\n" % strstats(sparse_covar_local_tree_times[burnin:])) f.write("sparse covar local math time: %s\n" % strstats(sparse_covar_local_math_times[burnin:])) f.write("sparse covar local tree/math time ratio: %s\n" % strstats(sparse_covar_local_tree_times[burnin:]/sparse_covar_local_math_times[burnin:])) f.write("sparse covar local terms: %s\n" % strstats(sparse_covar_local_terms[burnin:])) f.write("sparse covar local dfns: %s\n" % strstats(sparse_covar_local_dfn_evals[burnin:])) f.write("sparse covar local wfns: %s\n" % strstats(sparse_covar_local_wfn_evals[burnin:])) f.write("sparse covar local qftimes: %s\n" % strstats(sparse_covar_local_qftimes[burnin:])) f.write("sparse covar local nqftimes: %s\n" % strstats(sparse_covar_local_nonqftimes[burnin:])) f.write("sparse covar local error: %s\n" % strstats(np.abs(sparse_covar_local[burnin:] - sparse_covar[burnin:]))) f.write("\n") """ f.write("sparse covar spkernel_solve times: %s\n" % strstats(sparse_covar_spkernel_solve_times)) f.write("sparse covar spkernel_solve error: %f\n" % np.mean(np.abs(sparse_covar_spkernel_solve - sparse_covar))) f.write("\n") """ f.write("actual vars: %s\n" % strstats(sparse_covar[burnin:])) f.write("actual qfs: %s\n" % strstats(1-sparse_covar[burnin:])) f.write("\n") f.write("tree: eps_abs %f times: %s\n" % (eps_abs, strstats(tree_covar_times[burnin:]))) f.write("tree: eps_abs %f terms: %s\n" % (eps_abs, strstats(tree_covar_terms[burnin:]))) f.write("tree: eps_abs %f zeroterms: %s\n" % (eps_abs, strstats(tree_covar_zeroterms[burnin:]))) f.write("tree: eps_abs %f nodes touched: %s\n" % (eps_abs, strstats(tree_covar_nodes_touched[burnin:]))) f.write("tree: eps_abs %f dfn calls: %s\n" % (eps_abs, strstats(tree_covar_dfn_evals[burnin:]))) f.write("tree: eps_abs %f wfn calls: %s\n" % (eps_abs, strstats(tree_covar_wfn_evals[burnin:]))) f.write("tree: eps_abs %f dfn cache misses: %s\n" % (eps_abs, strstats(tree_covar_dfn_misses[burnin:]))) f.write("tree: eps_abs %f wfn cache misses: %s\n" % (eps_abs, strstats(tree_covar_wfn_misses[burnin:]))) f.write("tree: eps_abs %f qftimes: %s\n" % (eps_abs, strstats(tree_covar_qftimes[burnin:]))) f.write("tree: eps_abs %f nqftimes: %s\n" % (eps_abs, strstats(tree_covar_nonqftimes[burnin:]))) f.write("tree: eps_abs %f rel errors: %s \n" % (eps_abs, strstats(np.abs((tree_covar[burnin:] - sparse_covar[burnin:])/(1-sparse_covar[burnin:]))))) f.write("tree: eps_abs %f var-rel errors: %s \n" % (eps_abs, strstats(np.abs((tree_covar[burnin:] - sparse_covar[burnin:])/sparse_covar[burnin:])))) f.write("tree: eps_abs %f abs errors: %s \n" % (eps_abs, strstats(np.abs(tree_covar[burnin:] - sparse_covar[burnin:])))) f.write("\n") f.write("compiled tree: eps_abs %f times: %s\n" % (eps_abs, strstats(tree_compiled_times[burnin:]))) f.write("compiled tree: eps_abs %f terms: %s\n" % (eps_abs, strstats(tree_compiled_terms[burnin:]))) f.write("compiled tree: eps_abs %f zeroterms: %s\n" % (eps_abs, strstats(tree_compiled_zeroterms[burnin:]))) f.write("compiled tree: eps_abs %f nodes touched: %s\n" % (eps_abs, strstats(tree_compiled_nodes_touched[burnin:]))) f.write("compiled tree: eps_abs %f dfn calls: %s\n" % (eps_abs, strstats(tree_compiled_dfn_evals[burnin:]))) f.write("compiled tree: eps_abs %f qftimes: %s\n" % (eps_abs, strstats(tree_compiled_qftimes[burnin:]))) f.write("compiled tree: eps_abs %f nqftimes: %s\n" % (eps_abs, strstats(tree_compiled_nonqftimes[burnin:]))) f.write("compiled tree: eps_abs %f rel errors: %s \n" % (eps_abs, strstats(np.abs((tree_compiled[burnin:] - sparse_covar[burnin:])/(1-sparse_covar[burnin:]))))) f.write("compiled tree: eps_abs %f var-rel errors: %s \n" % (eps_abs, strstats(np.abs((tree_compiled[burnin:] - sparse_covar[burnin:])/sparse_covar[burnin:])))) f.write("compiled tree: eps_abs %f abs errors: %s \n" % (eps_abs, strstats(np.abs(tree_compiled[burnin:] - sparse_covar[burnin:])))) f.write("\n") max_i = np.argmax(tree_covar_times[burnin:]) + burnin f.write("most expensive variance point was %s at cost %f\n" % (X_test[max_i,:], tree_covar_times[max_i])) f.close() np.savez(errorfile, tree_compiled=tree_compiled, tree_covar=tree_covar, sparse_covar=sparse_covar, tree_predict=tree_predict, naive_predict=naive_predict, sparse_covar_spkernel=sparse_covar_spkernel, burnin=burnin) print "wrote results to", resultfile