def compute_ksd(z, target): # assume z is a (K, dimZ) object # first compute gradient (slow!) grad = z * 0.0 for i in xrange(z.shape[0]): grad[i] = target.grad(z[i]) return KSD(z, grad)
samples_q = samples1[:n, :] else: # p dist samples_q = samples0[n:, :] # Get samples for p samples_p = samples0[:n, :] assert samples_p.shape == samples_q.shape assert len(samples_p) == n # ------------------------- Perform KDSD test ------------------------- # print "Performing KDSD test ..." model = ERGM(d, rho=rho0, theta=theta0, tau=tau0) # Null model ksd = KSD(neg_fun=model.neg, score_fun=model.score, kernel_fun=wl_kernel) # Use null model kappa_vals = ksd.compute_kappa(samples=samples_q) # Compute U-statistics and bootstrap intervals ksd_stats, _ = ksd_est([kappa_vals]) ksd_boot_list, ksd_thres_list = ksd_boot([kappa_vals]) ksd_pvals = ksd_pvalue(ksd_boot_list, ksd_stats) ksd_stat = ksd_stats[0] ksd_thres = ksd_thres_list[0] ksd_pval = ksd_pvals[0] ksd_pred = 1 * (ksd_stat > ksd_thres) # 0 for p, 1 for q # ------------------------- Perform MMD test ------------------------- # print "Performing MMD test ..."
int_method = 'trapz' ksd_method = 'direct' else: raise ValueError("kernel_type %s not recognized!" % kernel_type) # ------------------------- Perform KSD test ------------------------- # print "Performing KSD test ..." ksd = KSD(dim=model_p.dim, bounds=model_p.bounds, papangelou_fun=model_p.papangelou, kernel_type=kernel_type, int_method=int_method, rbf_h=rbf_h, mp_npts=400, mc_npts=10**4, disp=False) kappa = ksd.compute_kappa(samples_q, method=ksd_method) ksd_stat = ksd.test_statistic(kappa) ksd_thres, ksd_boot = ksd.bootstrap(kappa) ksd_pval = ksd.p_value(ksd_stat, ksd_boot) ksd_pred = 1 * (ksd_stat > ksd_thres) # 0 for p, 1 for q # ------------------------- Perform MMD test ------------------------- # print "Performing MMD test ..."
_, theta = sess.run((opt, theta_q_train), feed_dict={X_ph: X_train[ind], y_ph: y_train[ind], \ lr_ph: lr_train, N_ph: N_train, theta_ph: theta}) if (iteration + 1) % 100 == 0: theta = init_theta(dim=dimTheta_train, n_particle=n_particle) # now start testing total_entropy_acc = [] total_entropy_ll = [] acc_op, ll_op = evaluate(X_ph_test, y_ph_test, theta_ph_test, shapes, activation='sigmoid') grad_test = grad_logp_func(X_ph_test, y_ph_test, theta_ph_test, N_ph, shapes) ksd_op = KSD(theta_ph_test, grad_test) def _chunck_eval(sess, X_test, y_test, theta, N): N_test = y_test.shape[0] acc_total = 0.0 ll_total = 0.0 ksd_total = 0.0 N_batch = int(N_test / batch_size) for i in xrange(N_batch): X_batch = X_test[i * batch_size:(i + 1) * batch_size] y_batch = y_test[i * batch_size:(i + 1) * batch_size] acc, ll, ksd_val = sess.run((acc_op, ll_op, ksd_op), \ feed_dict={X_ph_test: X_batch, y_ph_test: y_batch, \ theta_ph_test: theta, N_ph: N}) acc_total += acc / N_batch
def main(task='pima', dimH=50, seed=42, batch_size_test=32, n_particle_test=500, lr=0.001, method='sgld', dimH_nn=20, n_particle_train=50, T_unroll=10, power=2.0, hsquare=-1.0, lbd=0.01, train_task='crabs'): np.random.seed(seed) tf.set_random_seed(seed) # load data datapath = 'data/' total_dev, total_test = load_uci_data(datapath, task, merge = False) # now define ops N_ph = tf.placeholder(tf.float32, shape=(), name='N_ph') lr_ph = tf.placeholder(tf.float32, shape=(), name='lr_ph') # ops for testing dimX = total_dev[0][0].shape[1] dimY = total_dev[0][1].shape[1] print "=======================" print "define test time model:" shapes, dimTheta = make_shapes(dimX, dimH, dimY, n_hidden_layers=1) print "network shape:", shapes activation_test = 'sigmoid' def grad_logp_func_test(X, y, theta, data_N, shapes): return grad_bnn(X, y, theta, data_N, shapes, activation=activation_test) X_ph_test = tf.placeholder(tf.float32, shape=(batch_size_test, dimX), name = 'X_ph_test') y_ph_test = tf.placeholder(tf.float32, shape=(batch_size_test, dimY), name = 'y_ph_test') theta_ph_test = tf.placeholder(tf.float32, shape=(n_particle_test, dimTheta), name = 'theta_ph_test') if method == 'meta_mcmc': from meta_mcmc_sampler import init_nn_sampler dimH_nn = 20 q_sampler = init_nn_sampler(dimH_nn, name = 'nn_sampler') theta_q_test = q_sampler(theta_ph_test, X_ph_test, y_ph_test, N_ph, grad_logp_func_test, shapes, lr_ph, compute_gamma = True) elif method in ['map', 'kde', 'score', 'stein', 'random']: from nn_sampler import init_nn_sampler dimH_nn = 20 q_sampler = init_nn_sampler(dimH_nn, name = 'nn_sampler') theta_q_test = q_sampler(theta_ph_test, X_ph_test, y_ph_test, N_ph, grad_logp_func_test, shapes, lr_ph) elif method == 'sgld': from mcmc import one_step_dynamics theta_q_test = one_step_dynamics(X_ph_test, y_ph_test, theta_ph_test, N_ph, grad_logp_func_test, 'sgld', lr_ph, shapes) else: raise ValueError('sampler %s not implemented' % method) acc_op, ll_op, prob_op = evaluate(X_ph_test, y_ph_test, theta_ph_test, shapes, activation = activation_test) grad_test = grad_logp_func_test(X_ph_test, y_ph_test, theta_ph_test, N_ph, shapes) ksd_op = KSD(theta_ph_test, grad_test) def _chunck_eval(sess, X_test, y_test, theta, N): N_test = y_test.shape[0] acc_total = 0.0; ll_total = 0.0; ksd_total = 0.0 N_batch = int(N_test / batch_size_test) print N_test, batch_size_test, N_batch for i in xrange(N_batch): X_batch = X_test[i*batch_size_test:(i+1)*batch_size_test] y_batch = y_test[i*batch_size_test:(i+1)*batch_size_test] acc, ll, prob, ksd_val = sess.run((acc_op, ll_op, prob_op, ksd_op), \ feed_dict={X_ph_test: X_batch, y_ph_test: y_batch, \ theta_ph_test: theta, N_ph: N}) acc_total += acc / N_batch; ll_total += ll / N_batch; ksd_total += ksd_val / N_batch print y_batch[:5, 0], prob[:5, 0] return acc_total, ll_total, ksd_total / theta.shape[1] # now check init print "initialise tensorflow variables..." config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config = config) tf.set_random_seed(seed) initialised_var = set([]) init_var_list = set(tf.all_variables()) - initialised_var if len(init_var_list) > 0: init = tf.initialize_variables(var_list = init_var_list) sess.run(init) # load file if method == 'amc': filename = "k%s_H%d_T%d_N%d_p%.1f.pkl" \ % (train_task, dimH_nn, T_unroll, n_particle_train, power) load_params(sess, 'save/', filename) elif method in ['map', 'kde', 'score', 'stein']: filename = "%s_H%d_T%d_N%d_%s_hsquare%.2f_lbd%.2f.pkl" \ % (train_task, dimH_nn, T_unroll, n_particle_train, method, hsquare, lbd) load_params(sess, 'save/', filename) else: pass # now start testing total_entropy_acc = [] total_entropy_ll = [] print "Start testing on tasks %s with stepsize %.4f, seed %d" % (task, lr, seed) T_test = 2000 T_report = 50 if task == 'sonar': T_test = 5000 T_report = 100 print "for sonar dataset, run longer T=%d ..." % T_test results = {'acc':[], 'll':[], 'ksd':[], 'time': []} for data_i in range(0, len(total_dev)): # For each dataset training on dev and testing on test dataset X_dev, y_dev = total_dev[data_i] X_test, y_test = total_test[data_i] X_dev, y_dev = shuffle(X_dev, y_dev) X_test, y_test = shuffle(X_test, y_test) dev_N = X_dev.shape[0] print X_dev.shape, y_dev.shape total_m_acc = [] total_m_ll = [] total_m_ksd = [] total_time = [] # nn sampler theta = init_theta(dim = dimTheta, n_particle = n_particle_test, seed=seed) # evaluate the start point acc, ll, ksd = _chunck_eval(sess, X_test, y_test, theta, dev_N) print acc, ll, ksd, 0, theta[0, 0], theta[0, 1], dev_N total_m_acc.append(acc) total_m_ll.append(ll) total_m_ksd.append(ksd) total_time.append(0.0) start = time.time() lr_test = lr #/ dev_N for t in tqdm(range(T_test)): ind = np.random.randint(0, dev_N, batch_size_test) theta = sess.run(theta_q_test, {X_ph_test: X_dev[ind], y_ph_test: y_dev[ind], \ N_ph: dev_N, theta_ph_test: theta, lr_ph: lr}) if (t+1) % T_report == 0: end = time.time() acc, ll, ksd = _chunck_eval(sess, X_test, y_test, theta, dev_N) print acc, ll, ksd, t+1, theta[0, 0], theta[0, 1], dev_N total_m_acc.append(acc) total_m_ll.append(ll) total_m_ksd.append(ksd) total_time.append(end - start) start = time.time() results['acc'].append(total_m_acc) results['ll'].append(total_m_ll) results['ksd'].append(total_m_ksd) results['time'].append(total_time) print "Evaluation of dataset=%d" %(data_i) print "Evaluation of our methods, ", acc, ll, ksd print "\n" print "Final results" results['acc'] = np.array(results['acc']) results['ll'] = np.array(results['ll']) results['ksd'] = np.array(results['ksd']) print "\nOur methods----" print "acc", np.mean(results['acc'][:, -1]), np.std(results['acc'][:, -1]) print "ll", np.mean(results['ll'][:, -1]), np.std(results['ll'][:, -1]) print "ksd", np.mean(results['ksd'][:, -1]), np.std(results['ksd'][:, -1]) ## save results if method in ['kde', 'score', 'stein']: method = method + '_hsquare%.2f_lbd%.2f' % (hsquare, lbd) filename = "bnn_%s_%s.pkl" % (task, method) savepath = 'results/' if not os.path.isdir(savepath): os.mkdir(savepath) print 'create path ' + savepath f = open(savepath+filename, 'w') pickle.dump(results, f) print "results saved in results/%s" % (savepath+filename)