def test_gumbel_softmax_trick(): M = 100 K = 5 tau = 0.05 G = tf.placeholder(tf.float64, shape=(1, M, K)) beta_init = tf.random_normal(((K - 1,)), 0.0, 0.01, dtype=tf.float64) beta = tf.get_variable("beta", dtype=tf.float64, initializer=beta_init) exp_beta = tf.exp(beta) alpha = tf.concat((exp_beta, tf.ones((1,), tf.float64)), axis=0) / ( tf.reduce_sum(exp_beta) + 1.0 ) C = gumbel_softmax_trick(G, alpha, tau) sess = tf.Session() sess.run(tf.global_variables_initializer()) _alpha = sess.run(alpha) _G = sample_gumbel(M, K) _C = sess.run(C, {G: np.expand_dims(_G, 0)}) C_true = gumbel_softmax_trick_np(_G, _alpha, tau) assert approx_equal(C_true, _C, EPS) assert approx_equal(np.sum(_C, 1), 1.0, EPS) assert np.sum(np.isnan(_C)) + np.sum(np.isinf(_C)) == 0 return None
def test_isotropic_normal(): for D in Ds: df_fac = 2 * D for k in range(K): mu = np.random.normal(0.0, 1.0, (D, )) scale = np.abs(np.random.normal(0.0, 1.0, (1, ))) + 0.001 iso_mvn_dist = { "family": "isotropic_normal", "mu": mu, "scale": scale } Sigma = scale * np.eye(D) iso_mvn_sampler = get_sampler_func(iso_mvn_dist, D) iso_mvn_density = get_density_func(iso_mvn_dist, D) p_zs = np.zeros((n, )) for i in range(n): z = iso_mvn_sampler() assert np.all(np.isfinite(z)) assert np.all(np.isreal(z)) p_z = iso_mvn_density(z) diff = np.expand_dims(z - mu, 1) p_z_true = ( 1.0 / np.sqrt(np.linalg.det(2 * np.pi * Sigma)) * np.exp(-0.5 * np.dot(np.transpose(diff), np.dot(np.linalg.inv(Sigma), diff)))) assert approx_equal(p_z, p_z_true, EPS) p_zs[i] = p_z H = -np.mean(np.log(p_zs)) H_true = 0.5 * np.log(np.linalg.det(2 * np.pi * np.exp(1) * Sigma)) assert approx_equal(H, H_true, MVN_H_EPS) return None
def test_multivariate_normal(): for D in Ds: df_fac = 2 * D Sigma_dist = scipy.stats.invwishart(df=df_fac, scale=df_fac * np.eye(D)) for k in range(K): mu = np.random.normal(0.0, 1.0, (D, )) Sigma = Sigma_dist.rvs(1) mvn_dist = { "family": "multivariate_normal", "mu": mu, "Sigma": Sigma } mvn_sampler = get_sampler_func(mvn_dist, D) mvn_density = get_density_func(mvn_dist, D) p_zs = np.zeros((n, )) for i in range(n): z = mvn_sampler() assert np.all(np.isfinite(z)) assert np.all(np.isreal(z)) p_z = mvn_density(z) diff = np.expand_dims(z - mu, 1) p_z_true = ( 1.0 / np.sqrt(np.linalg.det(2 * np.pi * Sigma)) * np.exp(-0.5 * np.dot(np.transpose(diff), np.dot(np.linalg.inv(Sigma), diff)))) assert approx_equal(p_z, p_z_true, EPS) p_zs[i] = p_z H = -np.mean(np.log(p_zs)) H_true = 0.5 * np.log(np.linalg.det(2 * np.pi * np.exp(1) * Sigma)) assert approx_equal(H, H_true, MVN_H_EPS) return None
def test_STGCircuit(): np.random.seed(0) M = 500 dt = 0.025 T = 200 fft_start = 0 w = 20 true_sys = stg_circuit(dt, T, fft_start, w=w) mean = 0.55 variance = 0.0001 fixed_params = {} behavior = {"type": "freq", "mean": mean, "variance": variance} model_opts = {"dt": dt, "T": T, "fft_start": fft_start, "w": w} system = STGCircuit(fixed_params, behavior, model_opts) assert system.name == "STGCircuit" assert system.behavior_str == "freq_mu=5.50E-01_3.03E-01" assert approx_equal(system.behavior["mean"], 0.55, EPS) assert system.all_params == ["g_el", "g_synA", "g_synB"] assert system.free_params == ["g_el", "g_synA", "g_synB"] assert system.z_labels == [r"$g_{el}$", r"$g_{synA}$", r"$g_{synB}$"] assert system.T_x_labels == [r"$f_{h}$", r"$f_{h}^2$"] assert system.D == 3 assert system.num_suff_stats == 2 Z = tf.placeholder(dtype=DTYPE, shape=(1, M, 3)) _Z = np.random.uniform(0.0, 10.0, (1, M, 2)) * 1e-9 synB = 5e-9 _Z = np.concatenate((_Z, synB * np.ones((1, M, 1))), axis=2) x_true = np.zeros((M, 15, T + 1)) T_x_true = np.zeros((M, 10)) for i in range(M): g_el = _Z[0, i, 0] g_synA = _Z[0, i, 1] g_synB = _Z[0, i, 2] x_true[i, :, :] = true_sys.simulate(g_el, g_synA, g_synB).T T_x_true[i, :] = true_sys.T_x(g_el, g_synA, g_synB) print('txtrue', np.sum(np.isnan(T_x_true))) T_x = system.compute_suff_stats(Z) x_t = system.simulate(Z, db=True) with tf.Session() as sess: _x_t, _T_x = sess.run([x_t, T_x], {Z: _Z / 1.0e-9}) print('_T_x', np.sum(np.isnan(_T_x))) print('tf') print(_T_x[0]) print('np') print(T_x_true) assert approx_equal(np.transpose(_x_t, [1, 2, 0]), x_true, EPS) assert approx_equal(_T_x[0], T_x_true, EPS, allow_special=True) return None
def test_multivariate_normal(): for D in Ds: # test initialization family = MultivariateNormal(D) assert family.name == "MultivariateNormal" assert family.D_Z == D assert family.num_suff_stats == D + D * (D + 1) / 2 assert family.has_log_p assert not family.has_support_map assert family.eta_dist["family"] == "iso_mvn_and_iso_iw" assert approx_equal(family.eta_dist["mu"], np.zeros((D, )), EPS) assert family.eta_dist["scale"] == 0.1 assert family.eta_dist["df_fac"] == 5 # test methods true_family = multivariate_normal(D) Z = tf.placeholder(dtype=DTYPE, shape=(None, None, D)) T_z = family.compute_suff_stats(Z, [], []) log_base_measure = family.compute_log_base_measure(Z) _Z = np.random.normal(0.0, 1.0, ((K, n, D))) T_z_true = np.zeros((K, n, family.num_suff_stats)) log_base_measure_true = np.zeros((K, n)) for k in range(K): for i in range(n): T_z_true[k, i, :] = true_family.compute_suff_stats(_Z[k, i, :]) log_base_measure_true[ k, i] = true_family.compute_log_base_measure(Z) with tf.Session() as sess: _T_z, _log_base_measure = sess.run([T_z, log_base_measure], {Z: _Z}) assert approx_equal(_T_z, T_z_true, EPS) assert approx_equal(_log_base_measure, log_base_measure_true, EPS) # compute mu _, _, _, params = family.draw_etas(K) for k in range(K): mu = params[k]["mu"] Sigma = params[k]["Sigma"] mu_true = np.zeros((D + (D * (D + 1) // 2), )) mu_true[:D] = mu ind = D for i in range(D): for j in range(i, D): mu_true[ind] = mu[i] * mu[j] + Sigma[i, j] ind += 1 mu_fam = family.compute_mu(params[k]) assert approx_equal(mu_true, mu_fam, EPS) return None
def eval_rank1_spont_chaotic_solve(mu_0, delta0_0, deltainf_0, its, langevin_eps, EPS): n = 16 g_start = 0.25 g_end = 4.0 _g = np.linspace(g_start, g_end, n) # np variables _mu_init = mu_0*np.ones((n,)) _delta_0_init = delta0_0*np.ones((n,)) _delta_inf_init = deltainf_0*np.ones((n,)) _Mm = 1.1*np.ones((n,)) # np.abs(np.random.normal(0.0, 1.0, (n,))) + .001 _Mn = 2.0*np.ones((n,)) _Sm = 1.0*np.ones((n,)) # tf variables mu_init = tf.placeholder(dtype=DTYPE, shape=(n,)) delta_0_init = tf.placeholder(dtype=DTYPE, shape=(n,)) delta_inf_init = tf.placeholder(dtype=DTYPE, shape=(n,)) g = tf.placeholder(dtype=DTYPE, shape=(n,)) Mm = tf.placeholder(dtype=DTYPE, shape=(n,)) Mn = tf.placeholder(dtype=DTYPE, shape=(n,)) Sm = tf.placeholder(dtype=DTYPE, shape=(n,)) mu_true = np.zeros((n,)) delta_0_true = np.zeros((n,)) delta_inf_true = np.zeros((n,)) for k in range(n): y0_k = [_mu_init[k], _delta_0_init[k], _delta_inf_init[k]] VecPar_k = [_Mm[k], _Mn[k], _Sm[k], 0.0] y = mf.SolveChaotic(y0_k, _g[k], VecPar_k, tolerance=1e-8, backwards=1) mu_true[k] = y[0] delta_0_true[k] = y[1] delta_inf_true[k] = y[2] mu, delta_0, delta_inf = rank1_spont_chaotic_solve(mu_init, delta_0_init, delta_inf_init, \ g, Mm, Mn, Sm, its, langevin_eps, gauss_quad_pts=50) feed_dict = {mu_init:_mu_init, delta_0_init:_delta_0_init, delta_inf_init:_delta_inf_init, \ g:_g, Mm:_Mm, Mn:_Mn, Sm:_Sm} with tf.Session() as sess: _mu, _delta_0, _delta_inf = sess.run([mu, delta_0, delta_inf], feed_dict) assert(approx_equal(_mu, mu_true, EPS)) assert(approx_equal(_delta_0, delta_0_true, EPS)) assert(approx_equal(_delta_inf, delta_inf_true, EPS)) return None
def test_rank1_spont_static_solve(): n = 1000 _g = np.random.uniform(0.01, 5.0, n) # np variables _mu_init = np.random.uniform(0.01, 150.0, n) _delta_0_init = np.random.uniform(0.01, 150.0, n) _Mm = np.random.normal(-5.0, 5.0, (n, )) _Mn = np.random.normal(-5.0, 5.0, (n, )) _Sm = np.random.uniform(0.01, 5.0, n) # tf variables mu_init = tf.placeholder(dtype=DTYPE, shape=(n, )) delta_0_init = tf.placeholder(dtype=DTYPE, shape=(n, )) g = tf.placeholder(dtype=DTYPE, shape=(n, )) Mm = tf.placeholder(dtype=DTYPE, shape=(n, )) Mn = tf.placeholder(dtype=DTYPE, shape=(n, )) Sm = tf.placeholder(dtype=DTYPE, shape=(n, )) mu, delta_0 = rank1_spont_static_solve(mu_init, delta_0_init, g, Mm, Mn, Sm, its, langevin_eps, gauss_quad_pts=200) mu_np, delta_0_np = rank1_spont_static_solve_np(_mu_init, _delta_0_init, _g, _Mm, _Mn, _Sm, its, langevin_eps) feed_dict = { mu_init: _mu_init, delta_0_init: _delta_0_init, g: _g, Mm: _Mm, Mn: _Mn, Sm: _Sm, } with tf.Session() as sess: _mu, _delta_0 = sess.run([mu, delta_0], feed_dict) assert approx_equal(_mu, mu_np, 1e-6) assert approx_equal(_delta_0, delta_0_np, 1e-6) return None
def eval_single_gaussian_integral(np_func, tf_func, np_num_pts, tf_num_pts, EPS): y_true = np.zeros((num_mus, num_delta0s)) for i in range(num_mus): mu = mus[i] for j in range(num_delta0s): delta0 = delta0s[j] y_true[i, j] = np_func(mu, delta0, np_num_pts) mu = tf.placeholder(dtype=DTYPE, shape=(num_mus * num_delta0s, )) delta0 = tf.placeholder(dtype=DTYPE, shape=(num_mus * num_delta0s, )) _delta0, _mu = np.meshgrid(delta0s, mus) _mu = np.reshape(_mu, (num_mus * num_delta0s, )) _delta0 = np.reshape(_delta0, (num_mus * num_delta0s, )) y = tf_func(mu, delta0, tf_num_pts) y = tf.reshape(y, [num_mus, num_delta0s]) with tf.Session() as sess: _y = sess.run(y, {mu: _mu, delta0: _delta0}) assert approx_equal(y_true, _y, EPS, perc=False) return None
def test_quartic_formula(): M = 100 a = tf.placeholder(dtype=DTYPE, shape=(M, 1)) b = tf.placeholder(dtype=DTYPE, shape=(M, 1)) c = tf.placeholder(dtype=DTYPE, shape=(M, 1)) d = tf.placeholder(dtype=DTYPE, shape=(M, 1)) e = tf.placeholder(dtype=DTYPE, shape=(M, 1)) _a = np.random.normal(0.0, 100.0, (M, 1)) _b = np.random.normal(0.0, 100.0, (M, 1)) _c = np.random.normal(0.0, 100.0, (M, 1)) _d = np.random.normal(0.0, 100.0, (M, 1)) _e = np.random.normal(0.0, 100.0, (M, 1)) roots = quartic_roots(a, b, c, d, e) with tf.Session() as sess: _roots = sess.run(roots, {a: _a, b: _b, c: _c, d: _d, e: _e}) _roots = np.concatenate(_roots, axis=1) for i in range(M): p = np.array([_a[i, 0], _b[i, 0], _c[i, 0], _d[i, 0], _e[i, 0]]) roots_np = np.roots(p) sorted_roots_np = sort_quartic_roots(roots_np) sorted_roots_tf = sort_quartic_roots(_roots[i]) assert approx_equal(sorted_roots_np, sorted_roots_tf, EPS) return None
def eval_interval_flow_at_dim(dim, K, n): num_params = get_num_flow_params(IntervalFlow, dim) out_dim = get_flow_out_dim(IntervalFlow, dim) params1 = tf.placeholder(dtype=DTYPE, shape=(None, num_params)) inputs1 = tf.placeholder(dtype=DTYPE, shape=(None, None, dim)) _params = np.random.normal(0.0, 1.0, (K, num_params)) _inputs = np.random.normal(0.0, 1.0, (K, n, dim)) _a = np.random.normal(0.0, 1.0, (K, dim)) _b = _a + np.abs(np.random.normal(0.0, 1.0, (K, dim))) + 0.001 # compute ground truth out_true = np.zeros((K, n, out_dim)) log_det_jac_true = np.zeros((K, n)) for k in range(K): _params_k = _params[k, :] _a_k = _a[k, :] _b_k = _b[k, :] for j in range(n): out_true[k, j, :], log_det_jac_true[k, j] = interval_flow( _inputs[k, j, :], _params_k, _a_k, _b_k) _out1 = np.zeros((K, n, out_dim)) _f_inv_z1 = np.zeros((K, n, dim)) _log_det_jac1 = np.zeros((K, n)) with tf.Session() as sess: for k in range(K): _a_k = _a[k, :] _b_k = _b[k, :] flow1 = IntervalFlow(params1, inputs1, _a_k, _b_k) out1, log_det_jac1 = flow1.forward_and_jacobian() f_inv_z = flow1.inverse(out1) _params_k = np.expand_dims(_params[k, :], 0) _inputs_k = np.expand_dims(_inputs[k, :, :], 0) feed_dict = {params1: _params_k, inputs1: _inputs_k} _out1_k, _log_det_jac1_k, _f_inv_z = sess.run( [out1, log_det_jac1, f_inv_z], feed_dict) _out1[k, :, :] = _out1_k[0] _f_inv_z1[k, :, :] = _f_inv_z[0] _log_det_jac1[k, :] = _log_det_jac1_k[0] assert approx_equal(_out1, out_true, EPS) assert approx_equal(_f_inv_z1, _inputs, EPS) assert approx_equal(_log_det_jac1, log_det_jac_true, EPS) return None
def test_flow_param_initialization(): Ds = [1, 2, 4, 20, 100, 1000] all_glorot_uniform_flows = [AffineFlow, ElemMultFlow, ShiftFlow, RealNVP] all_no_param_flows = [ ExpFlow, SimplexBijectionFlow, SoftPlusFlow, TanhFlow ] with tf.Session() as sess: for D in Ds: for flow in all_glorot_uniform_flows: if (flow == RealNVP): opt_params = {'num_masks': 1, 'nlayers': 1, 'upl': 1} inits, dims = get_flow_param_inits(flow, D, opt_params) assert sum(dims) == get_num_flow_params( flow, D, opt_params) else: inits, dims = get_flow_param_inits(flow, D) assert sum(dims) == get_num_flow_params(flow, D) assert len(inits) == 1 # assert(isinstance(inits[0], tf.glorot_uniform_initializer)) for flow in all_no_param_flows: inits, dims = get_flow_param_inits(flow, D) assert len(inits) == 1 assert inits[0] == None assert len(dims) == 1 assert dims[0] == 0 """ inits, dims = get_flow_param_inits(CholProdFlow, D) """ inits, dims = get_flow_param_inits(PlanarFlow, D) assert approx_equal(sess.run(inits[0]), np.zeros(D), EPS) # assert(isinstance(inits[1], tf.glorot_uniform_initializer)) assert approx_equal(sess.run(inits[2]), 0.0, EPS) assert dims == [D, D, 1] """ inits, dims = get_flow_param_inits(RadialFlow, D) """ """ inits, dims = get_flow_param_inits(StructuredSpinnerFlow, D) """ """ inits, dims = get_flow_param_inits(StructuredSpinnerTanhFlow, D) """ return None
def test_fisher_information_matrix(): # Test D=1 W = tf.placeholder(tf.float64, (1,1,1)) log_q_z = 2 * tf.pow(W, 3) Z = 3.0 * W Z_INV = Z / 3.0 I = fisher_information_matrix(log_q_z, W, Z, Z_INV) with tf.Session() as sess: _W = np.ones((1,1,1)) _I = sess.run(I, {W:_W}) assert(approx_equal(np.array([[-(2.0/9.0)*12.0]]), _I, EPS)) _W = np.zeros((1,1,1)) _I = sess.run(I, {W:_W}) assert(approx_equal(np.array([[0.0]]), _I, EPS)) # Test D=2 W = tf.placeholder(tf.float64, (1,1,2)) A = np.array([[1.0, 0.0], [0.0, 1.0]]) log_q_z = tf.reduce_sum(tf.matmul(np.expand_dims(A, 0), tf.transpose(tf.square(W), [0, 2, 1]))) Z = W Z_INV = Z I = fisher_information_matrix(log_q_z, W, Z, Z_INV) with tf.Session() as sess: _W = np.ones((1,1,2)) _I = sess.run(I, {W:_W}) assert(approx_equal(-4.0*np.eye(2), _I, EPS)) W = tf.placeholder(tf.float64, (1,1,2)) W2 = tf.square(W) A = np.array([[1.0, 2.0], [3.0, 4.0]]) log_q_z = tf.reduce_sum(tf.matmul(np.expand_dims(A, 0), tf.transpose(W2, [0, 2, 1]))) print(log_q_z) log_q_z += tf.reduce_prod(W2) print(log_q_z.shape) Z = W Z_INV = Z I = fisher_information_matrix(log_q_z, W, Z, Z_INV) with tf.Session() as sess: _W = np.ones((1,1,2)) _I = sess.run(I, {W:_W}) ans = -np.array([[20.0, 8.0], [8.0, 28.0]]) assert(approx_equal(ans, _I, EPS)) return None
def test_approx_equal(): small_eps = 1e-16 big_eps = 1e-2 a = 0.0 b = 0.001 assert approx_equal(a, a, small_eps) assert approx_equal(b, b, small_eps) assert not approx_equal(a, b, small_eps) assert approx_equal(a, a, big_eps) assert approx_equal(b, b, big_eps) assert approx_equal(a, b, big_eps) a = np.random.normal(0.0, 1.0, (100, 100)) b = np.random.normal(0.0, 1.0, (100, 100)) assert approx_equal(a, a, small_eps) assert approx_equal(b, b, small_eps) assert not approx_equal(a, b, small_eps) return None
def test_dirichlet(): for D in Ds: for k in range(K): alpha = np.random.uniform(0.5, 5.0, (D, )) dir_dist = {"family": "dirichlet", "alpha": alpha} dist_true = scipy.stats.dirichlet(alpha) dir_sampler = get_sampler_func(dir_dist, D) dir_density = get_density_func(dir_dist, D) p_zs = np.zeros((n, )) for i in range(n): z = dir_sampler() assert np.sum(z < 0.0) == 0.0 assert np.all(np.isreal(z)) assert approx_equal(np.sum(z), 1.0, EPS) p_z = dir_density(z) p_z_true = dist_true.pdf(z) assert approx_equal(p_z, p_z_true, EPS) p_zs[i] = p_z H = -np.mean(np.log(p_zs)) H_true = dist_true.entropy() assert approx_equal(H, H_true, DIR_H_EPS) return None
def test_log_grads(): array_len = 1000 num_vars1 = 1 num_vars2 = 20 cur_ind = 113 cost_grads1 = np.zeros((array_len, num_vars1)) cost_grads2 = np.zeros((array_len, num_vars2)) cost_grads1[:cur_ind, :] = np.random.normal(2.0, 1.0, (cur_ind, num_vars1)) cost_grads2[:cur_ind, :] = np.random.normal(2.0, 1.0, (cur_ind, num_vars2)) new_grads1 = [np.array([42.0])] cost_grads1_1_true = cost_grads1.copy() cost_grads1_1_true[cur_ind, 0] = 42.0 new_grads2_1 = [np.arange(20)] cost_grads2_1_true = cost_grads2.copy() cost_grads2_1_true[cur_ind, :] = np.arange(20) new_grads2_2 = [ np.random.normal(0.0, 1.0, (2, 4)), np.random.normal(4.0, 1.0, (4, 3)), ] cost_grads2_2_true = cost_grads2_1_true.copy() cost_grads2_2_true[cur_ind + 1, :] = unroll_params(new_grads2_2) log_grads(new_grads1, cost_grads1, cur_ind) assert approx_equal(cost_grads1, cost_grads1_1_true, LG_EPS) log_grads(new_grads2_1, cost_grads2, cur_ind) assert approx_equal(cost_grads2, cost_grads2_1_true, LG_EPS) log_grads(new_grads2_2, cost_grads2, cur_ind + 1) assert approx_equal(cost_grads2, cost_grads2_2_true, LG_EPS) return None
def test_langevin_dyn(): x0 = np.random.normal(0.0, 10.0, (n, 3)) eps = 0.2 num_iters = 100 def f(x): f1 = x[:, 1] - x[:, 2] f2 = x[:, 0] + 3 f3 = x[:, 1] - 2 return tf.stack([f1, f2, f3], axis=1) def f_np(x): f1 = x[1] - x[2] f2 = x[0] + 3 f3 = x[1] - 2 return np.array([f1, f2, f3]) x0 = tf.placeholder(dtype=tf.float64, shape=(n, 3)) _x0 = np.random.normal(0.0, 1.0, (n, 3)) xs = langevin_dyn(f, x0, eps, num_iters) xs_true = np.zeros((n, 3)) for i in range(n): xs_true[i, :] = langevin_dyn_np(f_np, _x0[i, :], eps, num_iters) with tf.Session() as sess: _xs = sess.run(xs, {x0: _x0}) x_100_true = np.tile(np.array([[2.0, 5.0, 3.0]]), [n, 1]) assert approx_equal(_xs, xs_true, EPS) assert approx_equal(x_100_true, xs_true, CONV_EPS) assert approx_equal(x_100_true, _xs, CONV_EPS) return None
def test_delta(): for D in Ds: for k in range(K): a = np.random.normal(0.0, 100.0, (D, )) delta_dist = {"family": "delta", "a": a} delta_sampler = get_sampler_func(delta_dist, D) delta_density = get_density_func(delta_dist, D) p_zs = np.zeros((n, )) for i in range(n): z = delta_sampler() p_z = delta_density(z) assert approx_equal(z, a, EPS) assert p_z == 1.0 p_zs[i] = p_z H = -np.mean(np.log(p_zs)) assert H == 0.0 return None
def test_uniform_int(): for D in Ds: for k in range(K): a = np.random.randint(0, 100, (1, )) b = a + np.abs(np.random.randint(0.0, 100.0, (1, ))) + 0.001 uniform_int_dist = {"family": "uniform_int", "a": a, "b": b} uniform_int_sampler = get_sampler_func(uniform_int_dist, D) uniform_int_density = get_density_func(uniform_int_dist, D) p_zs = np.zeros((n, )) for i in range(n): z = uniform_int_sampler() p_z = uniform_int_density(z) assert a <= a assert z <= b assert p_z == 1.0 / (b - a) p_zs[i] = p_z H = -np.mean(np.log(p_zs)) assert approx_equal(H, -np.log(1.0 / (b - a)), EPS) return None
def test_min_barrier(): M = 1000 u = tf.placeholder(dtype=tf.float64, shape=(1, M)) alphas = [-1e10, 0.0, 1.0, 1e10] ts = [1e-10, 1.0, 1e10, 1e20] num_alpha = len(alphas) num_ts = len(ts) for i in range(num_alpha): alpha = alphas[i] _u = np.random.uniform(alpha, 1.0e20, (1, M)) _u[0, -1] = alpha + (1.0e-4) for j in range(num_ts): t = ts[j] I_x = min_barrier(u, alpha, t) I_x_true = _min_barrier(_u, alpha, t) with tf.Session() as sess: _I_x = sess.run(I_x, {u: _u}) assert approx_equal(_I_x, I_x_true, EPS) return None
def test_get_real_nvp_mask(): Ds = [8, 8, 8, 8, 8, 8, 8, 8, \ 2, 2, \ 3, 3, \ 17, 17] fs = [1, 1, 2, 2, 3, 3, 4, 4, \ 1, 1, \ 1, 1, \ 1, 8] firstOns = [True, False, True, False, True, False, True, False, \ True, False, \ True, False, \ True, True] true_masks = [ np.array([1, 1, 1, 1, 0, 0, 0, 0]), np.array([0, 0, 0, 0, 1, 1, 1, 1]), np.array([1, 1, 0, 0, 1, 1, 0, 0]), np.array([0, 0, 1, 1, 0, 0, 1, 1]), np.array([1, 0, 1, 0, 1, 0, 1, 0]), np.array([0, 1, 0, 1, 0, 1, 0, 1]), np.array([1, 0, 1, 0, 1, 0, 1, 0]), np.array([0, 1, 0, 1, 0, 1, 0, 1]), np.array([1, 0]), np.array([0, 1]), np.array([1, 1, 0]), np.array([0, 0, 1]), np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]), np.array([1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1]) ] num_tests = len(Ds) for i in range(num_tests): D = Ds[i] f = fs[i] firstOn = firstOns[i] mask = get_real_nvp_mask(D, f, firstOn) true_mask = true_masks[i] assert (approx_equal(mask, true_mask, EPS)) return None
def test_isotropic_inv_wishart(): for D in Ds: for k in range(K): df_fac = int(np.random.randint(1, 100, ())) iso_iw_dist = {"family": "isotropic_inv_wishart", "df_fac": df_fac} dist_true = scipy.stats.invwishart(df=df_fac * D, scale=df_fac * D * np.eye(D)) iso_iw_sampler = get_sampler_func(iso_iw_dist, D) iso_iw_density = get_density_func(iso_iw_dist, D) for i in range(n): z = iso_iw_sampler() assert np.all(np.linalg.eigvals(z) > 0.0) assert np.all(np.isreal(z)) p_z = iso_iw_density(z) p_z_true = dist_true.pdf(z) assert approx_equal(p_z, p_z_true, PERC_EPS, allow_special=True, perc=True) return None
def test_sort_quartic_roots(): roots = np.array( [ [2.0, 1.0, 4.0, 3.0], [1.0 + 1j, 1.0 - 1j, 3.0, 4.0], [1.0 + 1j, 1.0 - 1j, -1.0 - 2j, -1.0 + 2j], [1.0 + 1j, 1.0 - 2j, 1.0 - 1j, 1.0 + 2j], ] ) sorted_roots = np.array( [ [1.0, 2.0, 3.0, 4.0], [1.0 - 1j, 1.0 + 1j, 3.0, 4.0], [-1.0 - 2j, -1.0 + 2j, 1.0 - 1j, 1.0 + 1j], [1.0 - 2j, 1.0 - 1j, 1.0 + 1j, 1.0 + 2j], ] ) num_roots = roots.shape[0] for i in range(num_roots): assert approx_equal(sorted_roots[i], sort_quartic_roots(roots[i]), EPS) return None
def test_inv_wishart(): for D in Ds: df_fac = 2 * D Psi_dist = scipy.stats.invwishart(df=df_fac, scale=np.eye(D)) for k in range(K): df = int(np.random.randint(D, 100 * D, ())) Psi = Psi_dist.rvs(1) iw_dist = {"family": "inv_wishart", "df": df, "Psi": Psi} dist_true = scipy.stats.invwishart(df=df, scale=Psi) iw_sampler = get_sampler_func(iw_dist, D) iw_density = get_density_func(iw_dist, D) for i in range(n): z = iw_sampler() assert np.all(np.linalg.eigvals(z) > 0.0) assert np.all(np.isreal(z)) p_z = iw_density(z) p_z_true = dist_true.pdf(z) assert approx_equal(p_z, p_z_true, PERC_EPS, allow_special=True, perc=True) return None
def test_gumbel_softmax_log_density(): M = 100 Ks = [1, 3, 5, 10, 20] for K in Ks: tau = 0.05 _alpha = np.random.uniform(0.0, 1.0, (K,)) _alpha = _alpha / np.sum(_alpha) alpha = tf.placeholder(tf.float64, (K,)) G = sample_gumbel(M, K) _C = gumbel_softmax_trick_np(G, _alpha, tau) C = tf.placeholder(tf.float64, (M, K)) log_p_c = gumbel_softmax_log_density(K, C, alpha, tau) log_p_c_np = np.zeros((M,)) for i in range(M): log_p_c_np[i] = gumbel_softmax_log_density_np(_C[i], _alpha, tau) with tf.Session() as sess: _log_p_c = sess.run(log_p_c, {C: _C, alpha: _alpha}) assert approx_equal(log_p_c_np, _log_p_c, EPS) assert np.sum(np.isnan(_log_p_c) + np.isinf(_log_p_c)) == 0 return None
def test_get_real_nvp_mask_list(): mask_list = get_real_nvp_mask_list(5, 4) mask_list_true = [ np.array([1, 1, 1, 0, 0]), np.array([0, 0, 0, 1, 1]), np.array([1, 0, 1, 0, 1]), np.array([0, 1, 0, 1, 0]) ] for i in range(4): approx_equal(mask_list[i], mask_list_true[i], EPS) mask_list = get_real_nvp_mask_list(8, 6) mask_list_true = [ np.array([1, 1, 1, 1, 0, 0, 0, 0]), np.array([0, 0, 0, 0, 1, 1, 1, 1]), np.array([1, 0, 1, 0, 1, 0, 1, 0]), np.array([0, 1, 0, 1, 0, 1, 0, 1]), np.array([1, 1, 0, 0, 1, 1, 0, 0]), np.array([0, 0, 1, 1, 0, 0, 1, 1]) ] for i in range(6): approx_equal(mask_list[i], mask_list_true[i], EPS) mask_list = get_real_nvp_mask_list(16, 8) mask_list_true = [ np.array([1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]), np.array([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]), np.array([1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0]), np.array([0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1]), np.array([1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0]), np.array([0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1]), np.array([1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0]), np.array([0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1]) ] for i in range(8): approx_equal(mask_list[i], mask_list_true[i], EPS) return None
def test_SCCircuit(): EPS = 1e-12 DTYPE = tf.float64 np.random.seed(0) M = 100 sess = tf.Session() # true_sys = v1_circuit(T=T, dt=dt) Z = tf.placeholder(dtype=DTYPE, shape=(1, M, None)) p = 0.7 std = 0.05 var = std**2 inact_str = "NI" N = 500 param_dict = {"behavior_type": "WTA", "p": p, "var": var, "inact_str": inact_str, "N":N} system = get_system_from_template("SCCircuit", param_dict) print(system.behavior_str) assert ( system.behavior_str == "WTA_mu=7.00E-01_7.00E-01_2.50E-03_2.50E-03_0.00E+00_0.00E+00_1.00E+00_1.00E+00" ) assert system.D == 8 assert system.num_suff_stats == 8 # Test simulation _Z = np.random.normal(0.0, 3.0, (1, M, system.D)) r_t = system.simulate(Z) _r_t = sess.run(r_t, {Z: _Z}) true_sys = sc_circuit() _sW_P = _Z[0, :, 0] _sW_A = _Z[0, :, 1] _vW_PA = _Z[0, :, 2] _vW_AP = _Z[0, :, 3] _dW_PA = _Z[0, :, 4] _dW_AP = _Z[0, :, 5] _hW_P = _Z[0, :, 6] _hW_A = _Z[0, :, 7] W = np.array( [ [_sW_P, _vW_PA, _dW_PA, _hW_P], [_vW_AP, _sW_A, _hW_A, _dW_AP], [_dW_AP, _hW_A, _sW_A, _vW_AP], [_hW_P, _dW_PA, _vW_PA, _sW_P], ] ) W = np.transpose(W, [2, 0, 1]) E_constant = 0.0 E_Pbias = 0.0 E_Prule = 10.0 E_Arule = 10.0 E_choice = 2.0 E_light = 1.0 Es = [E_constant, E_Pbias, E_Prule, E_Arule, E_choice, E_light] _w = system.w N = _w.shape[4] eta_NI = np.ones((system.T,)) r_t_true = np.zeros((2, system.T, M, 4, system.N)) for i in range(M): for j in range(N): _r_t_true_ij = true_sys.simulate(W[i], Es, _w[:, 0, 0, :, j], eta_NI) r_t_true[:, :, i, :, j] = _r_t_true_ij r_t_true = np.transpose(r_t_true, [1, 0, 2, 3, 4]) assert approx_equal(_r_t, r_t_true, EPS) return None
def test_bounded_langevin_dyn(): x0 = tf.placeholder(dtype=tf.float64, shape=(n, 2)) def f(x): f1 = x[:, 1] + 2 f2 = 0.0 * x[:, 0] + 0.1 return tf.stack([f1, f2], axis=1) def f_np(x): f1 = x[:, 1] + 2 f2 = 0.0 * x[:, 0] + 0.1 return np.stack([f1, f2], axis=1) eps = 0.8 num_its = 30 non_neg = [False, True] x_ss, x = bounded_langevin_dyn(f, x0, eps, num_its, non_neg, db=True) _x0 = np.random.normal(0.0, 10.0, (n, 2)) x_ss_np, x_np = bounded_langevin_dyn_np(f_np, _x0, eps, num_its, non_neg, db=True) with tf.Session() as sess: _x = sess.run(x, {x0: _x0}) assert approx_equal(_x, x_np, EPS) x_ss_true = np.array([2.1, 0.1]) for i in range(n): assert approx_equal(_x[i, :, -1], x_ss_true, EPS) assert _x[i, 1, 1] >= 0.0 # Test top bound def f(x): f1 = x[:, 1] + 10.0 f2 = x[:, 0] + 10.0 return tf.stack([f1, f2], axis=1) def f_np(x): f1 = x[:, 1] + 10.0 f2 = x[:, 0] + 10.0 return np.stack([f1, f2], axis=1) x_ss, x = bounded_langevin_dyn(f, x0, eps, num_its, non_neg, db=True) _x0 = np.random.normal(0.0, 10.0, (n, 2)) x_ss_np, x_np = bounded_langevin_dyn_np(f_np, _x0, eps, num_its, non_neg, db=True) with tf.Session() as sess: _x = sess.run(x, {x0: _x0}) assert approx_equal(_x, x_np, EPS) assert (approx_equal(_x[:, 0, -1], 150.0 * np.ones((n, )), EPS)) assert (approx_equal(_x[:, 1, -1], 150.0 * np.ones((n, )), EPS)) # Test bottom bound def f(x): f1 = x[:, 1] - 200.0 f2 = x[:, 0] - 200.0 return tf.stack([f1, f2], axis=1) def f_np(x): f1 = x[:, 1] - 200.0 f2 = x[:, 0] - 200.0 return np.stack([f1, f2], axis=1) x_ss, x = bounded_langevin_dyn(f, x0, eps, num_its, non_neg, db=True) _x0 = np.random.normal(0.0, 10.0, (n, 2)) x_ss_np, x_np = bounded_langevin_dyn_np(f_np, _x0, eps, num_its, non_neg, db=True) with tf.Session() as sess: _x = sess.run(x, {x0: _x0}) assert approx_equal(_x, x_np, EPS) assert (approx_equal(_x[:, 0, -1], -150.0 * np.ones((n, )), EPS)) assert (approx_equal(_x[:, 1, -1], 0.0 * np.ones((n, )), EPS)) return None
def eval_nested_gaussian_integral(np_func, tf_func, np_num_pts, tf_num_pts, EPS): _mu = np.zeros((num_mus * num_delta0s * num_deltainfs, )) _delta0 = np.zeros((num_mus * num_delta0s * num_deltainfs, )) _deltainf = np.zeros((num_mus * num_delta0s * num_deltainfs, )) y_true = np.zeros((num_mus * num_delta0s * num_deltainfs)) ind = 0 invalid_inds = [] for i in range(num_mus): mu = mus[i] for j in range(num_delta0s): delta0 = delta0s[j] for k in range(num_deltainfs): deltainf = deltainfs[k] _mu[ind] = mu _delta0[ind] = delta0 _deltainf[ind] = deltainf if deltainf <= delta0: y_true[ind] = np_func(mu, delta0, deltainf, np_num_pts) if np.isnan(y_true[ind]): print("**nan**") print( "mu", _mu[ind], "delta_0", _delta0[ind], "delta_inf", _deltainf[ind], ) invalid_inds.append(ind) elif np.isinf(y_true[ind]): print("--inf--") print( "mu", _mu[ind], "delta_0", _delta0[ind], "delta_inf", _deltainf[ind], ) invalid_inds.append(ind) else: invalid_inds.append(ind) ind += 1 mu = tf.placeholder(dtype=DTYPE, shape=(num_mus * num_delta0s * num_deltainfs, )) delta0 = tf.placeholder(dtype=DTYPE, shape=(num_mus * num_delta0s * num_deltainfs, )) deltainf = tf.placeholder(dtype=DTYPE, shape=(num_mus * num_delta0s * num_deltainfs, )) y = tf_func(mu, delta0, deltainf, tf_num_pts) with tf.Session() as sess: _y = sess.run(y, {mu: _mu, delta0: _delta0, deltainf: _deltainf}) _y[invalid_inds] = 0.0 assert approx_equal(y_true, _y, EPS, allow_special=True, perc=False) return None
def test_rank2_CDD_static_solve(): n = 1000 _g = np.random.uniform(0.01, 5.0, n) # np variables _kappa1_init = np.random.uniform(-150.0, 0.0, n) _kappa2_init = np.random.uniform(-150.0, 0.0, n) _delta_0_init = np.random.uniform(0.01, 150.0, n) _g = np.random.uniform(0.01, 5.0, n) _cA = np.random.uniform(-5.0, 5.0, (n, )) _cB = np.random.uniform(-5.0, 5.0, (n, )) _rhom = np.random.uniform(-5.0, 5.0, (n, )) _rhon = np.random.uniform(-5.0, 5.0, (n, )) _betam = np.random.uniform(-5.0, 5.0, (n, )) _betan = np.random.uniform(-5.0, 5.0, (n, )) _gammaA = np.random.uniform(-5.0, 5.0, (n, )) _gammaB = np.random.uniform(-5.0, 5.0, (n, )) # tf variables kappa1_init = tf.placeholder(dtype=DTYPE, shape=(n, )) kappa2_init = tf.placeholder(dtype=DTYPE, shape=(n, )) delta_0_init = tf.placeholder(dtype=DTYPE, shape=(n, )) g = tf.placeholder(dtype=DTYPE, shape=(n, )) cA = tf.placeholder(dtype=DTYPE, shape=(n, )) cB = tf.placeholder(dtype=DTYPE, shape=(n, )) rhom = tf.placeholder(dtype=DTYPE, shape=(n, )) rhon = tf.placeholder(dtype=DTYPE, shape=(n, )) betam = tf.placeholder(dtype=DTYPE, shape=(n, )) betan = tf.placeholder(dtype=DTYPE, shape=(n, )) gammaA = tf.placeholder(dtype=DTYPE, shape=(n, )) gammaB = tf.placeholder(dtype=DTYPE, shape=(n, )) kappa1, kappa2, delta_0, z = rank2_CDD_static_solve(kappa1_init, kappa2_init, delta_0_init, cA, cB, g, rhom, rhon, betam, betan, gammaA, gammaB, its, langevin_eps, gauss_quad_pts=200) kappa1_np, kappa2_np, delta_0_np, z_np = rank2_CDD_static_solve_np( _kappa1_init, _kappa2_init, _delta_0_init, _cA, _cB, _g, _rhom, _rhon, _betam, _betan, _gammaA, _gammaB, its, langevin_eps) feed_dict = { kappa1_init: _kappa1_init, kappa2_init: _kappa2_init, delta_0_init: _delta_0_init, cA: _cA, cB: _cB, g: _g, rhom: _rhom, rhon: _rhon, betam: _betam, betan: _betan, gammaA: _gammaA, gammaB: _gammaB, } with tf.Session() as sess: _kappa1, _kappa2, _delta_0, _z = sess.run([kappa1, kappa2, delta_0, z], feed_dict) assert approx_equal(_kappa1, kappa1_np, 1e-6) assert approx_equal(_kappa2, kappa2_np, 1e-6) assert approx_equal(_delta_0, delta_0_np, 1e-6) assert approx_equal(_z, z_np, 1e-6) return None
def test_AL_cost(): D = 10 num_rand_draws = 10 W = tf.placeholder(dtype=DTYPE, shape=(1, n, D)) A1 = tf.get_variable("A1", shape=(1, D, D), dtype=DTYPE) b1 = tf.get_variable("b1", shape=(1, D, 1), dtype=DTYPE) all_params1 = tf.trainable_variables() A2 = tf.get_variable("A2", shape=(1, D, D), dtype=DTYPE) b2 = tf.get_variable("b2", shape=(1, D, 1), dtype=DTYPE) all_params2 = tf.trainable_variables() all_params = tf.trainable_variables() z1 = tf.matmul(A1, tf.transpose(W, [0, 2, 1])) + b1 z2 = tf.matmul(A2, z1) + b2 # Not the actual log_q_z, just an arbitrary function for testing. log_q_z1 = tf.log(tf.linalg.det(A1)) * np.ones((1, n)) + tf.reduce_sum(b1) log_q_z2 = ( tf.log(tf.linalg.det(A2)) * np.ones((1, n)) + tf.reduce_sum(b2) + log_q_z1 ) mu1 = np.random.normal(0.0, 1.0, (1, 1, 2 * D)) z1 = tf.transpose(z1, [0, 2, 1]) # [1, n, |T(x)|] T_x1 = tf.concat((z1, tf.square(z1)), axis=2) T_x_mu_centered1 = T_x1 - mu1 t = 1e2 I_x1 = tf.stack( (min_barrier(T_x1[:, :, 0], -1e6, t), max_barrier(T_x1[:, :, 1], 1e6, t)), axis=2, ) mu2 = np.random.normal(0.0, 1.0, (1, 1, 2 * D)) z2 = tf.transpose(z2, [0, 2, 1]) # [1, n, |T(x)|] T_x2 = tf.concat((z2, tf.square(z2)), axis=2) T_x_mu_centered2 = T_x2 - mu2 I_x2 = tf.stack( (min_barrier(T_x2[:, :, 0], -1e6, t), max_barrier(T_x2[:, :, 1], 1e6, t)), axis=2, ) Lambda = tf.placeholder(dtype=DTYPE, shape=(2 * D,)) c = tf.placeholder(dtype=DTYPE, shape=()) log_q_zs = [log_q_z1, log_q_z2] T_x_mu_centereds = [T_x_mu_centered1, T_x_mu_centered2] all_params_list = [all_params1, all_params2] I_xs = [I_x1, I_x2] for i in range(len(log_q_zs)): log_q_z = log_q_zs[i] T_x_mu_centered = T_x_mu_centereds[i] all_params = all_params_list[i] I_x = I_xs[i] for entropy in [True, False]: for _I_x in [None, I_x]: H = -tf.reduce_mean(log_q_z) costs_true, grads_true = aug_lag_cost( H, T_x_mu_centered, Lambda, c, all_params, 2 * D, entropy=entropy, I_x=_I_x, ) costs, grads, R = AL_cost( H, T_x_mu_centered, Lambda, c, all_params, entropy=entropy, I_x=_I_x ) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for j in range(num_rand_draws): _lambda = np.random.normal(0.0, 1.0, (2 * D,)) _c = np.random.normal(0.0, 1.0) _W = np.random.normal(0.0, 1.0, (1, n, D)) _grads_true, _grads = sess.run( [grads_true, grads], {W: _W, Lambda: _lambda, c: _c} ) for k in range(len(all_params)): assert approx_equal(_grads_true[k], _grads[k], EPS) return None