def test_metrics_classification(self): with self.test_session(): x = Bernoulli(probs=0.51) x_data = tf.constant(1) self.assertAllClose( 1.0, ed.evaluate('binary_accuracy', {x: x_data}, n_samples=1)) x = Bernoulli(probs=0.51, sample_shape=5) x_data = tf.constant([1, 1, 1, 0, 0]) self.assertAllClose( 0.6, ed.evaluate('binary_accuracy', {x: x_data}, n_samples=1)) x = Bernoulli(probs=tf.constant([0.51, 0.49, 0.49])) x_data = tf.constant([1, 0, 1]) self.assertAllClose( 2.0 / 3, ed.evaluate('binary_accuracy', {x: x_data}, n_samples=1)) x = Categorical(probs=tf.constant([0.48, 0.51, 0.01])) x_data = tf.constant(1) self.assertAllClose( 1.0, ed.evaluate('sparse_categorical_accuracy', {x: x_data}, n_samples=1)) x = Categorical(probs=tf.constant([0.48, 0.51, 0.01]), sample_shape=5) x_data = tf.constant([1, 1, 1, 0, 2]) self.assertAllClose( 0.6, ed.evaluate('sparse_categorical_accuracy', {x: x_data}, n_samples=1)) x = Categorical( probs=tf.constant([[0.48, 0.51, 0.01], [0.51, 0.48, 0.01]])) x_data = tf.constant([1, 2]) self.assertAllClose( 0.5, ed.evaluate('sparse_categorical_accuracy', {x: x_data}, n_samples=1)) x = Multinomial(total_count=1.0, probs=tf.constant([0.48, 0.51, 0.01])) x_data = tf.constant([0, 1, 0], dtype=x.dtype.as_numpy_dtype) self.assertAllClose( 1.0, ed.evaluate('categorical_accuracy', {x: x_data}, n_samples=1)) x = Multinomial(total_count=1.0, probs=tf.constant([0.48, 0.51, 0.01]), sample_shape=5) x_data = tf.constant( [[0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1]], dtype=x.dtype.as_numpy_dtype) self.assertAllClose( 0.6, ed.evaluate('categorical_accuracy', {x: x_data}, n_samples=1)) x = Multinomial(total_count=5.0, probs=tf.constant([0.4, 0.6, 0.0])) x_data = tf.constant([2, 3, 0], dtype=x.dtype.as_numpy_dtype) self.assertAllClose( 1.0, ed.evaluate('multinomial_accuracy', {x: x_data}, n_samples=1))
def _test(shape, n_minibatch): K = shape[-1] multinomial = Multinomial(shape, pi=tf.constant(1.0/K, shape=shape)) with sess.as_default(): pi = multinomial.pi.eval() z = np.zeros((n_minibatch, ) + tuple(shape)) for i in range(shape[0]): z[:, i, :] = np.random.multinomial(1, pi[i, :], size=n_minibatch) z_tf = tf.constant(z, dtype=tf.float32) for i in range(shape[0]): assert np.allclose( multinomial.log_prob_idx((i, ), z_tf).eval(), multinomial_logpmf_vec(z[:, i, :], 1, pi[i, :]))
def run(self, adj_mat, n_iter=1000): assert adj_mat.shape[0] == adj_mat.shape[1] n_node = adj_mat.shape[0] # model gamma = Dirichlet(concentration=tf.ones([self.n_cluster])) Pi = Beta(concentration0=tf.ones([self.n_cluster, self.n_cluster]), concentration1=tf.ones([self.n_cluster, self.n_cluster])) Z = Multinomial(total_count=1., probs=gamma, sample_shape=n_node) X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z)))) # inference (point estimation) qgamma = PointMass(params=tf.nn.softmax( tf.Variable(tf.random_normal([self.n_cluster])))) qPi = PointMass(params=tf.nn.sigmoid( tf.Variable(tf.random_normal([self.n_cluster, self.n_cluster])))) qZ = PointMass(params=tf.nn.softmax( tf.Variable(tf.random_normal([n_node, self.n_cluster])))) # map estimation inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: adj_mat}) inference.initialize(n_iter=n_iter) tf.global_variables_initializer().run() for _ in range(inference.n_iter): info_dict = inference.update() inference.print_progress(info_dict) inference.finalize() return qZ.mean().eval().argmax(axis=1)
def _test_log_prob_i(n_minibatch, num_factors, K): multinomial = Multinomial([num_factors, K], pi=tf.constant(1.0/K, shape=[num_factors, K])) with sess.as_default(): pi = multinomial.pi.eval() z = np.zeros((n_minibatch, K*num_factors)) for i in range(num_factors): z[:, (i*K):((i+1)*K)] = np.random.multinomial(1, pi[i, :], size=n_minibatch) z_tf = tf.constant(z, dtype=tf.float32) for i in range(num_factors): # NOTE: since Tensorflow has no special functions, the values here are # only an approximation assert np.allclose( multinomial.log_prob_i(i, z_tf).eval(), multinomial_logpmf_vec(z[:, (i*K):((i+1)*K)], 1, pi[i, :]), atol=1e-4)
def _test(shape, n): K = shape[-1] rv = Multinomial(shape, pi=tf.constant(1.0/K, shape=shape)) rv_sample = rv.sample(n) x = rv_sample.eval() x_tf = tf.constant(x, dtype=tf.float32) pi = rv.pi.eval() if len(shape) == 1: assert np.allclose( rv.log_prob_idx((), x_tf).eval(), multinomial_logpmf_vec(x[:, :], 1, pi[:])) elif len(shape) == 2: for i in range(shape[0]): assert np.allclose( rv.log_prob_idx((i, ), x_tf).eval(), multinomial_logpmf_vec(x[:, i, :], 1, pi[i, :])) else: assert False
def _test_log_prob_zi(n_minibatch, num_factors, K): multinomial = Multinomial(num_factors, K) multinomial.pi = tf.constant(1.0 / K, shape=[num_factors, K]) with sess.as_default(): pi = multinomial.pi.eval() z = np.zeros((n_minibatch, K * num_factors)) for i in range(num_factors): z[:, (i * K):((i + 1) * K)] = np.random.multinomial(1, pi[i, :], size=n_minibatch) z_tf = tf.constant(z, dtype=tf.float32) for i in range(num_factors): # NOTE: since Tensorflow has no special functions, the values here are # only an approximation assert np.allclose(multinomial.log_prob_zi(i, z_tf).eval(), multinomial_logpmf_vec( z[:, (i * K):((i + 1) * K)], 1, pi[i, :]), atol=1e-4)
def mmsb(N, K, data): # sparsity rho = 0.3 # MODEL # probability of belonging to each of K blocks for each node gamma = Dirichlet(concentration=tf.ones([K])) # block connectivity Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K])) # probability of belonging to each of K blocks for all nodes Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N) # adjacency X = Bernoulli(probs=(1 - rho) * tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z)))) # INFERENCE (EM algorithm) qgamma = PointMass( params=tf.nn.softmax(tf.Variable(tf.random_normal([K])))) qPi = PointMass( params=tf.nn.sigmoid(tf.Variable(tf.random_normal([K, K])))) qZ = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([N, K])))) #qgamma = Normal(loc=tf.get_variable("qgamma/loc", [K]), # scale=tf.nn.softplus( # tf.get_variable("qgamma/scale", [K]))) #qPi = Normal(loc=tf.get_variable("qPi/loc", [K, K]), # scale=tf.nn.softplus( # tf.get_variable("qPi/scale", [K, K]))) #qZ = Normal(loc=tf.get_variable("qZ/loc", [N, K]), # scale=tf.nn.softplus( # tf.get_variable("qZ/scale", [N, K]))) #inference = ed.KLqp({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: data}) inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: data}) #inference.run() n_iter = 6000 inference.initialize(optimizer=tf.train.AdamOptimizer(learning_rate=0.01), n_iter=n_iter) tf.global_variables_initializer().run() for _ in range(inference.n_iter): info_dict = inference.update() inference.print_progress(info_dict) inference.finalize() print('qgamma after: ', qgamma.mean().eval()) return qZ.mean().eval(), qPi.eval()
def model_stationary_dirichlet_multinomial(n_states, chain_len, total_counts_per_month): """ Models a stationary Dirichlet-Multinomial Markov Chain in Edward """ tf.reset_default_graph() # create default starting state probability vector pi_list = [Dirichlet(tf.ones(n_states)) for _ in range(chain_len)] # now instead of sample_shape we use total_count which # is how many times we sample from each categorical # i.e. number of accounts counts = [ Multinomial(probs=pi, total_count=float(total_counts_per_month)) for pi in pi_list ] return pi_list, counts
def main(_): ed.set_seed(42) # DATA X_data, Z_true = karate("~/data") N = X_data.shape[0] # number of vertices K = 2 # number of clusters # MODEL gamma = Dirichlet(concentration=tf.ones([K])) Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K])) Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N) X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z)))) # INFERENCE (EM algorithm) qgamma = PointMass(tf.nn.softmax(tf.get_variable("qgamma/params", [K]))) qPi = PointMass(tf.nn.sigmoid(tf.get_variable("qPi/params", [K, K]))) qZ = PointMass(tf.nn.softmax(tf.get_variable("qZ/params", [N, K]))) inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: X_data}) inference.initialize(n_iter=250) tf.global_variables_initializer().run() for _ in range(inference.n_iter): info_dict = inference.update() inference.print_progress(info_dict) # CRITICISM Z_pred = qZ.mean().eval().argmax(axis=1) print("Result (label flip can happen):") print("Predicted") print(Z_pred) print("True") print(Z_true) print("Adjusted Rand Index =", adjusted_rand_score(Z_pred, Z_true))
def bayes_mult_cmd(table_file, metadata_file, formula, output_file): #metadata = _type_cast_to_float(metadata.copy()) metadata = pd.read_table(metadata_file, index_col=0) G_data = dmatrix(formula, metadata, return_type='dataframe') table = load_table(table_file) # basic filtering parameters soil_filter = lambda val, id_, md: id_ in metadata.index read_filter = lambda val, id_, md: np.sum(val) > 10 #sparse_filter = lambda val, id_, md: np.mean(val > 0) > 0.1 sample_filter = lambda val, id_, md: np.sum(val) > 1000 table = table.filter(soil_filter, axis='sample') table = table.filter(sample_filter, axis='sample') table = table.filter(read_filter, axis='observation') #table = table.filter(sparse_filter, axis='observation') print(table.shape) y_data = pd.DataFrame(np.array(table.matrix_data.todense()).T, index=table.ids(axis='sample'), columns=table.ids(axis='observation')) y_data, G_data = y_data.align(G_data, axis=0, join='inner') psi = _gram_schmidt_basis(y_data.shape[1]) G_data = G_data.values y_data = y_data.values N, D = y_data.shape p = G_data.shape[1] # number of covariates r = G_data.shape[1] # rank of covariance matrix psi = tf.convert_to_tensor(psi, dtype=tf.float32) n = tf.convert_to_tensor(y_data.sum(axis=1), dtype=tf.float32) # hack to get multinomial working def _sample_n(self, n=1, seed=None): # define Python function which returns samples as a Numpy array def np_sample(p, n): return multinomial.rvs(p=p, n=n, random_state=seed).astype(np.float32) # wrap python function as tensorflow op val = tf.py_func(np_sample, [self.probs, n], [tf.float32])[0] # set shape from unknown shape batch_event_shape = self.batch_shape.concatenate(self.event_shape) shape = tf.concat( [tf.expand_dims(n, 0), tf.convert_to_tensor(batch_event_shape)], 0) val = tf.reshape(val, shape) return val Multinomial._sample_n = _sample_n # dummy variable for gradient G = tf.placeholder(tf.float32, [N, p]) b = Exponential(rate=1.0) B = Normal(loc=tf.zeros([p, D-1]), scale=tf.ones([p, D-1]) ) # Factorization of covariance matrix # http://edwardlib.org/tutorials/klqp l = Exponential(rate=1.0) L = Normal(loc=tf.zeros([p, D-1]), scale=tf.ones([p, D-1]) ) z = Normal(loc=tf.zeros([N, p]), scale=tf.ones([N, p])) # Cholesky trick to get multivariate normal v = tf.matmul(G, B) + tf.matmul(z, L) # get clr transformed values eta = tf.matmul(v, psi) Y = Multinomial(total_count=n, logits=eta) T = 100000 # the number of mixin samples from MCMC sampling qb = PointMass(params=tf.Variable(tf.random_normal([]))) qB = PointMass(params=tf.Variable(tf.random_normal([p, D-1]))) qz = Empirical(params=tf.Variable(tf.random_normal([T, N, p]))) ql = PointMass(params=tf.Variable(tf.random_normal([]))) qL = PointMass(params=tf.Variable(tf.random_normal([p, D-1]))) # Imputation inference_z = ed.SGLD( {z: qz}, data={G: G_data, Y: y_data, B: qB, L: qL} ) # Maximization inference_BL = ed.MAP( {B: qB, L: qL, b: qb, l: ql}, data={G: G_data, Y: y_data, z: qz} ) inference_z.initialize(step_size=1e-10) inference_BL.initialize(n_iter=1000) sess = ed.get_session() saver = tf.train.Saver() tf.global_variables_initializer().run() for i in range(inference_BL.n_iter): inference_z.update() # e-step # will need to compute the expectation of z info_dict = inference_BL.update() # m-step inference_BL.print_progress(info_dict) save_path = saver.save(sess, output_file) print("Model saved in file: %s" % save_path) pickle.dump({'qB': sess.run(qB.mean()), 'qL': sess.run(qL.mean()), 'qz': sess.run(qz.mean())}, open(output_file + '.params.pickle', 'wb') )
def _test(shape, p, n): x = Multinomial(shape, p) val_est = tuple(get_dims(x.sample(n))) val_true = (n, ) + shape assert val_est == val_true
def test_metrics_with_binary_averaging(self): x = Multinomial(total_count=10.0, probs=tf.constant([0.2, 0.7, 0.1])) x_data = tf.constant([5, 4, 1], dtype=x.dtype.as_numpy_dtype) self.assertAllEqual( np.array([9.0, 4.0, 1.0], dtype=np.float32), ed.evaluate([('mean_squared_error', { 'average': None })], {x: x_data}, n_samples=1, seed=self.RANDOM_SEED)) x = Multinomial(total_count=10.0, probs=tf.constant([0.2, 0.7, 0.1])) x_data = tf.constant([5, 4, 1], dtype=x.dtype.as_numpy_dtype) self.assertAllClose( 4.6666665, ed.evaluate([('mean_squared_error', { 'average': 'macro' })], {x: x_data}, n_samples=1, seed=self.RANDOM_SEED)) x = Multinomial(total_count=10.0, probs=tf.constant([0.2, 0.7, 0.1])) x_data = tf.constant([5, 4, 1], dtype=x.dtype.as_numpy_dtype) self.assertAllClose( 4.6666665, ed.evaluate([('mean_squared_error', { 'average': 'micro' })], {x: x_data}, n_samples=1, seed=self.RANDOM_SEED)) x = Multinomial(total_count=10.0, probs=tf.constant([0.2, 0.7, 0.1]), sample_shape=5) x_data = tf.constant( [[2, 7, 1], [3, 6, 1], [3, 5, 2], [4, 4, 2], [2, 7, 1]], dtype=x.dtype.as_numpy_dtype) self.assertAllEqual( np.array([1.2, 1.4, 0.6], dtype=np.float32), ed.evaluate([('mean_squared_error', { 'average': None })], {x: x_data}, n_samples=1, seed=self.RANDOM_SEED)) x = Multinomial(total_count=10.0, probs=tf.constant([0.2, 0.7, 0.1]), sample_shape=5) x_data = tf.constant( [[2, 7, 1], [3, 6, 1], [3, 5, 2], [4, 4, 2], [2, 7, 1]], dtype=x.dtype.as_numpy_dtype) self.assertAllClose( 1.066666603088379, ed.evaluate([('mean_squared_error', { 'average': 'macro' })], {x: x_data}, n_samples=1, seed=self.RANDOM_SEED)) x = Multinomial(total_count=10.0, probs=tf.constant([0.2, 0.7, 0.1]), sample_shape=5) x_data = tf.constant( [[2, 7, 1], [3, 6, 1], [3, 5, 2], [4, 4, 2], [2, 7, 1]], dtype=x.dtype.as_numpy_dtype) self.assertAllClose( 1.0666667222976685, ed.evaluate([('mean_squared_error', { 'average': 'micro' })], {x: x_data}, n_samples=1, seed=self.RANDOM_SEED))
return X, Z ed.set_seed(42) # DATA label_filepath = 'data/karate_labels.txt' graph_filepath = 'data/karate_edgelist.txt' X_data, Z_true = build_dataset(label_filepath, graph_filepath) N = X_data.shape[0] # number of vertices K = 2 # number of clusters # MODEL gamma = Dirichlet(concentration=tf.ones([K])) Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K])) Z = Multinomial(total_count=1., probs=gamma, sample_shape=N) X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z)))) # INFERENCE (EM algorithm) qgamma = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([K])))) qPi = PointMass(params=tf.nn.sigmoid(tf.Variable(tf.random_normal([K, K])))) qZ = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([N, K])))) inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: X_data}) n_iter = 100 inference.initialize(n_iter=n_iter) tf.global_variables_initializer().run() for _ in range(inference.n_iter):
# data N = 50 K = 10 t_true = np.random.randint(0, 2, size=[N]) t_true_2D = np.array([t_true, 1-t_true]) alpha_true = np.random.beta(a=1, b=1, size=[K, 2]) x_data = np.random.rand(K, N) < np.dot(alpha_true, t_true_2D) x_data = x_data + 0 # model pi = Dirichlet(concentration=tf.ones(2)) t = Multinomial(total_count=1., probs=pi, sample_shape=N) alpha = Beta(concentration0=tf.ones([K, 2]), concentration1=tf.ones([K, 2])) X = Bernoulli(probs=tf.matmul(alpha, tf.transpose(t))) # inference qpi = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([2])))) qt = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([N, 2])))) qalpha = PointMass(params=tf.nn.sigmoid(tf.Variable(tf.random_normal([K, 2])))) inference = ed.MAP({pi: qpi, t: qt, alpha: qalpha}, data={X: x_data}) inference.run(n_iter=5000) # criticism t_pred = qt.mean().eval().argmax(axis=1)
def _test(shape, p, size): x = Multinomial(shape, p) val_est = tuple(get_dims(x.sample(size=size))) val_true = (size, ) + shape assert val_est == val_true