def test_lexicographic_combinations(): """Test ordered combinations.""" mappings = (([], [[]]), ([[], [1]], []), ([[1], []], []), ([[1], [2]], [[1, 2]]), ([[1], [2, 3]], [[1, 2], [1, 3]]), ([[1, 3], [2]], [[1, 2], [3, 2]])) for (input, output) in mappings: assert list(utils.lexicographic_combinations(input)) == output
class TestSprinklerBayesNet(object): """Test training on a three-node rain/sprinkler/grass network.""" def setup(self): self.rng = utils.RandomState(seed=0) self.net = sprinkler_net.get(self.rng) @pytest.mark.slow @pytest.mark.parametrize( "evidence_index,precompute_gibbs", utils.lexicographic_combinations([[0, 1, 2, 3], [True, False]])) def test(self, evidence_index, precompute_gibbs): evidence = sprinkler_net.evidence(evidence_index) evidence_nodes = [self.net.nodes_by_index[index] for index in evidence.keys()] inverse_map = invert.compute_inverse_map( self.net, evidence_nodes, self.rng) assert len(inverse_map) == 2 trainer = train.Trainer(self.net, inverse_map, precompute_gibbs) num_samples = 30000 for _ in xrange(num_samples): sample = self.net.sample() trainer.observe(sample) trainer.finalize() # Compare marginal log probability for evidence node with prior marginals. empty_world = random_world.RandomWorld() enumerator = exact_inference.Enumerator(self.net, empty_world) exact_marginals = enumerator.marginals() for evidence_node in evidence_nodes: for value in [0, 1]: log_prob_true = math.log(exact_marginals[evidence_node.index][value]) for inverse_net in inverse_map.values(): log_prob_empirical = inverse_net.nodes_by_index[ evidence_node.index].log_probability(empty_world, value) print abs(log_prob_true - log_prob_empirical) assert abs(log_prob_true - log_prob_empirical) < .02 # For each inverse network, take unconditional samples, compare # marginals to prior network. num_samples = 30000 for inverse_net in inverse_map.values(): counts = [[0, 0], [0, 0], [0, 0]] for _ in xrange(num_samples): world = inverse_net.sample() for (index, value) in world.items(): counts[index][value] += 1 for index in [0, 1, 2]: true_dist = enumerator.marginalize_node( self.net.nodes_by_index[index]) empirical_dist = utils.normalize(counts[index]) for (p_true, p_empirical) in zip(true_dist, empirical_dist): print abs(p_true - p_empirical) assert abs(p_true - p_empirical) < .02
def test(self, training_source): """Compare dist on final nodes with and without Gibbs precomputation.""" evidence_nodes = [self.net.nodes_by_index[index] for index in self.evidence.keys()] # Set up two trainers, one with Gibbs precomputation, one without. print "Computing inverse maps..." inverse_map_with_gibbs = invert.compute_inverse_map( self.net, evidence_nodes, self.rng, max_inverse_size=1) inverse_map_without_gibbs = invert.compute_inverse_map( self.net, evidence_nodes, self.rng, max_inverse_size=1) trainer_with_gibbs = train.Trainer( self.net, inverse_map_with_gibbs, precompute_gibbs=True) trainer_without_gibbs = train.Trainer( self.net, inverse_map_without_gibbs, precompute_gibbs=False) trainers = [trainer_with_gibbs, trainer_without_gibbs] # Train based on sampled data. print "Training..." num_samples = 50000 if training_source == "prior": self.train_from_prior(trainers, num_samples) elif training_source == "gibbs-prior": self.train_from_gibbs_prior(trainers, num_samples) elif training_source == "gibbs-posterior": self.train_from_gibbs_posterior(trainers, num_samples) else: raise ValueError("Unknown training source.") # Go through all nodes, check that all estimated conditional # distributions are close to true distributions. print "Comparing distributions..." error = 0.0 num_checks = 0 for node in inverse_map_with_gibbs.keys(): net_with_gibbs = inverse_map_with_gibbs.get_net(node) net_without_gibbs = inverse_map_without_gibbs.get_net(node) gibbs_dist = net_with_gibbs.nodes_by_index[node.index].distribution estimated_dist = net_without_gibbs.nodes_by_index[node.index].distribution for markov_blanket_values in utils.lexicographic_combinations( [[0, 1]] * len(node.markov_blanket)): for value in node.support: true_value = math.exp( gibbs_dist.log_probability(markov_blanket_values, value)) estimated_value = math.exp( estimated_dist.log_probability(markov_blanket_values, value)) error += abs(true_value - estimated_value) num_checks += 1 average_error = error / num_checks print average_error assert average_error < .05
def test_gibbs_learner(): """Verify that Gibbs learner makes same predictions as enumerator.""" num_samples = 10000 rng = utils.RandomState(seed=0) net = sprinkler_net.get(rng) for node in net.nodes_by_index: learner = learn.GibbsLearner(node, rng) learner.finalize() for markov_blanket_values in utils.lexicographic_combinations( [[0, 1]] * len(node.markov_blanket)): world = random_world.RandomWorld( [n.index for n in node.markov_blanket], markov_blanket_values) enumerator = exact_inference.Enumerator(net, world) probabilities = enumerator.marginalize_node(node) check_scorer( lambda v: learner.log_probability( markov_blanket_values, v), probabilities) check_sampler( lambda: learner.sample( markov_blanket_values), probabilities, num_samples)
class TestInverseChain(object): def setup(self): self.rng = utils.RandomState(seed=0) self.net = sprinkler_net.get(self.rng) @pytest.mark.slow @pytest.mark.parametrize( ("proposal_size", "evidence_index"), utils.lexicographic_combinations([[1, 2, 3], [0, 1, 2, 3]])) def test_inverse_sampler(self, proposal_size, evidence_index): evidence = sprinkler_net.evidence(evidence_index) evidence_nodes = [self.net.nodes_by_index[index] for index in evidence.keys()] inverse_map = invert.compute_inverse_map( self.net, evidence_nodes, self.rng) # 2. Initialize inverses with uniform distributions trainer = train.Trainer(self.net, inverse_map, precompute_gibbs=False) # 3. Generate random data for _ in xrange(3): world = random_world.RandomWorld( range(self.net.node_count), [self.rng.choice(node.support) for node in self.net.nodes_by_index] ) trainer.observe(world) trainer.finalize() # 4. Compute true answer enumerator = exact_inference.Enumerator(self.net, evidence) true_marginals = enumerator.marginals() # 5. Compute answer using inverse sampling num_samples = 50000 test_sampler = mcmc.InverseChain( self.net, inverse_map, self.rng, evidence, proposal_size) test_sampler.initialize_state() inverse_marginals = test_sampler.marginals(num_samples) assert true_marginals - inverse_marginals < .02
class TestTriangleNetwork(object): def setup(self, determinism=95, seed=None): seed = 0 if seed is None else seed self.rng = utils.RandomState(seed) self.net = triangle_net.get(self.rng, determinism) self.evidence = triangle_net.evidence(0, determinism) self.evidence_nodes = [ self.net.nodes_by_index[index] for index in self.evidence.keys() ] self.num_latent_nodes = len(self.net.nodes()) - len( self.evidence_nodes) self.marginals = triangle_net.marginals(0, determinism) def test_gibbs(self): num_test_samples = 5000 gibbs_chain = mcmc.GibbsChain(self.net, self.rng, self.evidence) gibbs_chain.initialize_state() gibbs_marginals = gibbs_chain.marginals(num_test_samples) gibbs_error = (self.marginals - gibbs_marginals).mean() print "Error (Gibbs): {}".format(gibbs_error) assert gibbs_error < .05 def train_inverses(self, inverse_map, num_training_samples, precompute_gibbs): trainer = train.Trainer(self.net, inverse_map, precompute_gibbs) training_sampler = mcmc.GibbsChain(self.net, self.rng, self.evidence) training_sampler.initialize_state() counter = marg.MarginalCounter(self.net) for _ in xrange(num_training_samples): training_sampler.transition() trainer.observe(training_sampler.state) counter.observe(training_sampler.state) trainer.finalize() training_error = (self.marginals - counter.marginals()).mean() return training_error def check_inverses_by_samples(self, inverse_map, max_inverse_size, num_test_samples): test_sampler = mcmc.InverseChain(self.net, inverse_map, self.rng, self.evidence, max_inverse_size) test_sampler.initialize_state() counter = marg.MarginalCounter(self.net) num_proposals_accepted = 0 for _ in xrange(num_test_samples): accept = test_sampler.transition() counter.observe(test_sampler.state) num_proposals_accepted += accept inverse_marginals = counter.marginals() inverses_error = (self.marginals - inverse_marginals).mean() return inverses_error, num_proposals_accepted def check_inverses_by_time(self, inverse_map, max_inverse_size, test_seconds): test_sampler = mcmc.InverseChain(self.net, inverse_map, self.rng, self.evidence, max_inverse_size) test_sampler.initialize_state() counter = marg.MarginalCounter(self.net) num_proposals_accepted = 0 start_time = datetime.datetime.now() num_proposals = 0 while (datetime.datetime.now() - start_time).seconds < test_seconds: accept = test_sampler.transition() counter.observe(test_sampler.state) num_proposals_accepted += accept num_proposals += self.num_latent_nodes inverse_marginals = counter.marginals() inverses_error = (self.marginals - inverse_marginals).mean() return inverses_error, num_proposals, num_proposals_accepted @pytest.mark.slow @pytest.mark.parametrize("precompute_gibbs,max_inverse_size", utils.lexicographic_combinations([[True, False], [1, 2]])) def test_inverses_error(self, precompute_gibbs, max_inverse_size): """Verify that error in estimated inverse marginals is low.""" num_training_samples = 50000 num_test_samples = 10000 print "Computing inverse nets..." inverse_map = invert.compute_inverse_map(self.net, self.evidence_nodes, self.rng, max_inverse_size) print "Training on Gibbs samples..." training_error = self.train_inverses(inverse_map, num_training_samples, precompute_gibbs) print "Error (training): {}".format(training_error) assert training_error < .01 print "Testing (inverses)..." test_error, num_proposals_accepted = self.check_inverses_by_samples( inverse_map, max_inverse_size, num_test_samples) print "Error (inverses): {}".format(test_error) assert test_error < .03 num_proposals = num_test_samples * self.num_latent_nodes print "Accepted {} out of {} proposals".format(num_proposals_accepted, num_proposals) if max_inverse_size == 1 and precompute_gibbs: # Check that all proposals are accepted. assert num_proposals_accepted == num_proposals @pytest.mark.slow def test_inverses_performance(self): """Verify that bigger proposal sizes can result in better performance. This test uses clock time, not number of samples, to decide how many test samples to take. """ num_training_samples = 50000 test_seconds = 10 precompute_gibbs = True max_inverse_size = 8 print "Computing inverse nets..." inverse_map = invert.compute_inverse_map(self.net, self.evidence_nodes, self.rng, max_inverse_size) print "Training on Gibbs samples..." training_error = self.train_inverses(inverse_map, num_training_samples, precompute_gibbs) print "Error (training): {}".format(training_error) assert training_error < .01 print "Testing (inverses)..." for inverse_size in range(1, max_inverse_size + 1): test_error, num_proposals, num_accepted = self.check_inverses_by_time( inverse_map, inverse_size, test_seconds) print "Error (inverses, max inverse size {}): {}".format( inverse_size, test_error) print "Accepted {} out of {} proposals\n".format( num_accepted, num_proposals) def profile_inverses(self): num_training_samples = 5000 test_seconds = 10 precompute_gibbs = False max_inverse_size = 3 start_time = datetime.datetime.now() inverse_map = invert.compute_inverse_map(self.net, self.evidence_nodes, self.rng, max_inverse_size) t1 = datetime.datetime.now() print "Time to compute inverse map: {}".format(t1 - start_time) self.train_inverses(inverse_map, num_training_samples, precompute_gibbs) t2 = datetime.datetime.now() print "Time to train inverses: {}".format(t2 - t1) self.check_inverses_by_time(inverse_map, max_inverse_size, test_seconds) t3 = datetime.datetime.now() print "Time for test sampling: {}".format(t3 - t2)
def reorder_cpt(old_order, old_domain_sizes, old_probs, new_order): """Return reordered CPT based on old and new node indices. Args: old_order: list of indices old_domain_sizes: mapping from indices to integers old_probs: list of probabilities new_order: list of indices Example: old_order = [a, b, c] old_domain_sizes = [2, 2, 2] old_probs = [.1, .2, .3, .4, .5, .6, .7, .8] new_order = [c, a, b] Old (unnormalized) CPT: a=0, b=0, c=0 .1 a=0, b=0, c=1 .2 a=0, b=1, c=0 .3 a=0, b=1, c=1 .4 a=1, b=0, c=0 .5 a=1, b=0, c=1 .6 a=1, b=1, c=0 .7 a=1, b=1, c=1 .8 New (unnormalized) CPT: c=0, a=0, b=0: .1 c=0, a=0, b=1: .3 c=0, a=1, b=0: .5 c=0, a=1, b=1: .7 c=1, a=0, b=0: .2 c=1, a=0, b=1: .4 c=1, a=1, b=0: .6 c=1, a=1, b=1: .8 """ assert len(old_order) == len(old_domain_sizes) assert set(old_order) == set(new_order) assert len(old_probs) == reduce(operator.mul, old_domain_sizes, 1) # Create a mapping from old-order value tuples to probabilities. old_domains = [range(domain_size) for domain_size in old_domain_sizes] old_value_lists = utils.lexicographic_combinations(old_domains) old_value_tuples = [tuple(values) for values in old_value_lists] assert len(old_value_tuples) == len(old_probs) old_value_tuple_to_prob = dict(zip(old_value_tuples, old_probs)) # Create mapping from old to new clique member indices. old_to_new_index = dict(zip(old_order, new_order)) # Create mapping from old clique index to domain old_index_to_domain = dict(zip(old_order, old_domains)) # Iterate through new-order value tuples, create reordered list of # probabilities. new_domains = [old_index_to_domain[old_to_new_index[old_index]] for old_index in old_order] new_probs = [] for new_value_list in utils.lexicographic_combinations(new_domains): new_value_tuple = tuple(new_value_list) old_value_tuple = tuple( utils.reordered_list(new_order, old_order, new_value_tuple)) prob = old_value_tuple_to_prob[old_value_tuple] new_probs.append(prob) return new_probs
inverse_marginals = enumerator.marginals() print true_marginals - inverse_marginals assert true_marginals - inverse_marginals < .01 print "Testing (sampling)..." test_sampler = mcmc.InverseChain(net, inverse_map, rng, evidence, proposal_size=proposal_size) test_sampler.initialize_state() inverse_marginals = test_sampler.marginals(num_samples) print true_marginals - inverse_marginals assert true_marginals - inverse_marginals < .02 @pytest.mark.slow @pytest.mark.parametrize(("proposal_size", "evidence_index"), utils.lexicographic_combinations([[1, 2, 3], [0, 1, 2, 3]])) def test_sprinkler_net(proposal_size, evidence_index): rng = utils.RandomState(seed=0) net = sprinkler_net.get(rng) evidence = sprinkler_net.evidence(evidence_index) run_test(rng, net, evidence, proposal_size=proposal_size) if __name__ == "__main__": test_sprinkler_net(proposal_size=2, evidence_index=2)