def test_serialize_forget(seed): with tempfile.NamedTemporaryFile(prefix='serialized.ripl') as f: v1 = get_ripl(seed=seed) v1.assume('is_tricky', '(flip 0.2)') v1.assume('theta', '(if is_tricky (beta 1.0 1.0) 0.5)') v1.assume('flip_coin', '(lambda () (flip theta))') for i in range(10): v1.observe('(flip_coin)', 'true', label='y{}'.format(i)) v1.infer("(incorporate)") v1.save(f.name) v2 = get_ripl() v2.load(f.name) for i in range(10): v2.forget('y{}'.format(i)) v2.predict('is_tricky', label='pid') infer = "(resimulation_mh default one %s)" % default_num_transitions_per_sample( ) samples = collectStateSequence(v2, 'pid', infer=infer) ans = [(False, 0.8), (True, 0.2)] return reportKnownDiscrete(ans, samples)
def posterior_inference_action(): # Work around the fact that Puma doesn't have rejection sampling # by asking for a bunch of MH. if backend_name() == 'lite': return "(rejection default all 1)" else: return "(resimulation_mh default one %d)" % ( default_num_transitions_per_sample(), )
def checkPGibbsBasic2(in_parallel, seed): # Basic sanity test ripl = get_ripl(seed=seed) ripl.assume("x","(flip 0.1)",label="pid") infer = "(pgibbs default one 2 %s %s)" % (default_num_transitions_per_sample(), in_parallel) predictions = collectSamples(ripl,"pid",infer=infer) ans = [(False,.9),(True,.1)] return reportKnownDiscrete(ans, predictions)
def testDetachRegenInference(seed): ripl = custom_mh_ripl(seed=seed) ripl.assume("x", "(normal 0 1)") ripl.observe("(normal x 1)", 2) predictions = collectSamples(ripl, "x", infer="(repeat %d (custom_mh default all))" % default_num_transitions_per_sample()) return reportKnownGaussian(1, math.sqrt(0.5), predictions)
def checkPGibbsBasic1(in_parallel, seed): # Basic sanity test ripl = get_ripl(seed=seed) ripl.predict("(bernoulli)",label="pid") infer = "(pgibbs default one 2 %s %s)" % (default_num_transitions_per_sample(), in_parallel) predictions = collectSamples(ripl,"pid",infer=infer) ans = [(True,.5),(False,.5)] return reportKnownDiscrete(ans, predictions)
def testCustomProposalInference(seed): ripl = gaussian_drift_mh_ripl(seed) ripl.assume("x", "(normal 0 1)") ripl.observe("(normal x 1)", 2) predictions = collectSamples( ripl, "x", infer="(repeat %d (gaussian_drift_mh default all 0.5))" % default_num_transitions_per_sample()) return reportKnownGaussian(1, math.sqrt(0.5), predictions)
def try_at_five(maker): r = get_ripl(seed=seed) r.assume("mu", "(normal 0 1)") r.assume("obs", "(%s mu 1 1 1)" % maker) r.observe("(obs)", 5) infer = "(mh default all %d)" % default_num_transitions_per_sample() return collectSamples(r, "mu", infer=infer, num_samples=default_num_samples(2))
def testExecuteSmoke(seed): ripl = get_ripl(seed=seed) predictions = [] for _ in range(default_num_samples()): ripl.clear() ripl.execute_program("""[assume x (normal 0 1)] ;; An observation [observe (normal x 1) 2] ; with an end-of-line comment [infer (mh default one %s)]""" % default_num_transitions_per_sample()) predictions.append(ripl.sample("x")) return reportKnownGaussian(1, math.sqrt(0.5), predictions)
def checkEnumerativeGibbsBoostThrashClose(in_parallel, seed): # Enumerating two choices with almost the same posterior probability # should mix well. ripl = get_ripl(seed=seed) ripl.assume("x", "(flip 0.1)", label="pid") ripl.observe("(flip (if x .91 .09))", "true") infer = "(gibbs default one %s %s)" % \ (default_num_transitions_per_sample(), in_parallel) predictions = collectSamples(ripl, "pid", infer=infer) ans = [(False, .471), (True, .529)] return reportKnownDiscrete(ans, predictions)
def _test_serialize_program(v, label, action): engine = v.sivm.core_sivm.engine if action == 'serialize': trace1 = engine.getDistinguishedTrace() serialized = trace1.dump() trace2 = engine.model.restore_trace(serialized) assert isinstance(serialized, tuple) assert len(serialized) == 3 assert isinstance(serialized[0], list) assert all(isinstance(x, dict) for x in serialized[0]) assert isinstance(serialized[1], dict) # Mapping directive ids to directives for (key, val) in serialized[1].iteritems(): assert isinstance(key, int) assert isinstance(val, list) assert isinstance(serialized[2], set) # Names of bound foreign sps for elem in serialized[2]: assert isinstance(elem, basestring) assert isinstance(trace2, type(trace1)) assert isinstance(trace2.trace, type(trace1.trace)) elif action == 'copy': trace1 = engine.getDistinguishedTrace() trace2 = engine.model.copy_trace(trace1) assert isinstance(trace2, type(trace1)) assert isinstance(trace2.trace, type(trace1.trace)) elif action == 'convert_puma': try: from venture.puma import trace except ImportError: raise SkipTest("Puma backend does not appear to be installed") trace1 = engine.getDistinguishedTrace() engine.to_puma() trace2 = engine.getDistinguishedTrace() assert 'venture.puma' in trace2.trace.__module__ elif action == 'convert_lite': trace1 = engine.getDistinguishedTrace() engine.to_lite() trace2 = engine.getDistinguishedTrace() assert 'venture.lite' in trace2.trace.__module__ else: assert False infer = "(resimulation_mh default one %s)" % default_num_transitions_per_sample( ) engine.model.create_trace_pool([trace2]) r2 = collectStateSequence(v, label, infer=infer) engine.model.create_trace_pool([trace1]) r1 = collectStateSequence(v, label, infer=infer) return reportSameDiscrete(r1, r2)
def testCycleKernel(seed): # Same example as testBlockingExample0, but a cycle kernel that # covers everything should solve it ripl = get_ripl(seed=seed) ripl.assume("a", "(tag 0 0 (normal 10.0 1.0))", label="pid") ripl.assume("b", "(tag 1 1 (normal a 1.0))") ripl.observe("(normal b 1.0)", 14.0) infer = "(repeat %s (do (mh 0 0 1) (mh 1 1 1)))" % \ default_num_transitions_per_sample() predictions = collectSamples(ripl, "pid", infer=infer) return reportKnownGaussian(34.0 / 3.0, math.sqrt(2.0 / 3.0), predictions)
def checkEnumerativeGibbsXOR2(in_parallel, seed): # Tests that an XOR chain mixes with enumerative gibbs. ripl = get_ripl(seed=seed) ripl.assume("x", "(tag 0 0 (bernoulli 0.0015))", label="pid") ripl.assume("y", "(tag 0 0 (bernoulli 0.0005))") ripl.assume("noisy_true", "(lambda (pred noise) (flip (if pred 1.0 noise)))") ripl.observe("(noisy_true (= (+ x y) 1) .000001)", "true") infer = "(gibbs 0 0 %s %s)" % \ (default_num_transitions_per_sample(), in_parallel) predictions = collectSamples(ripl, "pid", infer=infer) ans = [(True, .75), (False, .25)] return reportKnownDiscrete(ans, predictions)
def testOccasionalRejectionBrush(seed): # Another version, this time with explicit brush creating the mix mh # correction. # Note that in this case, the correction is sound: The number of # random choices available in the default one scope really is # changing, whereas in `testOccasionalRejection` it is not. # To see why, consider what transition the operator (gibbs default # one 1) induces on this model (assuming the current behavior of # always claiming the proposal weight is 0). # - False, False is not possible # - From False, True: # - With probability 50%, enumerate the second coin, and propose # to keep it at True. # - Else, enumerate the first coin, find that both states are # equally good, and # - With probability 50%, propose to leave it # - Else, propose to change it to True, which is accepted (with # or without the correction) # - Ergo, move to the True state 25% of the time. # - From True, enumerate the first coin # - With probability 50%, the second coin comes up False in the brush; # propose to stay in the True state. # - Else, both states equally good # - With probability 50%, propose to stay in the True state # - Else, propose to move to the False, True state # - If the correction is applied, this proposal will be # rejected with probability 50%. # - Ergo, move to the False, True state 25% (no correction) or # 12.5% (correction) of the time. # - The former will induce a 50/50 stationary distribution on the # value of flip1, whereas the right answer is 2:1 odds in favor of # True. r = get_ripl(seed=seed) r.execute_program(""" (assume flip1 (flip)) (assume flip1_or_flip2 (if flip1 true (flip))) (observe (exactly flip1_or_flip2) true) ;; Reject with a non-negligible probability per transition, which would ;; cause a crash if Gibbs couldn't handle rejection (gibbs default one 50 false) """) infer = "(gibbs default one %s false)" % default_num_transitions_per_sample( ) predictions = collectSamples(r, address="flip1", infer=infer) ans = [(True, 2.0 / 3), (False, 1.0 / 3)] return reportKnownDiscrete(ans, predictions)
def testModelSwitchingSmoke(seed): ripl = get_ripl(seed=seed, persistent_inference_trace=True) ripl.execute_program(""" [define normal_through_model (lambda (mu sigma) (do (m <- (new_model)) (res <- (in_model m (do (assume x (normal 0 ,(* (sqrt 2) sigma))) (assume y (normal x ,(* (sqrt 2) sigma))) (observe y (* 2 mu)) (mh default one %s) (sample x)))) (return (first res))))] """ % default_num_transitions_per_sample()) predictions = [ripl.infer("(normal_through_model 0 1)") for _ in range(default_num_samples())] return reportKnownGaussian(0.0, 1.0, predictions)
def checkForEachParticleCustomMH(mode, seed): n = max(2, default_num_samples()) ripl = get_ripl(seed=seed, persistent_inference_trace=True) ripl.define( "drift_mh", """\ (lambda (scope block) (mh_correct (on_subproblem default all (symmetric_local_proposal (lambda (x) (normal x 1)))))) """) ripl.assume("x", "(normal 0 1)") ripl.observe("(normal x 1)", 2) ripl.infer("(resample%s %s)" % (mode, n)) for _ in range(default_num_transitions_per_sample()): ripl.infer("(for_each_particle (drift_mh default all))") predictions = ripl.infer("(for_each_particle (sample x))") return reportKnownGaussian(1, 0.5**0.5, predictions)
def testModelForkingSmoke(seed): ripl = get_ripl(seed=seed, persistent_inference_trace=True) ripl.execute_program(""" [assume p (beta 1 1)] [define beta_through_model (lambda (a b) (do (m <- (fork_model)) (res <- (in_model m (do (repeat (- a 1) (observe (flip p) true)) (repeat (- b 1) (observe (flip p) false)) (mh default one %s) (sample p)))) (return (first res))))] """ % default_num_transitions_per_sample()) predictions = [ripl.infer("(beta_through_model 3 2)") for _ in range(default_num_samples())] cdf = stats.beta(3,2).cdf return reportKnownContinuous(cdf, predictions)
def checkEnumerativeGibbsXOR3(in_parallel, seed): # A regression catching a mysterious math domain error. ripl = get_ripl(seed=seed) ripl.assume("x", "(tag 0 0 (bernoulli 0.0015))", label="pid") ripl.assume("y", "(tag 0 0 (bernoulli 0.0005))") ripl.assume("noisy_true", "(lambda (pred noise) (tag 0 0 (flip (if pred 1.0 noise))))") # This predict is the different between this test and # testEnumerativeGibbsXOR2, and currently causes a mystery math # domain error. ripl.predict("(noisy_true (= (+ x y) 1) .000001)") ripl.observe("(noisy_true (= (+ x y) 1) .000001)", "true") infer = "(gibbs 0 0 %s %s)" % \ (default_num_transitions_per_sample(), in_parallel) predictions = collectSamples(ripl, "pid", infer=infer) ans = [(True, .75), (False, .25)] return reportKnownDiscrete(ans, predictions)
def testBinomial3(seed): # A simple test that checks the binomial enumerate method ripl = get_ripl(seed=seed) b = 0.7 p1 = 0.3 p2 = 0.4 n = 4 ripl.assume("p", "(tag 0 1 (if (flip %f) %f %f))" % (b, p1, p2)) ripl.predict("(tag 0 0 (binomial %d p))" % n, label="pid") predictions = collectSamples( ripl, "pid", infer="(repeat %s (do (resimulation_mh 0 1 1) (gibbs 0 0 1)))" % default_num_transitions_per_sample()) ans = [(x, b * scipy.stats.binom.pmf(x, n, p1) + (1 - b) * scipy.stats.binom.pmf(x, n, p2)) for x in range(n + 1)] assert_almost_equal(sum([xx[1] for xx in ans]), 1) return reportKnownDiscrete(ans, predictions)
def checkUCKernel(name, params, seed): package = simulation_agreement_packages[name] maker = package['gibbs'] def enstring(param): if isinstance(param, list): return "(" + " ".join(enstring(p) for p in param) + ")" else: return str(param) param_str = " ".join([enstring(p) for p in params]) report = package['reporter'] r = get_ripl(seed=seed) r.assume("made", "(tag 'latent 0 (%s %s))" % (maker, param_str)) for i in range(5): r.predict("(tag 'data %d (made))" % i, label="datum_%d" % i) def samples(infer): return collectIidSamples(r, address="datum_0", num_samples=default_num_samples(), infer=infer) prior = samples("pass") geweke = """(repeat %d (do (resimulation_mh 'latent one 1) (resimulation_mh 'data all 1)))""" % default_num_transitions_per_sample() geweke_result = samples(geweke) return report(prior, geweke_result)
def testOccasionalRejectionBrushScope(seed): # Another version, this time requiring correct computation of the # correction on a custom scope (which is carefully arranged to avoid # creating blocks where some principal node might be in the brush). # # This particular arrangment of blocks is chosen to falsify the # heuristic present at the time of writing in both Lite and Puma's # correction computation, which is to add the number of blocks the # scope had remaining in the pre-proposal trace with the number of # blocks that gained root nodes in the proposal. This heuristic is # wrong if a block that is not empty in the pre-proposal trace gains # a new node due to the proposal, which is what happens here, when # `flip2` proposes to move from True to False. r = get_ripl(seed=seed) r.execute_program(""" (assume flip1 (tag "frob" 1 (flip))) (assume flip2 (tag "frob" 2 (flip))) (assume flip2_or_flip3 (if flip2 true (tag "frob" 1 (flip)))) (observe (exactly (or flip1 flip2_or_flip3)) true) """) infer = '(gibbs "frob" one %s false)' % default_num_transitions_per_sample( ) predictions = collectSamples(r, address="flip2", infer=infer, num_samples=default_num_samples(10)) # TODO Would be nice to do the power analysis to pick the number of # samples. Not sure exactly what distribution the expected bug # produces, but empirically it looks like it might be 2:1 # True:False. (The incorrect computation being singled out # overcorrects, which I think means more rejections of True->False # moves than are justified.) A reasonable fallback might be "Pick # the closest distribution given by comparably small integer ratios # that is skewed in the expected direction". ans = [(True, 4.0 / 7), (False, 3.0 / 7)] return reportKnownDiscrete(ans, predictions)
def inferCommand(slice_method): ntransitions = default_num_transitions_per_sample() return "(%s default one 0.5 10000 %s)" % (slice_method, ntransitions)
def testGoldwater1(): # Fairly complicated program. Just checks to make sure it runs # without crashing. raise SkipTest( "This test blocked the inference quality suite for 9 hours once. Issue: https://app.asana.com/0/11127829865276/12392223521813" ) ripl = get_ripl() brent = [ "catanddog", "dogandcat", "birdandcat", "dogandbird", "birdcatdog" ] N = default_num_transitions_per_sample() alphabet = "".join(set("".join(list( itertools.chain.from_iterable(brent))))) d = {} for i in xrange(len(alphabet)): d[alphabet[i]] = i ripl.assume("parameter_for_dirichlet", "(if (flip) (normal 10 1) (gamma 1 1))") ripl.assume("alphabet_length", str(len(alphabet))) ripl.assume( "sample_phone", "((if (flip) make_sym_dir_cat make_uc_sym_dir_cat) parameter_for_dirichlet alphabet_length)" ) # TODO What is the second parameter to make_crp supposed to be? # This line used to say "((if (flip) make_crp make_crp) (gamma 1.0 1.0) (uniform_continuous 0.001 0.01))" ripl.assume("sample_word_id", "((if (flip) make_crp make_crp) (gamma 1.0 1.0))") ripl.assume("sample_letter_in_word", """ (mem (lambda (word_id pos) (sample_phone))) """) #7 ripl.assume("is_end", """ (mem (lambda (word_id pos) (flip .3))) """) ripl.assume( "get_word_id", """ (mem (lambda (sentence sentence_pos) (if (= sentence_pos 0) (sample_word_id) (if (is_end (get_word_id sentence (- sentence_pos 1)) (get_pos sentence (- sentence_pos 1))) (sample_word_id) (get_word_id sentence (- sentence_pos 1)))))) """) ripl.assume( "get_pos", """ (mem (lambda (sentence sentence_pos) (if (= sentence_pos 0) 0 (if (is_end (get_word_id sentence (- sentence_pos 1)) (get_pos sentence (- sentence_pos 1))) 0 (+ (get_pos sentence (- sentence_pos 1)) 1))))) """) ripl.assume( "sample_symbol", """ (mem (lambda (sentence sentence_pos) (sample_letter_in_word (get_word_id sentence sentence_pos) (get_pos sentence sentence_pos)))) """) ripl.assume("noise", "(gamma 1 1)") ripl.assume("noisy_true", "(lambda (pred noise) (flip (if pred 1.0 noise)))") for i in range(len(brent)): #for each sentence for j in range(len(brent[i])): #for each letter ripl.predict("(sample_symbol %d %d)" % (i, j)) ripl.observe( "(noisy_true (eq (sample_symbol %d %d) atom<%d>) noise)" % (i, j, d[str(brent[i][j])]), "true") ripl.infer(N) # TODO Make this an actual inference quality test.