def test_serialize_forget(seed):
with tempfile.NamedTemporaryFile(prefix='serialized.ripl') as f:
v1 = get_ripl(seed=seed)
v1.assume('is_tricky', '(flip 0.2)')
v1.assume('theta', '(if is_tricky (beta 1.0 1.0) 0.5)')
v1.assume('flip_coin', '(lambda () (flip theta))')
for i in range(10):
v1.observe('(flip_coin)', 'true', label='y{}'.format(i))
v1.infer("(incorporate)")
v1.save(f.name)
v2 = get_ripl()
v2.load(f.name)
for i in range(10):
v2.forget('y{}'.format(i))
v2.predict('is_tricky', label='pid')
infer = "(resimulation_mh default one %s)" % default_num_transitions_per_sample(
)
samples = collectStateSequence(v2, 'pid', infer=infer)
ans = [(False, 0.8), (True, 0.2)]
return reportKnownDiscrete(ans, samples)
def posterior_inference_action():
# Work around the fact that Puma doesn't have rejection sampling
# by asking for a bunch of MH.
if backend_name() == 'lite':
return "(rejection default all 1)"
else:
return "(resimulation_mh default one %d)" % (
default_num_transitions_per_sample(), )
def checkPGibbsBasic2(in_parallel, seed):
# Basic sanity test
ripl = get_ripl(seed=seed)
ripl.assume("x","(flip 0.1)",label="pid")
infer = "(pgibbs default one 2 %s %s)" % (default_num_transitions_per_sample(), in_parallel)
predictions = collectSamples(ripl,"pid",infer=infer)
ans = [(False,.9),(True,.1)]
return reportKnownDiscrete(ans, predictions)
def testDetachRegenInference(seed):
ripl = custom_mh_ripl(seed=seed)
ripl.assume("x", "(normal 0 1)")
ripl.observe("(normal x 1)", 2)
predictions = collectSamples(ripl,
"x",
infer="(repeat %d (custom_mh default all))" %
default_num_transitions_per_sample())
return reportKnownGaussian(1, math.sqrt(0.5), predictions)
def checkPGibbsBasic1(in_parallel, seed):
# Basic sanity test
ripl = get_ripl(seed=seed)
ripl.predict("(bernoulli)",label="pid")
infer = "(pgibbs default one 2 %s %s)" % (default_num_transitions_per_sample(), in_parallel)
predictions = collectSamples(ripl,"pid",infer=infer)
ans = [(True,.5),(False,.5)]
return reportKnownDiscrete(ans, predictions)
def testCustomProposalInference(seed):
ripl = gaussian_drift_mh_ripl(seed)
ripl.assume("x", "(normal 0 1)")
ripl.observe("(normal x 1)", 2)
predictions = collectSamples(
ripl,
"x",
infer="(repeat %d (gaussian_drift_mh default all 0.5))" %
default_num_transitions_per_sample())
return reportKnownGaussian(1, math.sqrt(0.5), predictions)
def try_at_five(maker):
r = get_ripl(seed=seed)
r.assume("mu", "(normal 0 1)")
r.assume("obs", "(%s mu 1 1 1)" % maker)
r.observe("(obs)", 5)
infer = "(mh default all %d)" % default_num_transitions_per_sample()
return collectSamples(r,
"mu",
infer=infer,
num_samples=default_num_samples(2))
def testExecuteSmoke(seed):
ripl = get_ripl(seed=seed)
predictions = []
for _ in range(default_num_samples()):
ripl.clear()
ripl.execute_program("""[assume x (normal 0 1)]
;; An observation
[observe (normal x 1) 2] ; with an end-of-line comment
[infer (mh default one %s)]""" % default_num_transitions_per_sample())
predictions.append(ripl.sample("x"))
return reportKnownGaussian(1, math.sqrt(0.5), predictions)
def checkEnumerativeGibbsBoostThrashClose(in_parallel, seed):
# Enumerating two choices with almost the same posterior probability
# should mix well.
ripl = get_ripl(seed=seed)
ripl.assume("x", "(flip 0.1)", label="pid")
ripl.observe("(flip (if x .91 .09))", "true")
infer = "(gibbs default one %s %s)" % \
(default_num_transitions_per_sample(), in_parallel)
predictions = collectSamples(ripl, "pid", infer=infer)
ans = [(False, .471), (True, .529)]
return reportKnownDiscrete(ans, predictions)
def _test_serialize_program(v, label, action):
engine = v.sivm.core_sivm.engine
if action == 'serialize':
trace1 = engine.getDistinguishedTrace()
serialized = trace1.dump()
trace2 = engine.model.restore_trace(serialized)
assert isinstance(serialized, tuple)
assert len(serialized) == 3
assert isinstance(serialized[0], list)
assert all(isinstance(x, dict) for x in serialized[0])
assert isinstance(serialized[1],
dict) # Mapping directive ids to directives
for (key, val) in serialized[1].iteritems():
assert isinstance(key, int)
assert isinstance(val, list)
assert isinstance(serialized[2], set) # Names of bound foreign sps
for elem in serialized[2]:
assert isinstance(elem, basestring)
assert isinstance(trace2, type(trace1))
assert isinstance(trace2.trace, type(trace1.trace))
elif action == 'copy':
trace1 = engine.getDistinguishedTrace()
trace2 = engine.model.copy_trace(trace1)
assert isinstance(trace2, type(trace1))
assert isinstance(trace2.trace, type(trace1.trace))
elif action == 'convert_puma':
try:
from venture.puma import trace
except ImportError:
raise SkipTest("Puma backend does not appear to be installed")
trace1 = engine.getDistinguishedTrace()
engine.to_puma()
trace2 = engine.getDistinguishedTrace()
assert 'venture.puma' in trace2.trace.__module__
elif action == 'convert_lite':
trace1 = engine.getDistinguishedTrace()
engine.to_lite()
trace2 = engine.getDistinguishedTrace()
assert 'venture.lite' in trace2.trace.__module__
else:
assert False
infer = "(resimulation_mh default one %s)" % default_num_transitions_per_sample(
)
engine.model.create_trace_pool([trace2])
r2 = collectStateSequence(v, label, infer=infer)
engine.model.create_trace_pool([trace1])
r1 = collectStateSequence(v, label, infer=infer)
return reportSameDiscrete(r1, r2)
def testCycleKernel(seed):
# Same example as testBlockingExample0, but a cycle kernel that
# covers everything should solve it
ripl = get_ripl(seed=seed)
ripl.assume("a", "(tag 0 0 (normal 10.0 1.0))", label="pid")
ripl.assume("b", "(tag 1 1 (normal a 1.0))")
ripl.observe("(normal b 1.0)", 14.0)
infer = "(repeat %s (do (mh 0 0 1) (mh 1 1 1)))" % \
default_num_transitions_per_sample()
predictions = collectSamples(ripl, "pid", infer=infer)
return reportKnownGaussian(34.0 / 3.0, math.sqrt(2.0 / 3.0), predictions)
def checkEnumerativeGibbsXOR2(in_parallel, seed):
# Tests that an XOR chain mixes with enumerative gibbs.
ripl = get_ripl(seed=seed)
ripl.assume("x", "(tag 0 0 (bernoulli 0.0015))", label="pid")
ripl.assume("y", "(tag 0 0 (bernoulli 0.0005))")
ripl.assume("noisy_true",
"(lambda (pred noise) (flip (if pred 1.0 noise)))")
ripl.observe("(noisy_true (= (+ x y) 1) .000001)", "true")
infer = "(gibbs 0 0 %s %s)" % \
(default_num_transitions_per_sample(), in_parallel)
predictions = collectSamples(ripl, "pid", infer=infer)
ans = [(True, .75), (False, .25)]
return reportKnownDiscrete(ans, predictions)
def testOccasionalRejectionBrush(seed):
# Another version, this time with explicit brush creating the mix mh
# correction.
# Note that in this case, the correction is sound: The number of
# random choices available in the default one scope really is
# changing, whereas in `testOccasionalRejection` it is not.
# To see why, consider what transition the operator (gibbs default
# one 1) induces on this model (assuming the current behavior of
# always claiming the proposal weight is 0).
# - False, False is not possible
# - From False, True:
# - With probability 50%, enumerate the second coin, and propose
# to keep it at True.
# - Else, enumerate the first coin, find that both states are
# equally good, and
# - With probability 50%, propose to leave it
# - Else, propose to change it to True, which is accepted (with
# or without the correction)
# - Ergo, move to the True state 25% of the time.
# - From True, enumerate the first coin
# - With probability 50%, the second coin comes up False in the brush;
# propose to stay in the True state.
# - Else, both states equally good
# - With probability 50%, propose to stay in the True state
# - Else, propose to move to the False, True state
# - If the correction is applied, this proposal will be
# rejected with probability 50%.
# - Ergo, move to the False, True state 25% (no correction) or
# 12.5% (correction) of the time.
# - The former will induce a 50/50 stationary distribution on the
# value of flip1, whereas the right answer is 2:1 odds in favor of
# True.
r = get_ripl(seed=seed)
r.execute_program("""
(assume flip1 (flip))
(assume flip1_or_flip2
(if flip1 true (flip)))
(observe (exactly flip1_or_flip2) true)
;; Reject with a non-negligible probability per transition, which would
;; cause a crash if Gibbs couldn't handle rejection
(gibbs default one 50 false)
""")
infer = "(gibbs default one %s false)" % default_num_transitions_per_sample(
)
predictions = collectSamples(r, address="flip1", infer=infer)
ans = [(True, 2.0 / 3), (False, 1.0 / 3)]
return reportKnownDiscrete(ans, predictions)
def testModelSwitchingSmoke(seed):
ripl = get_ripl(seed=seed, persistent_inference_trace=True)
ripl.execute_program("""
[define normal_through_model
(lambda (mu sigma)
(do (m <- (new_model))
(res <- (in_model m
(do (assume x (normal 0 ,(* (sqrt 2) sigma)))
(assume y (normal x ,(* (sqrt 2) sigma)))
(observe y (* 2 mu))
(mh default one %s)
(sample x))))
(return (first res))))]
""" % default_num_transitions_per_sample())
predictions = [ripl.infer("(normal_through_model 0 1)") for _ in range(default_num_samples())]
return reportKnownGaussian(0.0, 1.0, predictions)
def checkForEachParticleCustomMH(mode, seed):
n = max(2, default_num_samples())
ripl = get_ripl(seed=seed, persistent_inference_trace=True)
ripl.define(
"drift_mh", """\
(lambda (scope block)
(mh_correct
(on_subproblem default all
(symmetric_local_proposal
(lambda (x) (normal x 1))))))
""")
ripl.assume("x", "(normal 0 1)")
ripl.observe("(normal x 1)", 2)
ripl.infer("(resample%s %s)" % (mode, n))
for _ in range(default_num_transitions_per_sample()):
ripl.infer("(for_each_particle (drift_mh default all))")
predictions = ripl.infer("(for_each_particle (sample x))")
return reportKnownGaussian(1, 0.5**0.5, predictions)
def testModelForkingSmoke(seed):
ripl = get_ripl(seed=seed, persistent_inference_trace=True)
ripl.execute_program("""
[assume p (beta 1 1)]
[define beta_through_model
(lambda (a b)
(do (m <- (fork_model))
(res <- (in_model m
(do (repeat (- a 1) (observe (flip p) true))
(repeat (- b 1) (observe (flip p) false))
(mh default one %s)
(sample p))))
(return (first res))))]
""" % default_num_transitions_per_sample())
predictions = [ripl.infer("(beta_through_model 3 2)") for _ in range(default_num_samples())]
cdf = stats.beta(3,2).cdf
return reportKnownContinuous(cdf, predictions)
def checkEnumerativeGibbsXOR3(in_parallel, seed):
# A regression catching a mysterious math domain error.
ripl = get_ripl(seed=seed)
ripl.assume("x", "(tag 0 0 (bernoulli 0.0015))", label="pid")
ripl.assume("y", "(tag 0 0 (bernoulli 0.0005))")
ripl.assume("noisy_true",
"(lambda (pred noise) (tag 0 0 (flip (if pred 1.0 noise))))")
# This predict is the different between this test and
# testEnumerativeGibbsXOR2, and currently causes a mystery math
# domain error.
ripl.predict("(noisy_true (= (+ x y) 1) .000001)")
ripl.observe("(noisy_true (= (+ x y) 1) .000001)", "true")
infer = "(gibbs 0 0 %s %s)" % \
(default_num_transitions_per_sample(), in_parallel)
predictions = collectSamples(ripl, "pid", infer=infer)
ans = [(True, .75), (False, .25)]
return reportKnownDiscrete(ans, predictions)
def testBinomial3(seed):
# A simple test that checks the binomial enumerate method
ripl = get_ripl(seed=seed)
b = 0.7
p1 = 0.3
p2 = 0.4
n = 4
ripl.assume("p", "(tag 0 1 (if (flip %f) %f %f))" % (b, p1, p2))
ripl.predict("(tag 0 0 (binomial %d p))" % n, label="pid")
predictions = collectSamples(
ripl,
"pid",
infer="(repeat %s (do (resimulation_mh 0 1 1) (gibbs 0 0 1)))" %
default_num_transitions_per_sample())
ans = [(x, b * scipy.stats.binom.pmf(x, n, p1) +
(1 - b) * scipy.stats.binom.pmf(x, n, p2)) for x in range(n + 1)]
assert_almost_equal(sum([xx[1] for xx in ans]), 1)
return reportKnownDiscrete(ans, predictions)
def checkUCKernel(name, params, seed):
package = simulation_agreement_packages[name]
maker = package['gibbs']
def enstring(param):
if isinstance(param, list):
return "(" + " ".join(enstring(p) for p in param) + ")"
else:
return str(param)
param_str = " ".join([enstring(p) for p in params])
report = package['reporter']
r = get_ripl(seed=seed)
r.assume("made", "(tag 'latent 0 (%s %s))" % (maker, param_str))
for i in range(5):
r.predict("(tag 'data %d (made))" % i, label="datum_%d" % i)
def samples(infer):
return collectIidSamples(r, address="datum_0",
num_samples=default_num_samples(),
infer=infer)
prior = samples("pass")
geweke = """(repeat %d (do (resimulation_mh 'latent one 1)
(resimulation_mh 'data all 1)))""" % default_num_transitions_per_sample()
geweke_result = samples(geweke)
return report(prior, geweke_result)
def testOccasionalRejectionBrushScope(seed):
# Another version, this time requiring correct computation of the
# correction on a custom scope (which is carefully arranged to avoid
# creating blocks where some principal node might be in the brush).
#
# This particular arrangment of blocks is chosen to falsify the
# heuristic present at the time of writing in both Lite and Puma's
# correction computation, which is to add the number of blocks the
# scope had remaining in the pre-proposal trace with the number of
# blocks that gained root nodes in the proposal. This heuristic is
# wrong if a block that is not empty in the pre-proposal trace gains
# a new node due to the proposal, which is what happens here, when
# `flip2` proposes to move from True to False.
r = get_ripl(seed=seed)
r.execute_program("""
(assume flip1 (tag "frob" 1 (flip)))
(assume flip2 (tag "frob" 2 (flip)))
(assume flip2_or_flip3
(if flip2 true (tag "frob" 1 (flip))))
(observe (exactly (or flip1 flip2_or_flip3)) true)
""")
infer = '(gibbs "frob" one %s false)' % default_num_transitions_per_sample(
)
predictions = collectSamples(r,
address="flip2",
infer=infer,
num_samples=default_num_samples(10))
# TODO Would be nice to do the power analysis to pick the number of
# samples. Not sure exactly what distribution the expected bug
# produces, but empirically it looks like it might be 2:1
# True:False. (The incorrect computation being singled out
# overcorrects, which I think means more rejections of True->False
# moves than are justified.) A reasonable fallback might be "Pick
# the closest distribution given by comparably small integer ratios
# that is skewed in the expected direction".
ans = [(True, 4.0 / 7), (False, 3.0 / 7)]
return reportKnownDiscrete(ans, predictions)
def inferCommand(slice_method):
ntransitions = default_num_transitions_per_sample()
return "(%s default one 0.5 10000 %s)" % (slice_method, ntransitions)
def testGoldwater1():
# Fairly complicated program. Just checks to make sure it runs
# without crashing.
raise SkipTest(
"This test blocked the inference quality suite for 9 hours once. Issue: https://app.asana.com/0/11127829865276/12392223521813"
)
ripl = get_ripl()
brent = [
"catanddog", "dogandcat", "birdandcat", "dogandbird", "birdcatdog"
]
N = default_num_transitions_per_sample()
alphabet = "".join(set("".join(list(
itertools.chain.from_iterable(brent)))))
d = {}
for i in xrange(len(alphabet)):
d[alphabet[i]] = i
ripl.assume("parameter_for_dirichlet",
"(if (flip) (normal 10 1) (gamma 1 1))")
ripl.assume("alphabet_length", str(len(alphabet)))
ripl.assume(
"sample_phone",
"((if (flip) make_sym_dir_cat make_uc_sym_dir_cat) parameter_for_dirichlet alphabet_length)"
)
# TODO What is the second parameter to make_crp supposed to be?
# This line used to say "((if (flip) make_crp make_crp) (gamma 1.0 1.0) (uniform_continuous 0.001 0.01))"
ripl.assume("sample_word_id",
"((if (flip) make_crp make_crp) (gamma 1.0 1.0))")
ripl.assume("sample_letter_in_word", """
(mem (lambda (word_id pos)
(sample_phone)))
""")
#7
ripl.assume("is_end", """
(mem (lambda (word_id pos)
(flip .3)))
""")
ripl.assume(
"get_word_id", """
(mem (lambda (sentence sentence_pos)
(if (= sentence_pos 0)
(sample_word_id)
(if (is_end (get_word_id sentence (- sentence_pos 1))
(get_pos sentence (- sentence_pos 1)))
(sample_word_id)
(get_word_id sentence (- sentence_pos 1))))))
""")
ripl.assume(
"get_pos", """
(mem (lambda (sentence sentence_pos)
(if (= sentence_pos 0)
0
(if (is_end (get_word_id sentence (- sentence_pos 1))
(get_pos sentence (- sentence_pos 1)))
0
(+ (get_pos sentence (- sentence_pos 1)) 1)))))
""")
ripl.assume(
"sample_symbol", """
(mem (lambda (sentence sentence_pos)
(sample_letter_in_word (get_word_id sentence sentence_pos) (get_pos sentence sentence_pos))))
""")
ripl.assume("noise", "(gamma 1 1)")
ripl.assume("noisy_true",
"(lambda (pred noise) (flip (if pred 1.0 noise)))")
for i in range(len(brent)): #for each sentence
for j in range(len(brent[i])): #for each letter
ripl.predict("(sample_symbol %d %d)" % (i, j))
ripl.observe(
"(noisy_true (eq (sample_symbol %d %d) atom<%d>) noise)" %
(i, j, d[str(brent[i][j])]), "true")
ripl.infer(N) # TODO Make this an actual inference quality test.
