def visit(self, e):
if hasattr(e, "_nosimpl"): return e
if isinstance(e, Exp) and not isinstance(e, ELambda): t = e.type
new = super().visit(e)
if isinstance(e, Exp) and not isinstance(e, ELambda): assert new.type == e.type or (is_collection(new.type) and is_collection(e.type)), repr(e)
if self.debug and isinstance(e, Exp) and not isinstance(e, ELambda):
model = satisfy(ENot(EBinOp(e, "===", new).with_type(BOOL)))
if model is not None:
raise Exception("bad simplification: {} ---> {} (under model {!r}, got {!r} and {!r})".format(pprint(e), pprint(new), model, eval(e, model), eval(new, model)))
return new
def visit(self, e):
if hasattr(e, "_nosimpl"): return e
if isinstance(e, Exp) and not isinstance(e, ELambda): t = e.type
new = super().visit(e)
if isinstance(e, Exp) and not isinstance(e, ELambda): assert new.type == e.type or (is_collection(new.type) and is_collection(e.type)), repr(e)
if self.debug and isinstance(e, Exp) and not isinstance(e, ELambda):
model = satisfy(ENot(EBinOp(e, "===", new).with_type(BOOL)))
if model is not None:
raise Exception("bad simplification: {} ---> {} (under model {!r}, got {!r} and {!r})".format(pprint(e), pprint(new), model, eval(e, model), eval(new, model)))
return new
def assert_same(self, e1, e2):
assert e1.type == e2.type, "{} | {}".format(pprint(e1.type), pprint(e2.type))
def dbg(model):
print("model: {!r}".format(model))
r1 = eval(e1, model)
r2 = eval(e2, model)
print("e1: {}".format(pprint(e1)))
print(" ---> {!r}".format(r1))
print("e2: {}".format(pprint(e2)))
print(" ---> {!r}".format(r2))
assert satisfy(ENot(EBinOp(e1, "===", e2).with_type(BOOL)), model_callback=dbg) is None
def test_bag_plus_minus(self):
t = THandle("H", INT)
x = EVar("x").with_type(t)
xs = EVar("xs").with_type(TBag(t))
spec = EBinOp(EBinOp(xs, "+", ESingleton(x)), "-", ESingleton(x))
expected = xs
assert retypecheck(spec)
assert valid(EEq(spec, expected))
ex = satisfy(ENot(EBinOp(spec, "===", expected).with_type(BOOL)))
assert ex is not None
assert check_discovery(spec=spec, expected=expected, args=[x, xs], examples=[ex])
def compute_sharing(state_map: dict, true_types: dict) -> dict:
"""
Takes a dictionary mapping { state_var_id : state_exp } and a
dictionary mapping { state_var_id : refined_type } and returns
a dictionary { ht : groups } for each handle type ht. Each group
is a list of implementation types whose intrusive data will
never be used at the same time.
"""
types = set(t for e in state_map.values()
for t in syntax_tools.all_types(e.type))
handle_types = set(t for t in types
if isinstance(t, target_syntax.THandle))
out = {}
# for (var, exp) in state_map.items():
# print(" --> {} = {}".format(var, syntax_tools.pprint(exp)))
for ht in handle_types:
groups = []
handle = syntax_tools.fresh_var(ht, "handle")
# print(ht)
# for (var, exp) in state_map.items():
# print(" --> {} iff {}".format(var, syntax_tools.pprint(uses_intrusive_data(exp, handle))))
type_uses_intrusive_data = {}
for (var, exp) in state_map.items():
use = uses_intrusive_data(exp, handle)
for t in syntax_tools.all_types(true_types[var]):
# print(syntax_tools.pprint(t))
if hasattr(t, "intrusive_data"):
type_uses_intrusive_data[t] = use
# else:
# print(" no intrusive data for " + syntax_tools.pprint(t))
# print(type_uses_intrusive_data)
for t, cond in type_uses_intrusive_data.items():
found = False
for g in groups:
if all(not solver.satisfy(
target_syntax.EAll([cond, type_uses_intrusive_data[t]
])) for t in g):
found = True
g.append(t)
break
if not found:
groups.append([t])
# print(" --> {}".format(groups))
out[ht] = groups
return out
def assert_same(self, e1, e2, assumptions : Exp = ETRUE, op = "==="):
assert e1.type == e2.type, "{} | {}".format(pprint(e1.type), pprint(e2.type))
def dbg(model):
print("model: {!r}".format(model))
r1 = eval(e1, model)
r2 = eval(e2, model)
print("e1: {}".format(pprint(e1)))
print(" ---> {!r}".format(r1))
print("e2: {}".format(pprint(e2)))
print(" ---> {!r}".format(r2))
assert satisfy(EAll([
assumptions,
ENot(EBinOp(e1, op, e2).with_type(BOOL))]), model_callback=dbg, validate_model=True) is None
def test_bag_plus_minus(self):
t = THandle("H", INT)
x = EVar("x").with_type(t)
xs = EVar("xs").with_type(TBag(t))
spec = EBinOp(EBinOp(xs, "+", ESingleton(x)), "-", ESingleton(x))
expected = xs
assert retypecheck(spec)
assert valid(EEq(spec, expected))
ex = satisfy(ENot(EBinOp(spec, "===", expected).with_type(BOOL)))
assert ex is not None
assert check_discovery(spec=spec,
expected=expected,
args=[x, xs],
examples=[ex])
def assert_same(self, e1, e2, assumptions: Exp = ETRUE, op="==="):
assert e1.type == e2.type, "{} | {}".format(pprint(e1.type),
pprint(e2.type))
def dbg(model):
print("model: {!r}".format(model))
r1 = eval(e1, model)
r2 = eval(e2, model)
print("e1: {}".format(pprint(e1)))
print(" ---> {!r}".format(r1))
print("e2: {}".format(pprint(e2)))
print(" ---> {!r}".format(r2))
assert satisfy(EAll([
assumptions, ENot(EBinOp(e1, op, e2).with_type(BOOL))
]),
model_callback=dbg,
validate_model=True) is None
def improve(target: Exp,
context: Context,
assumptions: Exp = T,
stop_callback=never_stop,
hints: [Exp] = (),
examples: [{
str: object
}] = (),
cost_model: CostModel = None):
"""
Improve the target expression using enumerative synthesis.
This function is a generator that yields increasingly better and better
versions of the input expression `target`.
Notes on internals of this algorithm follow.
Key differences from "regular" enumerative synthesis:
- Expressions are either "state" expressions or "runtime" expressions,
allowing this algorithm to choose what things to store on the data
structure and what things to compute at query execution time. (The
cost model is ultimately responsible for this choice.)
- If a better version of *any subexpression* for the target is found,
it is immediately substituted in and the overall expression is
returned. This "smooths out" the search space a little, and lets us
find kinda-good solutions very quickly, even if the best possible
solution is out of reach.
"""
print("call to improve:")
print("""improve(
target={target!r},
context={context!r},
assumptions={assumptions!r},
stop_callback={stop_callback!r},
hints={hints!r},
examples={examples!r},
cost_model={cost_model!r})""".format(target=target,
context=context,
assumptions=assumptions,
stop_callback=stop_callback,
hints=hints,
examples=examples,
cost_model=cost_model))
target = freshen_binders(target, context)
assumptions = freshen_binders(assumptions, context)
print()
print("improving: {}".format(pprint(target)))
print("subject to: {}".format(pprint(assumptions)))
print()
try:
assert exp_wf(target, context=context, assumptions=assumptions)
except ExpIsNotWf as ex:
print(
"WARNING: initial target is not well-formed [{}]; this might go poorly..."
.format(str(ex)))
print(pprint(ex.offending_subexpression))
print(pprint(ex.offending_subexpression.type))
# raise
state_vars = [v for (v, p) in context.vars() if p == STATE_POOL]
if eliminate_vars.value and can_elim_vars(target, assumptions, state_vars):
print("This job does not depend on state_vars.")
# TODO: what can we do about it?
hints = ([freshen_binders(h, context) for h in hints] + [
freshen_binders(wrap_naked_statevars(a, state_vars), context)
for a in break_conj(assumptions)
] + [target])
print("{} hints".format(len(hints)))
for h in hints:
print(" - {}".format(pprint(h)))
vars = list(v for (v, p) in context.vars())
funcs = context.funcs()
solver = None
if incremental.value:
solver = IncrementalSolver(vars=vars, funcs=funcs)
solver.add_assumption(assumptions)
_sat = solver.satisfy
else:
_sat = lambda e: satisfy(e, vars=vars, funcs=funcs)
if _sat(assumptions) is None:
print("assumptions are unsat; this query will never be called")
yield construct_value(target.type)
return
examples = list(examples)
if cost_model is None:
cost_model = CostModel(funcs=funcs, assumptions=assumptions)
watched_targets = [target]
learner = Learner(watched_targets, assumptions, context, examples,
cost_model, stop_callback, hints)
try:
while True:
# 1. find any potential improvement to any sub-exp of target
for new_target in learner.next():
print("Found candidate improvement: {}".format(
pprint(new_target)))
# 2. check
with task("verifying candidate"):
if incremental.value:
solver.push()
solver.add_assumption(
ENot(
EBinOp(target, "==",
new_target).with_type(BOOL)))
counterexample = _sat(T)
else:
formula = EAll([
assumptions,
ENot(
EBinOp(target, "==",
new_target).with_type(BOOL))
])
counterexample = _sat(formula)
if counterexample is not None:
if counterexample in examples:
print("assumptions = {!r}".format(assumptions))
print("duplicate example: {!r}".format(counterexample))
print("old target = {!r}".format(target))
print("new target = {!r}".format(new_target))
raise Exception("got a duplicate example")
# a. if incorrect: add example, reset the learner
examples.append(counterexample)
event("new example: {!r}".format(counterexample))
print("wrong; restarting with {} examples".format(
len(examples)))
learner.reset(examples)
break
else:
# b. if correct: yield it, watch the new target, goto 1
print("The candidate is valid!")
print(repr(new_target))
print("Determining whether to yield it...")
with task("updating frontier"):
to_evict = []
keep = True
old_better = None
for old_target in watched_targets:
evc = eviction_policy(new_target, context,
old_target, context,
RUNTIME_POOL, cost_model)
if old_target not in evc:
to_evict.append(old_target)
if new_target not in evc:
old_better = old_target
keep = False
break
for t in to_evict:
watched_targets.remove(t)
if not keep:
print(
"Whoops! Looks like we already found something better."
)
print(" --> {}".format(pprint(old_better)))
continue
if target in to_evict:
print("Yep, it's an improvement!")
yield new_target
if heuristic_done(new_target):
print("target now matches doneness heuristic")
raise NoMoreImprovements()
target = new_target
else:
print("Nope, it isn't substantially better!")
watched_targets.append(new_target)
print("Now watching {} targets".format(
len(watched_targets)))
learner.watch(watched_targets)
break
if incremental.value:
solver.pop()
except NoMoreImprovements:
return
except KeyboardInterrupt:
raise
def improve(
target : Exp,
assumptions : Exp,
binders : [EVar],
state_vars : [EVar],
args : [EVar],
cost_model : CostModel,
builder : ExpBuilder,
stop_callback = never_stop,
hints : [Exp] = None,
examples = None):
"""
Improve the target expression using enumerative synthesis.
This function is a generator that yields increasingly better and better
versions of the input expression `target`.
Notes on internals of this algorithm follow.
Key differences from "regular" enumerative synthesis:
- Expressions may be built using a set of "binders"---extra free
variables thrown into the mix at the beginning.
- Expressions are either "state" expressions or "runtime" expressions,
allowing this algorithm to choose what things to store on the data
structure and what things to compute at query execution time. (The
cost model is ultimately responsible for this choice.)
Other features of this algorithm:
- If a better version of *any subexpression* for the target is found,
it is immediately substituted in and the overall expression is
returned. This "smooths out" the search space a little, and lets us
find kinda-good solutions very quickly, even if the best possible
solution is out of reach.
"""
print("call to improve:")
print("""improve(
target={target!r},
assumptions={assumptions!r},
binders={binders!r},
state_vars={state_vars!r},
args={args!r},
cost_model={cost_model!r},
builder={builder!r},
stop_callback={stop_callback!r},
hints={hints!r},
examples={examples!r})""".format(
target=target,
assumptions=assumptions,
binders=binders,
state_vars=state_vars,
args=args,
cost_model=cost_model,
builder=builder,
stop_callback=stop_callback,
hints=hints,
examples=examples))
print()
print("improving: {}".format(pprint(target)))
print("subject to: {}".format(pprint(assumptions)))
print()
assert exp_wf(
target,
state_vars=set(state_vars),
args=set(args),
assumptions=assumptions)
binders = list(binders)
target = fixup_binders(target, binders, allow_add=False)
hints = [fixup_binders(h, binders, allow_add=False) for h in (hints or ())]
assumptions = fixup_binders(assumptions, binders, allow_add=False)
builder = FixedBuilder(builder, state_vars, args, binders, assumptions)
target_cost = cost_model.cost(target, RUNTIME_POOL)
if eliminate_vars.value and can_elim_vars(target, assumptions, state_vars):
print("This job does not depend on state_vars.")
builder = StateElimBuilder(builder)
vars = list(free_vars(target) | free_vars(assumptions))
funcs = free_funcs(EAll([target, assumptions]))
solver = None
if incremental.value:
solver = IncrementalSolver(vars=vars, funcs=funcs, collection_depth=check_depth.value)
solver.add_assumption(assumptions)
_sat = solver.satisfy
else:
_sat = lambda e: satisfy(e, vars=vars, funcs=funcs, collection_depth=check_depth.value)
if _sat(T) is None:
print("assumptions are unsat; this query will never be called")
yield construct_value(target.type)
return
if examples is None:
examples = []
learner = Learner(target, assumptions, binders, state_vars, args, vars + binders, examples, cost_model, builder, stop_callback, hints, solver=solver)
try:
while True:
# 1. find any potential improvement to any sub-exp of target
try:
old_e, new_e, local_assumptions, repl = learner.next()
except NoMoreImprovements:
break
# 2. substitute-in the improvement
print("Found candidate replacement [{}] for [{}] in".format(pprint(new_e), pprint(old_e)))
print(pprint(repl(EVar("@___"))))
new_target = repl(new_e)
# 3. check
if incremental.value:
solver.push()
solver.add_assumption(ENot(EBinOp(target, "==", new_target).with_type(BOOL)))
counterexample = _sat(T)
else:
formula = EAll([assumptions, ENot(EBinOp(target, "==", new_target).with_type(BOOL))])
counterexample = _sat(formula)
if counterexample is not None:
# Ok they aren't equal. Now we need an example that
# differentiates BOTH target/new_target AND old_e/new_e.
if incremental.value:
counterexample = _sat(EAll([
EAll(local_assumptions),
ENot(EBinOp(old_e, "===", new_e).with_type(BOOL))]))
else:
counterexample = _sat(EAll([
assumptions,
EAll(local_assumptions),
ENot(EBinOp(target, "==", new_target).with_type(BOOL)),
ENot(EBinOp(old_e, "===", new_e).with_type(BOOL))]))
if counterexample is None:
print("!!! unable to satisfy top- and sub-expressions")
print("assumptions = {!r}".format(assumptions))
print("local_assumptions = {!r}".format(EAll(local_assumptions)))
print("old_e = {!r}".format(old_e))
print("target = {!r}".format(target))
print("new_e = {!r}".format(new_e))
print("new_target = {!r}".format(new_target))
raise Exception("unable to find an example that differentiates both the toplevel- and sub-expressions")
if counterexample in examples:
print("assumptions = {!r}".format(assumptions))
print("duplicate example: {!r}".format(counterexample))
print("old target = {!r}".format(target))
print("new target = {!r}".format(new_target))
print("old fp = {}".format(learner._fingerprint(old_e)))
print("new fp = {}".format(learner._fingerprint(new_e)))
print("old target fp = {}".format(learner._fingerprint(target)))
print("new target fp = {}".format(learner._fingerprint(new_target)))
raise Exception("got a duplicate example")
# a. if incorrect: add example, reset the learner
examples.append(counterexample)
print("new example: {}".format(truncate(repr(counterexample))))
print("restarting with {} examples".format(len(examples)))
learner.reset(examples)
else:
# b. if correct: yield it, watch the new target, goto 1
if CHECK_FINAL_COST:
new_cost = cost_model.cost(new_target, RUNTIME_POOL)
print("cost: {} -----> {}".format(target_cost, new_cost))
if incremental.value:
ordering = new_cost.compare_to(target_cost, solver=solver)
else:
ordering = new_cost.compare_to(target_cost, assumptions=assumptions)
if ordering == Cost.WORSE:
if CHECK_SUBST_COST:
print("WHOOPS! COST GOT WORSE!")
if save_testcases.value:
with open(save_testcases.value, "a") as f:
f.write("def testcase():\n")
f.write(" costmodel = {}\n".format(repr(cost_model)))
f.write(" old_e = {}\n".format(repr(old_e)))
f.write(" new_e = {}\n".format(repr(new_e)))
f.write(" target = {}\n".format(repr(target)))
f.write(" new_target = {}\n".format(repr(new_target)))
f.write(" if costmodel.cost(new_e, RUNTIME_POOL) <= costmodel.cost(old_e, RUNTIME_POOL) and costmodel.cost(new_target, RUNTIME_POOL) > costmodel.cost(target, RUNTIME_POOL):\n")
f.write(' for name, x in zip(["old_e", "new_e", "target", "new_target"], [old_e, new_e, target, new_target]):\n')
f.write(' print("{}: {}".format(name, pprint(x)))\n')
f.write(' print(" cost = {}".format(costmodel.cost(x, RUNTIME_POOL)))\n')
f.write(" assert False\n")
# raise Exception("detected nonmonotonicity")
else:
print("*** cost is worse")
# print(repr(target))
# print(repr(new_target))
continue
elif ordering == Cost.UNORDERED:
print("*** cost is unchanged")
# print(repr(target))
# print(repr(new_target))
continue
target_cost = new_cost
print("found improvement: {} -----> {}".format(pprint(old_e), pprint(new_e)))
# print(repr(target))
# print(repr(new_target))
# binders are not allowed to "leak" out
to_yield = new_target
if any(v in binders for v in free_vars(new_target)):
print("WARNING: stripping binders in {}".format(pprint(new_target)), file=sys.stderr)
to_yield = subst(new_target, { b.id : construct_value(b.type) for b in binders })
yield to_yield
if reset_on_success.value and (not CHECK_FINAL_COST or ordering != Cost.UNORDERED):
learner.reset(examples)
learner.watch(new_target)
target = new_target
if heuristic_done(new_target, args):
print("target now matches doneness heuristic")
break
if incremental.value:
solver.pop()
except KeyboardInterrupt:
for e in learner.cache.random_sample(50):
print(pprint(e))
raise
