def watch(self, new_target):
print("watching new target...")
self.backlog_counter = 0
self.target = new_target
self.roots = OrderedSet()
types = OrderedSet()
for e in itertools.chain(all_exps(new_target), *[all_exps(h) for h in self.hints]):
if isinstance(e, ELambda):
continue
for pool in ALL_POOLS:
exp = e
if pool == STATE_POOL:
exp = strip_EStateVar(e)
fvs = free_vars(exp)
if all(v in self.legal_free_vars for v in fvs) and self.is_legal_in_pool(exp, pool):
_on_exp(exp, "new root", pool_name(pool))
exp._root = True
self.roots.add((exp, pool))
if pool == STATE_POOL and all(v in self.state_vars for v in fvs):
self.roots.add((EStateVar(exp).with_type(exp.type), RUNTIME_POOL))
types.add(exp.type)
else:
_on_exp(exp, "rejected root", pool_name(pool))
for b in self.binders:
types.add(b.type)
for t in types:
self.roots.add((construct_value(t), RUNTIME_POOL))
self.roots = list(self.roots)
self.roots.sort(key = lambda tup: tup[0].size())
self._watches = group_by(
enumerate_fragments2(new_target),
k=lambda ctx: (ctx.pool, ctx.e.type),
v=lambda ctxs: sorted(ctxs, key=lambda ctx: -ctx.e.size()))
print("done!")
def _consider(e, size, context, pool):
if _interesting(e, size, context, pool) and not verbose.value:
print("considering {} @ size={} in {}/{}".format(
pprint(e), size, context, pool_name(pool)))
task_begin("considering expression",
expression=pprint(e),
size=size,
context=context,
pool=pool_name(pool),
interesting=_interesting(e, size, context, pool))
def _consider(e, size, context, pool):
"""Called when an Enumerator sees an expression for the first time."""
if _interesting(e, size, context, pool) and not verbose.value:
print("considering {} @ size={} in {}/{}".format(
pprint(e), size, context, pool_name(pool)))
task_begin("considering expression",
expression=pprint(e),
size=size,
context=context,
pool=pool_name(pool),
interesting=_interesting(e, size, context, pool))
def _accept(e, size, context, pool, fingerprint):
"""Called when an Enumerator "accepts" an expression and adds it to the cache."""
if _interesting(e, size, context, pool) and not verbose.value:
print("accepting [fp={}]".format(fingerprint))
event("accepting {} @ {} in {}/{}".format(pprint(e), size, context,
pool_name(pool)))
task_end()
def _possible_replacements(self, e, pool, cost):
"""
Yields watched expressions that appear as worse versions of the given
expression. There may be more than one.
"""
# return
free_binders = OrderedSet(v for v in free_vars(e) if v in self.binders)
for ctx in self._watched_contexts(pool, e.type):
watched_e = ctx.e
p = ctx.pool
r = ctx.replace_e_with
assert e.type == watched_e.type
assert p == pool
_on_exp(e, "considering replacement of", watched_e)
# if e.type != watched_e.type:
# # _on_exp(e, "wrong type")
# continue
# if p != pool:
# # _on_exp(e, "wrong pool")
# continue
if e == watched_e:
# _on_exp(e, "no change")
continue
unbound_binders = [b for b in free_binders if b not in ctx.bound_vars]
if unbound_binders:
_on_exp(e, "skipped exp with free binders", ", ".join(b.id for b in unbound_binders))
continue
if CHECK_SUBST_COST:
watched_cost = self.cost_model.cost(watched_e, pool=pool)
ordering = self.compare_costs(cost, watched_cost)
if ordering == Cost.WORSE:
_on_exp(e, "skipped worse replacement", pool_name(pool), watched_e)
continue
if ordering == Cost.UNORDERED:
_on_exp(e, "skipped equivalent replacement", pool_name(pool), watched_e)
# print(" e1 = {!r}".format(e))
# print(" e2 = {!r}".format(watched_e))
continue
# assert all(eval_bulk(self.assumptions, self.all_examples))
if all(eval_bulk(EEq(self.target, r(e)), self.all_examples)):
yield (watched_e, e, ctx.facts, r)
else:
_on_exp(e, "visited pointless replacement", watched_e)
def enumerate_with_info(self, context: Context, size: int,
pool: Pool) -> [EnumeratedExp]:
canonical_context = self.canonical_context(context)
if canonical_context is not context:
print("adapting request: {} ---> {}".format(
context, canonical_context))
for info in self.enumerate_with_info(canonical_context, size,
pool):
yield info._replace(e=context.adapt(info.e, canonical_context))
return
if context.parent() is not None:
yield from self.enumerate_with_info(context.parent(), size, pool)
k = (pool, size, context)
res = self.cache.get(k)
if res is not None:
# print("[[{} cached @ size={}]]".format(len(res), size))
for e in res:
yield e
else:
# print("ENTER {}".format(k))
examples = context.instantiate_examples(self.examples)
assert k not in self.in_progress, "recursive enumeration?? {}".format(
k)
self.in_progress.add(k)
res = []
self.cache[k] = res
queue = self.enumerate_core(context, size, pool)
cost_model = self.cost_model
while True:
if self.stop_callback():
raise StopException()
try:
e = next(queue)
except StopIteration:
break
fvs = free_vars(e)
if not belongs_in_context(fvs, context):
continue
e = freshen_binders(e, context)
_consider(e, context, pool)
wf = self.check_wf(e, context, pool)
if not wf:
_skip(e, context, pool, "wf={}".format(wf))
continue
fp = fingerprint(e, examples)
# collect all expressions from parent contexts
with task("collecting prev exps",
size=size,
context=context,
pool=pool_name(pool)):
prev = []
for sz in range(0, size + 1):
prev.extend(self.enumerate_with_info(
context, sz, pool))
prev = [p.e for p in prev if p.fingerprint == fp]
if any(alpha_equivalent(e, p) for p in prev):
_skip(e, context, pool, "duplicate")
should_keep = False
else:
# decide whether to keep this expression,
# decide which can be evicted
should_keep = True
# cost = self.cost_model.cost(e, pool)
# print("prev={}".format(prev))
# print("seen={}".format(self.seen))
with task("comparing to cached equivalents"):
for prev_exp in prev:
event("previous: {}".format(pprint(prev_exp)))
# prev_cost = self.cost_model.cost(prev_exp, pool)
# ordering = cost.compare_to(prev_cost)
to_keep = eviction_policy(e, context, prev_exp,
context, pool,
cost_model)
if e not in to_keep:
_skip(e, context, pool,
"preferring {}".format(pprint(prev_exp)))
should_keep = False
break
# if ordering == Order.LT:
# pass
# elif ordering == Order.GT:
# self.blacklist.add(e_key)
# _skip(e, context, pool, "worse than {}".format(pprint(prev_exp)))
# should_keep = False
# break
# else:
# self.blacklist.add(e_key)
# _skip(e, context, pool, "{} to cached {}".format(
# "equal" if ordering == Order.EQUAL else "similar",
# pprint(prev_exp)))
# assert ordering in (Order.EQUAL, Order.AMBIGUOUS)
# should_keep = False
# break
if should_keep:
with task("evicting"):
to_evict = []
for (key, exps) in self.cache.items():
(p, s, c) = key
if p == pool and c in itertools.chain(
[context], parent_contexts(context)):
for ee in exps:
if ee.fingerprint == fp: # and cost_model.compare(e, ee.e, context, pool) == Order.LT:
# to_evict.append((key, ee))
to_keep = eviction_policy(
e, context, ee.e, c, pool,
cost_model)
if ee.e not in to_keep:
to_evict.append((key, ee))
for key, ee in to_evict:
(p, s, c) = key
# self.blacklist.add((ee.e, c, pool))
_evict(ee.e, c, pool, e)
self.cache[key].remove(ee)
self.seen[(c, p, fp)].remove(ee.e)
_accept(e, context, pool)
seen_key = (context, pool, fp)
if seen_key not in self.seen:
self.seen[seen_key] = []
self.seen[seen_key].append(e)
info = EnumeratedExp(e=e, fingerprint=fp, cost=None)
res.append(info)
yield info
with task("accelerating"):
to_try = make_random_access(
self.heuristics(e, context, pool))
if to_try:
# print("trying {} accelerations".format(len(to_try)))
queue = itertools.chain(to_try, queue)
# print("EXIT {}".format(k))
self.in_progress.remove(k)
def _accept(e, size, context, pool):
if _interesting(e, size, context, pool) and not verbose.value:
print("accepting")
event("accepting {} @ {} in {}/{}".format(pprint(e), size, context,
pool_name(pool)))
task_end()
def enumerate_with_info(self, context: Context, size: int,
pool: Pool) -> [EnumeratedExp]:
canonical_context = self.canonical_context(context)
if canonical_context is not context:
print("adapting request: {} ---> {}".format(
context, canonical_context))
for info in self.enumerate_with_info(canonical_context, size,
pool):
yield info._replace(e=context.adapt(info.e, canonical_context))
return
examples = context.instantiate_examples(self.examples)
if context.parent() is not None:
for info in self.enumerate_with_info(context.parent(), size, pool):
e = info.e
yield EnumeratedExp(e=e, fingerprint=fingerprint(e, examples))
k = (pool, size, context)
res = self.cache.get(k)
if res is not None:
for e in res:
yield e
else:
assert k not in self.in_progress, "recursive enumeration?? {}".format(
k)
self.in_progress.add(k)
res = []
self.cache[k] = res
queue = self.enumerate_core(context, size, pool)
cost_model = self.cost_model
while True:
if self.stop_callback():
raise StopException()
try:
e = next(queue)
except StopIteration:
break
fvs = free_vars(e)
if not belongs_in_context(fvs, context):
continue
e = freshen_binders(e, context)
_consider(e, size, context, pool)
wf = self.check_wf(e, context, pool)
if not wf:
_skip(e, size, context, pool, "wf={}".format(wf))
continue
fp = fingerprint(e, examples)
# collect all expressions from parent contexts
with task("collecting prev exps",
size=size,
context=context,
pool=pool_name(pool)):
prev = []
for sz in range(0, size + 1):
prev.extend(self.enumerate_with_info(
context, sz, pool))
prev = [p.e for p in prev if p.fingerprint == fp]
if any(alpha_equivalent(e, p) for p in prev):
_skip(e, size, context, pool, "duplicate")
should_keep = False
else:
# decide whether to keep this expression
should_keep = True
with task("comparing to cached equivalents"):
for prev_exp in prev:
event("previous: {}".format(pprint(prev_exp)))
to_keep = eviction_policy(e, context, prev_exp,
context, pool,
cost_model)
if e not in to_keep:
_skip(e, size, context, pool,
"preferring {}".format(pprint(prev_exp)))
should_keep = False
break
if should_keep:
if self.do_eviction:
with task("evicting"):
to_evict = []
for (key, exps) in self.cache.items():
(p, s, c) = key
if p == pool and c == context:
for ee in exps:
if ee.fingerprint == fp:
event("considering eviction of {}".
format(pprint(ee.e)))
to_keep = eviction_policy(
e, context, ee.e, c, pool,
cost_model)
if ee.e not in to_keep:
to_evict.append((key, ee))
for key, ee in to_evict:
(p, s, c) = key
_evict(ee.e, s, c, pool, e)
self.cache[key].remove(ee)
self.seen[(c, p, fp)].remove(ee.e)
_accept(e, size, context, pool)
seen_key = (context, pool, fp)
if seen_key not in self.seen:
self.seen[seen_key] = []
self.seen[seen_key].append(e)
info = EnumeratedExp(e=e, fingerprint=fp)
res.append(info)
yield info
with task("accelerating"):
to_try = make_random_access(
self.heuristics(e, context, pool))
if to_try:
event("trying {} accelerations of {}".format(
len(to_try), pprint(e)))
queue = itertools.chain(to_try, queue)
self.in_progress.remove(k)
def next(self):
target_cost = self.cost_model.cost(self.target, RUNTIME_POOL)
self.ncount += 1
while True:
if self.backlog is not None:
if self.stop_callback():
raise StopException()
(e, pool, cost) = self.backlog
improvements = list(self._possible_replacements(e, pool, cost))
if self.backlog_counter < len(improvements):
i = improvements[self.backlog_counter]
self.backlog_counter += 1
return i
else:
self.backlog = None
self.backlog_counter = 0
for (e, pool) in self.builder_iter:
self._on_exp(e, pool)
if self.stop_callback():
raise StopException()
# # Stopgap measure... long story --Calvin
# bad = False
# for x in all_exps(e):
# if isinstance(x, EStateVar):
# if any(v not in self.state_vars for v in free_vars(x.e)):
# bad = True
# _on_exp(e, "skipping due to illegal free vars under EStateVar")
# if bad:
# continue
new_e = self.pre_optimize(e, pool) if preopt.value else e
if new_e is not e:
_on_exp(e, "preoptimized", new_e)
e = new_e
cost = self.cost_model.cost(e, pool)
if pool == RUNTIME_POOL and (self.cost_model.is_monotonic() or hyperaggressive_culling.value) and self.compare_costs(cost, target_cost) == Cost.WORSE:
_on_exp(e, "too expensive", cost, target_cost)
continue
fp = self._fingerprint(e)
prev = list(self.seen.find_all(pool, fp))
should_add = True
if not prev:
_on_exp(e, "new", pool_name(pool))
elif any(alpha_equivalent(e, ee) for (ee, _, _) in prev):
_on_exp(e, "duplicate")
should_add = False
else:
better_than = None
worse_than = None
for prev_exp, prev_size, prev_cost in prev:
self._on_cost_cmp()
ordering = self.compare_costs(cost, prev_cost)
assert ordering in (Cost.WORSE, Cost.BETTER, Cost.UNORDERED)
if enforce_strong_progress.value and ordering != Cost.WORSE:
bad = find_one(all_exps(e), lambda ee: alpha_equivalent(ee, prev_exp))
if bad:
_on_exp(e, "failed strong progress requirement", bad)
should_add = False
break
_on_exp(e, ordering, pool_name(pool), prev_exp)
if ordering == Cost.UNORDERED:
continue
elif ordering == Cost.BETTER:
better_than = (prev_exp, prev_size, prev_cost)
_on_exp(prev_exp, "found better alternative", e)
self.cache.evict(prev_exp, size=prev_size, pool=pool)
self.seen.remove(prev_exp, pool, fp)
if (self.cost_model.is_monotonic() or hyperaggressive_culling.value) and hyperaggressive_eviction.value:
for (cached_e, size, p) in list(self.cache):
if p != pool:
continue
if prev_exp in all_exps(cached_e):
_on_exp(cached_e, "evicted since it contains", prev_exp)
self.cache.evict(cached_e, size=size, pool=pool)
else:
should_add = False
worse_than = (prev_exp, prev_size, prev_cost)
# break
if worse_than and better_than:
print("Uh-oh! Strange cost relationship between")
print(" (1) this exp: {}".format(pprint(e)))
print(" (2) prev. A: {}".format(pprint(worse_than[0])))
print(" (2) prev. B: {}".format(pprint(better_than[0])))
print("e1 = {}".format(repr(e)))
print("e2 = {}".format(repr(worse_than[0])))
print("e3 = {}".format(repr(better_than[0])))
print("(1) vs (2): {}".format(cost.compare_to(worse_than[2], self.assumptions)))
print("(2) vs (3): {}".format(worse_than[2].compare_to(better_than[2], self.assumptions)))
print("(3) vs (1): {}".format(better_than[2].compare_to(cost, self.assumptions)))
# raise Exception("insane cost model behavior")
if should_add:
self.cache.add(e, pool=pool, size=self.current_size)
self.seen.add(e, pool, fp, self.current_size, cost)
self.last_progress = self.current_size
else:
continue
for pr in self._possible_replacements(e, pool, cost):
self.backlog = (e, pool, cost)
self.backlog_counter = 1
return pr
if self.last_progress < (self.current_size+1) // 2:
raise NoMoreImprovements("hit termination condition")
self.current_size += 1
self.builder_iter = self.builder.build(self.cache, self.current_size)
if self.current_size == 0:
self.builder_iter = itertools.chain(self.builder_iter, list(self.roots))
for f, ct in sorted(_fates.items(), key=lambda x: x[1], reverse=True):
print(" {:6} | {}".format(ct, f))
_fates.clear()
self._start_minor_it()
def search_for_improvements(targets: [Exp], wf_solver: ModelCachingSolver,
context: Context, examples: [{
str: object
}], cost_model: CostModel,
stop_callback: Callable[[], bool], hints: [Exp],
ops: [Op], blacklist: {
(Exp, Context, Pool, Exp): str
}):
"""Search for potential improvements to any of the target expressions.
This function yields expressions that look like improvements (or are
ambiguous with respect to some target). The expressions are only
guaranteed to be correct on the given examples.
This function may add new items to the given blacklist.
"""
root_ctx = context
def check_wf(e, ctx, pool):
with task("pruning", size=e.size()):
is_wf = exp_wf(e, pool=pool, context=ctx, solver=wf_solver)
if not is_wf:
return is_wf
res = possibly_useful(wf_solver, e, ctx, pool, ops=ops)
if not res:
return res
if cost_pruning.value and pool == RUNTIME_POOL and cost_model.compare(
e, targets[0], ctx, pool) == Order.GT:
return No("too expensive")
return True
with task("setting up hints"):
frags = list(
unique(
itertools.chain(
*[
all_subexpressions_with_context_information(
t, root_ctx) for t in targets
], *[
all_subexpressions_with_context_information(
h, root_ctx) for h in hints
])))
frags.sort(key=hint_order)
enum = Enumerator(examples=examples,
cost_model=cost_model,
check_wf=check_wf,
hints=frags,
heuristics=try_optimize,
stop_callback=stop_callback,
do_eviction=enable_eviction.value)
target_fp = Fingerprint.of(targets[0], examples)
with task("setting up watches"):
watches_by_context = OrderedDict()
for target in targets:
for e, ctx, pool in unique(
all_subexpressions_with_context_information(
target, context=root_ctx, pool=RUNTIME_POOL)):
l = watches_by_context.get(ctx)
if l is None:
l = []
watches_by_context[ctx] = l
l.append((target, e, pool))
watches = OrderedDict()
for ctx, exprs in watches_by_context.items():
exs = ctx.instantiate_examples(examples)
for target, e, pool in exprs:
fp = Fingerprint.of(e, exs)
k = (fp, ctx, pool)
l = watches.get(k)
if l is None:
l = []
watches[k] = l
l.append((target, e))
watched_ctxs = list(
unique(
(ctx, pool)
for _, _, ctx, pool in exploration_order(targets, root_ctx)))
search_info = SearchInfo(context=root_ctx,
targets=targets,
target_fingerprint=target_fp,
examples=examples,
check_wf=check_wf,
cost_model=cost_model,
blacklist=blacklist)
size = 0
while True:
print("starting minor iteration {} with |cache|={}".format(
size, enum.cache_size()))
if stop_callback():
raise StopException()
for ctx, pool in watched_ctxs:
with task("searching for obvious substitutions",
ctx=ctx,
pool=pool_name(pool)):
for info in enum.enumerate_with_info(size=size,
context=ctx,
pool=pool):
with task("searching for obvious substitution",
expression=pprint(info.e)):
fp = info.fingerprint
for ((fpx, cc, pp), reses) in watches.items():
if cc != ctx or pp != pool:
continue
if not fpx.equal_to(fp):
continue
for target, watched_e in reses:
replacement = info.e
event(
"possible substitution: {} ---> {}".format(
pprint(watched_e),
pprint(replacement)))
event("replacement locations: {}".format(
pprint(
replace(target, root_ctx, RUNTIME_POOL,
watched_e, ctx, pool,
EVar("___")))))
if alpha_equivalent(watched_e, replacement):
event("no change")
continue
yield from _consider_replacement(
target, watched_e, ctx, pool, replacement,
search_info)
if check_blind_substitutions.value:
print("Guessing at substitutions...")
for target, e, ctx, pool in exploration_order(targets, root_ctx):
with task("checking substitutions",
target=pprint(
replace(target, root_ctx, RUNTIME_POOL, e, ctx,
pool, EVar("___"))),
e=pprint(e)):
for info in enum.enumerate_with_info(size=size,
context=ctx,
pool=pool):
with task("checking substitution",
expression=pprint(info.e)):
if stop_callback():
raise StopException()
replacement = info.e
if replacement.type != e.type:
event("wrong type (is {}, need {})".format(
pprint(replacement.type), pprint(e.type)))
continue
if alpha_equivalent(replacement, e):
event("no change")
continue
should_consider = should_consider_replacement(
target, root_ctx, e, ctx, pool,
Fingerprint.of(
e, ctx.instantiate_examples(examples)),
info.e, info.fingerprint)
if not should_consider:
event(
"skipped; `should_consider_replacement` returned {}"
.format(should_consider))
continue
yield from _consider_replacement(
target, e, ctx, pool, replacement, search_info)
if not enum.expressions_may_exist_above_size(context, RUNTIME_POOL,
size):
raise StopException(
"no more expressions can exist above size={}".format(size))
size += 1
def search(self):
root_ctx = self.context
def check_wf(e, ctx, pool):
with task("checking well-formedness", size=e.size()):
is_wf = exp_wf(e,
pool=pool,
context=ctx,
solver=self.wf_solver)
if not is_wf:
return is_wf
res = good_idea_recursive(self.wf_solver,
e,
ctx,
pool,
ops=self.ops)
if not res:
return res
if pool == RUNTIME_POOL and self.cost_model.compare(
e, self.targets[0], ctx, pool) == Order.GT:
return No("too expensive")
return True
frags = list(
unique(
itertools.chain(*[shred(t, root_ctx) for t in self.targets],
*[shred(h, root_ctx) for h in self.hints])))
frags.sort(key=hint_order)
enum = Enumerator(examples=self.examples,
cost_model=self.cost_model,
check_wf=check_wf,
hints=frags,
heuristics=try_optimize,
stop_callback=self.stop_callback,
do_eviction=enable_eviction.value)
size = 0
target_fp = fingerprint(self.targets[0], self.examples)
watches = OrderedDict()
for target in self.targets:
for e, ctx, pool in unique(
shred(target, context=root_ctx, pool=RUNTIME_POOL)):
exs = ctx.instantiate_examples(self.examples)
fp = fingerprint(e, exs)
k = (fp, ctx, pool)
l = watches.get(k)
if l is None:
l = []
watches[k] = l
l.append((target, e))
watched_ctxs = list(
unique((ctx, pool) for fp, ctx, pool in watches.keys()))
def consider_new_target(old_target, e, ctx, pool, replacement):
nonlocal n
n += 1
k = (e, ctx, pool, replacement)
if enable_blacklist.value and k in self.blacklist:
event("blacklisted")
print("skipping blacklisted substitution: {} ---> {} ({})".
format(pprint(e), pprint(replacement),
self.blacklist[k]))
return
new_target = freshen_binders(
replace(target, root_ctx, RUNTIME_POOL, e, ctx, pool,
replacement), root_ctx)
if any(alpha_equivalent(t, new_target) for t in self.targets):
event("already seen")
return
wf = check_wf(new_target, root_ctx, RUNTIME_POOL)
if not wf:
msg = "not well-formed [wf={}]".format(wf)
event(msg)
self.blacklist[k] = msg
return
if not fingerprints_match(fingerprint(new_target, self.examples),
target_fp):
msg = "not correct"
event(msg)
self.blacklist[k] = msg
return
if self.cost_model.compare(new_target, target, root_ctx,
RUNTIME_POOL) not in (Order.LT,
Order.AMBIGUOUS):
msg = "not an improvement"
event(msg)
self.blacklist[k] = msg
return
print("FOUND A GUESS AFTER {} CONSIDERED".format(n))
print(" * in {}".format(pprint(old_target), pprint(e),
pprint(replacement)))
print(" * replacing {}".format(pprint(e)))
print(" * with {}".format(pprint(replacement)))
yield new_target
while True:
print("starting minor iteration {} with |cache|={}".format(
size, enum.cache_size()))
if self.stop_callback():
raise StopException()
n = 0
for ctx, pool in watched_ctxs:
with task("searching for obvious substitutions",
ctx=ctx,
pool=pool_name(pool)):
for info in enum.enumerate_with_info(size=size,
context=ctx,
pool=pool):
with task("searching for obvious substitution",
expression=pprint(info.e)):
fp = info.fingerprint
for ((fpx, cc, pp), reses) in watches.items():
if cc != ctx or pp != pool:
continue
if not fingerprints_match(fpx, fp):
continue
for target, watched_e in reses:
replacement = info.e
event("possible substitution: {} ---> {}".
format(pprint(watched_e),
pprint(replacement)))
event("replacement locations: {}".format(
pprint(
replace(target, root_ctx,
RUNTIME_POOL, watched_e,
ctx, pool, EVar("___")))))
if alpha_equivalent(
watched_e, replacement):
event("no change")
continue
yield from consider_new_target(
target, watched_e, ctx, pool,
replacement)
if check_all_substitutions.value:
print("Guessing at substitutions...")
for target, e, ctx, pool in exploration_order(
self.targets, root_ctx):
with task("checking substitutions",
target=pprint(
replace(target, root_ctx, RUNTIME_POOL, e,
ctx, pool, EVar("___"))),
e=pprint(e)):
for info in enum.enumerate_with_info(size=size,
context=ctx,
pool=pool):
with task("checking substitution",
expression=pprint(info.e)):
if self.stop_callback():
raise StopException()
replacement = info.e
if replacement.type != e.type:
event("wrong type (is {}, need {})".format(
pprint(replacement.type),
pprint(e.type)))
continue
if alpha_equivalent(replacement, e):
event("no change")
continue
should_consider = should_consider_replacement(
target, root_ctx, e, ctx, pool,
fingerprint(
e,
ctx.instantiate_examples(
self.examples)), info.e,
info.fingerprint)
if not should_consider:
event(
"skipped; `should_consider_replacement` returned {}"
.format(should_consider))
continue
yield from consider_new_target(
target, e, ctx, pool, replacement)
print("CONSIDERED {}".format(n))
size += 1
def search_for_improvements(
targets : [Exp],
wf_solver : ModelCachingSolver,
context : Context,
examples : [{str:object}],
cost_model : CostModel,
stop_callback : Callable[[], bool],
hints : [Exp],
ops : [Op],
blacklist : {(Exp, Context, Pool, Exp) : str}):
"""Search for potential improvements to any of the target expressions.
This function yields expressions that look like improvements (or are
ambiguous with respect to some target). The expressions are only
guaranteed to be correct on the given examples.
This function may add new items to the given blacklist.
"""
root_ctx = context
def check_wf(e, ctx, pool):
with task("pruning", size=e.size()):
is_wf = exp_wf(e, pool=pool, context=ctx, solver=wf_solver)
if not is_wf:
return is_wf
res = possibly_useful(wf_solver, e, ctx, pool, ops=ops)
if not res:
return res
if cost_pruning.value and pool == RUNTIME_POOL and cost_model.compare(e, targets[0], ctx, pool) == Order.GT:
return No("too expensive")
return True
with task("setting up hints"):
frags = list(unique(itertools.chain(
*[all_subexpressions_with_context_information(t, root_ctx) for t in targets],
*[all_subexpressions_with_context_information(h, root_ctx) for h in hints])))
frags.sort(key=hint_order)
enum = Enumerator(
examples=examples,
cost_model=cost_model,
check_wf=check_wf,
hints=frags,
heuristics=try_optimize,
stop_callback=stop_callback,
do_eviction=enable_eviction.value)
target_fp = Fingerprint.of(targets[0], examples)
with task("setting up watches"):
watches_by_context = OrderedDict()
for target in targets:
for e, ctx, pool in unique(all_subexpressions_with_context_information(target, context=root_ctx, pool=RUNTIME_POOL)):
l = watches_by_context.get(ctx)
if l is None:
l = []
watches_by_context[ctx] = l
l.append((target, e, pool))
watches = OrderedDict()
for ctx, exprs in watches_by_context.items():
exs = ctx.instantiate_examples(examples)
for target, e, pool in exprs:
fp = Fingerprint.of(e, exs)
k = (fp, ctx, pool)
l = watches.get(k)
if l is None:
l = []
watches[k] = l
l.append((target, e))
watched_ctxs = list(unique((ctx, pool) for _, _, ctx, pool in exploration_order(targets, root_ctx)))
search_info = SearchInfo(
context=root_ctx,
targets=targets,
target_fingerprint=target_fp,
examples=examples,
check_wf=check_wf,
cost_model=cost_model,
blacklist=blacklist)
size = 0
while True:
print("starting minor iteration {} with |cache|={}".format(size, enum.cache_size()))
if stop_callback():
raise StopException()
for ctx, pool in watched_ctxs:
with task("searching for obvious substitutions", ctx=ctx, pool=pool_name(pool)):
for info in enum.enumerate_with_info(size=size, context=ctx, pool=pool):
with task("searching for obvious substitution", expression=pprint(info.e)):
fp = info.fingerprint
for ((fpx, cc, pp), reses) in watches.items():
if cc != ctx or pp != pool:
continue
if not fpx.equal_to(fp):
continue
for target, watched_e in reses:
replacement = info.e
event("possible substitution: {} ---> {}".format(pprint(watched_e), pprint(replacement)))
event("replacement locations: {}".format(pprint(replace(target, root_ctx, RUNTIME_POOL, watched_e, ctx, pool, EVar("___")))))
if alpha_equivalent(watched_e, replacement):
event("no change")
continue
yield from _consider_replacement(target, watched_e, ctx, pool, replacement, search_info)
if check_blind_substitutions.value:
print("Guessing at substitutions...")
for target, e, ctx, pool in exploration_order(targets, root_ctx):
with task("checking substitutions",
target=pprint(replace(target, root_ctx, RUNTIME_POOL, e, ctx, pool, EVar("___"))),
e=pprint(e)):
for info in enum.enumerate_with_info(size=size, context=ctx, pool=pool):
with task("checking substitution", expression=pprint(info.e)):
if stop_callback():
raise StopException()
replacement = info.e
if replacement.type != e.type:
event("wrong type (is {}, need {})".format(pprint(replacement.type), pprint(e.type)))
continue
if alpha_equivalent(replacement, e):
event("no change")
continue
should_consider = should_consider_replacement(
target, root_ctx,
e, ctx, pool, Fingerprint.of(e, ctx.instantiate_examples(examples)),
info.e, info.fingerprint)
if not should_consider:
event("skipped; `should_consider_replacement` returned {}".format(should_consider))
continue
yield from _consider_replacement(target, e, ctx, pool, replacement, search_info)
if not enum.expressions_may_exist_above_size(context, RUNTIME_POOL, size):
raise StopException("no more expressions can exist above size={}".format(size))
size += 1
