def __init__(self, n_dist, rng_fixed, add_trace, probs=None):
self.n_dist = n_dist / np.sum(n_dist)
self.rng_fixed = rng_fixed
self.add_trace = add_trace
self.probs = probs
self.rng = np.random.RandomState()
self.parser = parser_for_synthesis.KarelForSynthesisParser(
build_tree=True)
self.executor = executor.KarelExecutor(action_limit=250)
def __init__(self, code):
self.parser = parser_for_synthesis.KarelForSynthesisParser(
build_tree=True)
self.executor = executor.KarelExecutor()
self.code = code
tree = self.parser.parse(code)
# Statement coverage: actions
self.stmt_coverage = {span: 0 for span in self.parser.action_spans}
# Branch coverage: if, ifelse, while
self.branch_coverage = {(span, cond_value): 0
for span in self.parser.cond_block_spans
for cond_value in (True, False)}
def eval_traces(trace_model, eval_dataset, beam_size):
parser = parser_for_synthesis.KarelForSynthesisParser()
tracer = karel_trace_model.KarelTracer(parser.karel)
def get_trace(tracer, prog, input_grid):
tracer.reset(grid=input_grid)
prog()
return tracer.actions
def check_trace_passes(kr, tracer, input_grid, output_grid, candidates,
gold_trace):
passes = collections.defaultdict(bool)
exact_match = collections.defaultdict(bool)
for i, cand in enumerate(candidates):
# print(cand, gold_trace)
if cand == gold_trace:
exact_match[i] = True
tracer.reset(grid=input_grid)
for action in cand:
success = getattr(kr, action, lambda: False)()
if np.all(tracer.full_grid == output_grid):
passes[i] = True
break
return passes, exact_match
trace_model.args.max_beam_trees = beam_size
report = []
iterator = tqdm.tqdm(eval_dataset, dynamic_ncols=True)
for batch in iterator:
res = trace_model.inference(batch)
idx = 0
input_grids = batch.input_grids.data.numpy().astype(bool)
output_grids = batch.output_grids.data.numpy().astype(bool)
for ex in batch.orig_examples:
info = []
prog = parser.parse(ex.code_sequence)
for i, test in enumerate(ex.input_tests + ex.tests):
candidates = res[idx].info['candidates']
gold_trace = get_trace(tracer, prog, input_grids[idx])
passes, exact_match = check_trace_passes(
trace_model.kr, trace_model.tracer, input_grids[idx],
output_grids[idx], candidates, gold_trace)
info.append((passes, exact_match))
idx += 1
report.append(info)
assert idx == len(res)
return report
def __init__(self, vocab, for_eval, args):
self.vocab = vocab
self.for_eval = for_eval
self.args = args
self.karel_parser = parser_for_synthesis.KarelForSynthesisParser()
self.tracer = KarelTracer(self.karel_parser.karel)
def load_model(model_dir, model_type, step=None):
args = saver.ArgsDict(model_dir=model_dir,
model_type=model_type,
step=step)
saver.restore_args(args)
arguments.backport_default_args(args)
dataset.set_vocab(args)
m = models.get_model(args)
eval_dataset = dataset.get_eval_dataset(args, m)
m.model.eval()
the_executor = executor.get_executor(args)()
return m, eval_dataset, the_executor
parser = parser_for_synthesis.KarelForSynthesisParser()
tracer = karel_trace_model.KarelTracer(parser.karel)
def get_traces(code, tests):
result = []
prog = parser.parse(code)
for test in tests:
tracer.reset(indices=test['input'])
prog()
result.append(tracer.actions)
return result
def check_correct(code, tests, e):
res = executor.evaluate_code(code, None, tests, e)
class TestComputeAddOps(object):
parser = parser_for_synthesis.KarelForSynthesisParser(build_tree=True)
def linearize_to_tokens(self, code):
tokens, orig_spans = refine_env.ComputeAddOps.linearize(
self.parser.parse(code))
return tuple(refine_env.ComputeAddOps.idx_to_token[i]
for i in tokens), orig_spans
@pytest.mark.parametrize('tokens,linearized', [
('DEF run m( move putMarker m)', (('move', 'putMarker'), ((3, 3),
(4, 4)))),
('DEF run m( move '
'IF c( not c( frontIsClear c) c) i( putMarker i) '
'turnLeft m)', (('move', ('if', ('not', 'frontIsClear')), 'putMarker',
('end-if', ('not', 'frontIsClear')), 'turnLeft'),
((3, 3), (4, 11), (12, 12), (13, 13), (14, 14)))),
('DEF run m( move REPEAT R=5 r( putMarker r) turnLeft m)',
(('move', ('repeat', 5), 'putMarker', ('end-repeat', 5), 'turnLeft'),
((3, 3), (4, 6), (7, 7), (8, 8), (9, 9)))),
('DEF run m( pickMarker '
'IFELSE c( noMarkersPresent c) i( putMarker i) '
'ELSE e( turnRight e) move m)',
(('pickMarker', ('ifElse', 'noMarkersPresent'), 'putMarker',
('else', 'noMarkersPresent'), 'turnRight',
('end-ifElse', 'noMarkersPresent'), 'move'),
((3, 3), (4, 8), (9, 9), (10, 12), (13, 13), (14, 14), (15, 15)))),
])
def testLinearizeSimple(self, tokens, linearized):
assert self.linearize_to_tokens(tokens) == linearized
def _goal_reached_systematic(self, goal):
goal_atree = refine_env.AnnotatedTree(code=goal)
queue = collections.deque([(('DEF', 'run', 'm(', 'm)'), None)])
closed = set()
goal_reached = False
while queue:
current, prev = queue.popleft()
if current in closed:
continue
closed.add(current)
current_atree = refine_env.AnnotatedTree(code=current)
assert refine_env.is_subseq(current_atree.linearized[0],
goal_atree.linearized[0])
actions = set(
refine_env.ComputeAddOps.run(current_atree, goal_atree))
#bad_actions = set(
# a
# for a in refine_env.MutationActionSpace(code=current)
# .enumerate_additive_actions() if a not in actions)
for action in actions:
mutation_space = refine_env.MutationActionSpace(
atree=copy.deepcopy(current_atree))
assert mutation_space.contains(action)
mutation_space.apply(action)
new_code = mutation_space.atree.code
if new_code not in closed:
queue.append((new_code, current))
assert refine_env.is_subseq(current_atree.linearized[0],
mutation_space.atree.linearized[0])
if not actions:
assert current == goal
goal_reached = True
continue
# Check that every other action is invalid
#for bad_action in bad_actions:
# mutation_space = refine_env.MutationActionSpace(
# tree=copy.deepcopy(current_tree))
# mutation_space.apply(bad_action)
# new_code_linearized, _ = refine_env.ComputeAddOps.linearize(mutation_space.tree)
# assert not refine_env.is_subseq(new_code_linearized,
# goal_atree.linearized[0]))
assert goal_reached
@pytest.mark.parametrize('code', [
'''DEF run m( m)''',
'''DEF run m( move turnLeft m)''',
'''DEF run m( move move move move move m)''',
'''DEF run m(
REPEAT R=10 r( putMarker move r)
putMarker turnLeft m)''',
'''DEF run m(
IF c( markersPresent c) i(
move
turnLeft
i)
IF c( leftIsClear c) i(
move
turnLeft
i)
m)''',
'''DEF run m(
IF c( markersPresent c) i(
move
turnLeft
i)
IF c( markersPresent c) i(
move
turnLeft
i)
m)''',
'''DEF run m(
IF c( leftIsClear c) i(
IF c( leftIsClear c) i(
move
i)
i)
m)''',
'''DEF run m(
IF c( leftIsClear c) i(
turnLeft
IF c( leftIsClear c) i(
turnLeft
i)
turnLeft
i)
m)''',
'''DEF run m(
REPEAT R=5 r(
move
IFELSE c( markersPresent c) i(
move
i) ELSE e(
move
IFELSE c( markersPresent c) i(
move
i) ELSE e(
move
e)
move
e)
r)
move
m)''',
])
def testRun(self, code):
self._goal_reached_systematic(tuple(code.split()))
@pytest.mark.parametrize(
'code',
open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'testdata', 'short_val_code.txt')).readlines()[:300])
def testRunValCode(self, code):
self._goal_reached_systematic(tuple(code.split()))
def batch_processor(self, for_eval):
return lambda x: x
if __name__ == '__main__':
parser = arguments.get_arg_parser('Evaluating Text2Code', 'eval')
parser.add_argument('--mutate-n', type=int, required=True)
parser.add_argument('--trials', type=int, required=True)
args = parser.parse_args()
args.dataset = 'karel'
args.karel_trace_enc = 'none'
args.load_sync = True
ds = dataset.get_karel_eval_final_dataset(args, DummyModel())
the_executor = executor.KarelExecutor(action_limit=1000)
parser = parser_for_synthesis.KarelForSynthesisParser(build_tree=True)
total, correct, match = 0, 0, 0
rng = np.random.RandomState(112358)
for batch in tqdm.tqdm(ds):
for item in batch:
tried_code = set()
orig_tree = parser.parse(item.ref_example.code_sequence)
match_found, correct_found = False, False
total += 1
for t in range(args.trials):
mut_success = False
for i in range(1000):
tree = mutation.mutate_n(orig_tree,
args.mutate_n,
rng=rng,
def setUp(self):
self.tree = parser_for_synthesis.KarelForSynthesisParser(
build_tree=True).parse(
('DEF run m( REPEAT R=5 r( turnLeft IFELSE c( '
'markersPresent c) i( turnRight i) ELSE e( move e) r) '
'pickMarker m)').split())
class AnnotatedTree(object):
parser = parser_for_synthesis.KarelForSynthesisParser(build_tree=True)
def __init__(self, tree=None, code=None):
assert (tree is None) ^ (code is None)
if tree:
self.tree = tree
else:
self.tree = self.parser.parse(code)
def notify_mutation(self):
self.__dict__.pop('code', None)
self.__dict__.pop('index', None)
self.__dict__.pop('pre_insert_locs', None)
self.__dict__.pop('post_insert_locs', None)
self.__dict__.pop('remove_action_locs', None)
self.__dict__.pop('replace_action_locs', None)
self.__dict__.pop('unwrap_block_locs', None)
self.__dict__.pop('linearized', None)
# Tree -> code -> tree so that self.tree has correct spans
self.tree = self.parser.parse(self.code)
@cached_property
def code(self):
return parser_for_synthesis.tree_to_tokens(self.tree)
@cached_property
def index(self):
return mutation.TreeIndex(self.tree)
# points to token before insert point
# v v v
# DEF run m( move IF c( markersPresent c) i( turnLeft i) m)
# 0 1 2
@cached_property
def pre_insert_locs(self):
index = self.index
pre_insert_locs = {}
for body in index.all_bodies:
for i in range(len(body.elems)):
pre_insert_locs[body.elems[i]['span'][1]] = (body.elems, i + 1)
if body.elems:
pre_insert_locs[body.elems[0]['span'][0] - 1] = (body.elems, 0)
else:
if body.type in ('run', 'if', 'while', 'repeat'):
pre_insert_locs[body.node['span'][1] - 1] = (body.elems, 0)
elif body.type == 'ifElse-if':
pre_insert_locs[body.node['ifSpan'][0]] = (body.elems, 0)
elif body.type == 'ifElse-else':
pre_insert_locs[body.node['elseSpan'][0]] = (body.elems, 0)
else:
raise ValueError(body.type)
return pre_insert_locs
# points to token after insert point
# v v v
# DEF run m( move IF c( markersPresent c) i( turnLeft i) m)
# 0 1 2
# v v v
# DEF run m( move move m)
# 0 1 2
@cached_property
def post_insert_locs(self):
index = self.index
post_insert_locs = {}
for body in index.all_bodies:
for i in range(len(body.elems)):
post_insert_locs[body.elems[i]['span'][0]] = (body.elems, i)
if body.elems:
post_insert_locs[body.elems[-1]['span'][1] +
1] = (body.elems, len(body.elems))
else:
if body.type in ('run', 'if', 'while', 'repeat'):
post_insert_locs[body.node['span'][1]] = (body.elems,
len(body.elems))
elif body.type == 'ifElse-if':
post_insert_locs[body.node['ifSpan'][1]] = (
body.elems, len(body.elems))
elif body.type == 'ifElse-else':
post_insert_locs[body.node['elseSpan'][1]] = (
body.elems, len(body.elems))
else:
raise ValueError(body.type)
return post_insert_locs
# ADD_ACTION
@property
def add_action_locs(self):
#return self.pre_insert_locs
return self.post_insert_locs
# REMOVE_ACTION
@cached_property
def remove_action_locs(self):
return {
body_elems[i]['span'][0]: (body_elems, i)
for body_elems, i in self.index.remove_locs
}
# REPLACE_ACTION
@cached_property
def replace_action_locs(self):
return {
body_elems[i]['span'][0]: (body_elems, i)
for body_elems, i in self.index.action_locs
}
# UNWRAP_BLOCK
@cached_property
def unwrap_block_locs(self):
return {
body_elems[i]['span'][0]: (body_elems, i)
for body_elems, i in self.index.unwrappables
}
# WRAP_BLOCK
# block start: pre_insert_locs
# block end: post_insert_locs
# WRAP_IFELSE
# if block start: pre_insert_locs
# if block end/else block start: post_insert_locs
# else block end: post_insert_locs
# REPLACE_COND
# Not yet implemented
# SWITCH_IF_WHILE
# Not yet implemented
@cached_property
def linearized(self):
return ComputeAddOps.linearize(self.tree)