def __init__(self, n_dist, rng_fixed, add_trace, probs=None): self.n_dist = n_dist / np.sum(n_dist) self.rng_fixed = rng_fixed self.add_trace = add_trace self.probs = probs self.rng = np.random.RandomState() self.parser = parser_for_synthesis.KarelForSynthesisParser( build_tree=True) self.executor = executor.KarelExecutor(action_limit=250)
def __init__(self, code): self.parser = parser_for_synthesis.KarelForSynthesisParser( build_tree=True) self.executor = executor.KarelExecutor() self.code = code tree = self.parser.parse(code) # Statement coverage: actions self.stmt_coverage = {span: 0 for span in self.parser.action_spans} # Branch coverage: if, ifelse, while self.branch_coverage = {(span, cond_value): 0 for span in self.parser.cond_block_spans for cond_value in (True, False)}
def eval_traces(trace_model, eval_dataset, beam_size): parser = parser_for_synthesis.KarelForSynthesisParser() tracer = karel_trace_model.KarelTracer(parser.karel) def get_trace(tracer, prog, input_grid): tracer.reset(grid=input_grid) prog() return tracer.actions def check_trace_passes(kr, tracer, input_grid, output_grid, candidates, gold_trace): passes = collections.defaultdict(bool) exact_match = collections.defaultdict(bool) for i, cand in enumerate(candidates): # print(cand, gold_trace) if cand == gold_trace: exact_match[i] = True tracer.reset(grid=input_grid) for action in cand: success = getattr(kr, action, lambda: False)() if np.all(tracer.full_grid == output_grid): passes[i] = True break return passes, exact_match trace_model.args.max_beam_trees = beam_size report = [] iterator = tqdm.tqdm(eval_dataset, dynamic_ncols=True) for batch in iterator: res = trace_model.inference(batch) idx = 0 input_grids = batch.input_grids.data.numpy().astype(bool) output_grids = batch.output_grids.data.numpy().astype(bool) for ex in batch.orig_examples: info = [] prog = parser.parse(ex.code_sequence) for i, test in enumerate(ex.input_tests + ex.tests): candidates = res[idx].info['candidates'] gold_trace = get_trace(tracer, prog, input_grids[idx]) passes, exact_match = check_trace_passes( trace_model.kr, trace_model.tracer, input_grids[idx], output_grids[idx], candidates, gold_trace) info.append((passes, exact_match)) idx += 1 report.append(info) assert idx == len(res) return report
def __init__(self, vocab, for_eval, args): self.vocab = vocab self.for_eval = for_eval self.args = args self.karel_parser = parser_for_synthesis.KarelForSynthesisParser() self.tracer = KarelTracer(self.karel_parser.karel)
def load_model(model_dir, model_type, step=None): args = saver.ArgsDict(model_dir=model_dir, model_type=model_type, step=step) saver.restore_args(args) arguments.backport_default_args(args) dataset.set_vocab(args) m = models.get_model(args) eval_dataset = dataset.get_eval_dataset(args, m) m.model.eval() the_executor = executor.get_executor(args)() return m, eval_dataset, the_executor parser = parser_for_synthesis.KarelForSynthesisParser() tracer = karel_trace_model.KarelTracer(parser.karel) def get_traces(code, tests): result = [] prog = parser.parse(code) for test in tests: tracer.reset(indices=test['input']) prog() result.append(tracer.actions) return result def check_correct(code, tests, e): res = executor.evaluate_code(code, None, tests, e)
class TestComputeAddOps(object): parser = parser_for_synthesis.KarelForSynthesisParser(build_tree=True) def linearize_to_tokens(self, code): tokens, orig_spans = refine_env.ComputeAddOps.linearize( self.parser.parse(code)) return tuple(refine_env.ComputeAddOps.idx_to_token[i] for i in tokens), orig_spans @pytest.mark.parametrize('tokens,linearized', [ ('DEF run m( move putMarker m)', (('move', 'putMarker'), ((3, 3), (4, 4)))), ('DEF run m( move ' 'IF c( not c( frontIsClear c) c) i( putMarker i) ' 'turnLeft m)', (('move', ('if', ('not', 'frontIsClear')), 'putMarker', ('end-if', ('not', 'frontIsClear')), 'turnLeft'), ((3, 3), (4, 11), (12, 12), (13, 13), (14, 14)))), ('DEF run m( move REPEAT R=5 r( putMarker r) turnLeft m)', (('move', ('repeat', 5), 'putMarker', ('end-repeat', 5), 'turnLeft'), ((3, 3), (4, 6), (7, 7), (8, 8), (9, 9)))), ('DEF run m( pickMarker ' 'IFELSE c( noMarkersPresent c) i( putMarker i) ' 'ELSE e( turnRight e) move m)', (('pickMarker', ('ifElse', 'noMarkersPresent'), 'putMarker', ('else', 'noMarkersPresent'), 'turnRight', ('end-ifElse', 'noMarkersPresent'), 'move'), ((3, 3), (4, 8), (9, 9), (10, 12), (13, 13), (14, 14), (15, 15)))), ]) def testLinearizeSimple(self, tokens, linearized): assert self.linearize_to_tokens(tokens) == linearized def _goal_reached_systematic(self, goal): goal_atree = refine_env.AnnotatedTree(code=goal) queue = collections.deque([(('DEF', 'run', 'm(', 'm)'), None)]) closed = set() goal_reached = False while queue: current, prev = queue.popleft() if current in closed: continue closed.add(current) current_atree = refine_env.AnnotatedTree(code=current) assert refine_env.is_subseq(current_atree.linearized[0], goal_atree.linearized[0]) actions = set( refine_env.ComputeAddOps.run(current_atree, goal_atree)) #bad_actions = set( # a # for a in refine_env.MutationActionSpace(code=current) # .enumerate_additive_actions() if a not in actions) for action in actions: mutation_space = refine_env.MutationActionSpace( atree=copy.deepcopy(current_atree)) assert mutation_space.contains(action) mutation_space.apply(action) new_code = mutation_space.atree.code if new_code not in closed: queue.append((new_code, current)) assert refine_env.is_subseq(current_atree.linearized[0], mutation_space.atree.linearized[0]) if not actions: assert current == goal goal_reached = True continue # Check that every other action is invalid #for bad_action in bad_actions: # mutation_space = refine_env.MutationActionSpace( # tree=copy.deepcopy(current_tree)) # mutation_space.apply(bad_action) # new_code_linearized, _ = refine_env.ComputeAddOps.linearize(mutation_space.tree) # assert not refine_env.is_subseq(new_code_linearized, # goal_atree.linearized[0])) assert goal_reached @pytest.mark.parametrize('code', [ '''DEF run m( m)''', '''DEF run m( move turnLeft m)''', '''DEF run m( move move move move move m)''', '''DEF run m( REPEAT R=10 r( putMarker move r) putMarker turnLeft m)''', '''DEF run m( IF c( markersPresent c) i( move turnLeft i) IF c( leftIsClear c) i( move turnLeft i) m)''', '''DEF run m( IF c( markersPresent c) i( move turnLeft i) IF c( markersPresent c) i( move turnLeft i) m)''', '''DEF run m( IF c( leftIsClear c) i( IF c( leftIsClear c) i( move i) i) m)''', '''DEF run m( IF c( leftIsClear c) i( turnLeft IF c( leftIsClear c) i( turnLeft i) turnLeft i) m)''', '''DEF run m( REPEAT R=5 r( move IFELSE c( markersPresent c) i( move i) ELSE e( move IFELSE c( markersPresent c) i( move i) ELSE e( move e) move e) r) move m)''', ]) def testRun(self, code): self._goal_reached_systematic(tuple(code.split())) @pytest.mark.parametrize( 'code', open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'testdata', 'short_val_code.txt')).readlines()[:300]) def testRunValCode(self, code): self._goal_reached_systematic(tuple(code.split()))
def batch_processor(self, for_eval): return lambda x: x if __name__ == '__main__': parser = arguments.get_arg_parser('Evaluating Text2Code', 'eval') parser.add_argument('--mutate-n', type=int, required=True) parser.add_argument('--trials', type=int, required=True) args = parser.parse_args() args.dataset = 'karel' args.karel_trace_enc = 'none' args.load_sync = True ds = dataset.get_karel_eval_final_dataset(args, DummyModel()) the_executor = executor.KarelExecutor(action_limit=1000) parser = parser_for_synthesis.KarelForSynthesisParser(build_tree=True) total, correct, match = 0, 0, 0 rng = np.random.RandomState(112358) for batch in tqdm.tqdm(ds): for item in batch: tried_code = set() orig_tree = parser.parse(item.ref_example.code_sequence) match_found, correct_found = False, False total += 1 for t in range(args.trials): mut_success = False for i in range(1000): tree = mutation.mutate_n(orig_tree, args.mutate_n, rng=rng,
def setUp(self): self.tree = parser_for_synthesis.KarelForSynthesisParser( build_tree=True).parse( ('DEF run m( REPEAT R=5 r( turnLeft IFELSE c( ' 'markersPresent c) i( turnRight i) ELSE e( move e) r) ' 'pickMarker m)').split())
class AnnotatedTree(object): parser = parser_for_synthesis.KarelForSynthesisParser(build_tree=True) def __init__(self, tree=None, code=None): assert (tree is None) ^ (code is None) if tree: self.tree = tree else: self.tree = self.parser.parse(code) def notify_mutation(self): self.__dict__.pop('code', None) self.__dict__.pop('index', None) self.__dict__.pop('pre_insert_locs', None) self.__dict__.pop('post_insert_locs', None) self.__dict__.pop('remove_action_locs', None) self.__dict__.pop('replace_action_locs', None) self.__dict__.pop('unwrap_block_locs', None) self.__dict__.pop('linearized', None) # Tree -> code -> tree so that self.tree has correct spans self.tree = self.parser.parse(self.code) @cached_property def code(self): return parser_for_synthesis.tree_to_tokens(self.tree) @cached_property def index(self): return mutation.TreeIndex(self.tree) # points to token before insert point # v v v # DEF run m( move IF c( markersPresent c) i( turnLeft i) m) # 0 1 2 @cached_property def pre_insert_locs(self): index = self.index pre_insert_locs = {} for body in index.all_bodies: for i in range(len(body.elems)): pre_insert_locs[body.elems[i]['span'][1]] = (body.elems, i + 1) if body.elems: pre_insert_locs[body.elems[0]['span'][0] - 1] = (body.elems, 0) else: if body.type in ('run', 'if', 'while', 'repeat'): pre_insert_locs[body.node['span'][1] - 1] = (body.elems, 0) elif body.type == 'ifElse-if': pre_insert_locs[body.node['ifSpan'][0]] = (body.elems, 0) elif body.type == 'ifElse-else': pre_insert_locs[body.node['elseSpan'][0]] = (body.elems, 0) else: raise ValueError(body.type) return pre_insert_locs # points to token after insert point # v v v # DEF run m( move IF c( markersPresent c) i( turnLeft i) m) # 0 1 2 # v v v # DEF run m( move move m) # 0 1 2 @cached_property def post_insert_locs(self): index = self.index post_insert_locs = {} for body in index.all_bodies: for i in range(len(body.elems)): post_insert_locs[body.elems[i]['span'][0]] = (body.elems, i) if body.elems: post_insert_locs[body.elems[-1]['span'][1] + 1] = (body.elems, len(body.elems)) else: if body.type in ('run', 'if', 'while', 'repeat'): post_insert_locs[body.node['span'][1]] = (body.elems, len(body.elems)) elif body.type == 'ifElse-if': post_insert_locs[body.node['ifSpan'][1]] = ( body.elems, len(body.elems)) elif body.type == 'ifElse-else': post_insert_locs[body.node['elseSpan'][1]] = ( body.elems, len(body.elems)) else: raise ValueError(body.type) return post_insert_locs # ADD_ACTION @property def add_action_locs(self): #return self.pre_insert_locs return self.post_insert_locs # REMOVE_ACTION @cached_property def remove_action_locs(self): return { body_elems[i]['span'][0]: (body_elems, i) for body_elems, i in self.index.remove_locs } # REPLACE_ACTION @cached_property def replace_action_locs(self): return { body_elems[i]['span'][0]: (body_elems, i) for body_elems, i in self.index.action_locs } # UNWRAP_BLOCK @cached_property def unwrap_block_locs(self): return { body_elems[i]['span'][0]: (body_elems, i) for body_elems, i in self.index.unwrappables } # WRAP_BLOCK # block start: pre_insert_locs # block end: post_insert_locs # WRAP_IFELSE # if block start: pre_insert_locs # if block end/else block start: post_insert_locs # else block end: post_insert_locs # REPLACE_COND # Not yet implemented # SWITCH_IF_WHILE # Not yet implemented @cached_property def linearized(self): return ComputeAddOps.linearize(self.tree)