def run(): asdl_text = open('asdl/lang/py/py_asdl.txt').read() grammar = ASDLGrammar.from_text(asdl_text) annot_file = 'data/django/all.anno' code_file = 'data/django/all.code' transition_system = PythonTransitionSystem(grammar) for idx, (src_query, tgt_code) in enumerate(zip(open(annot_file), open(code_file))): src_query = src_query.strip() tgt_code = tgt_code.strip() query_tokens, tgt_canonical_code, str_map = Django.canonicalize_example(src_query, tgt_code) python_ast = ast.parse(tgt_canonical_code).body[0] gold_source = astor.to_source(python_ast) tgt_ast = python_ast_to_asdl_ast(python_ast, grammar) tgt_actions = transition_system.get_actions(tgt_ast) # sanity check hyp = Hypothesis() hyp2 = Hypothesis() for action in tgt_actions: assert action.__class__ in transition_system.get_valid_continuation_types(hyp) if isinstance(action, ApplyRuleAction): assert action.production in transition_system.get_valid_continuating_productions(hyp) hyp = hyp.clone_and_apply_action(action) hyp2.apply_action(action) src_from_hyp = astor.to_source(asdl_ast_to_python_ast(hyp.tree, grammar)) assert src_from_hyp == gold_source assert hyp.tree == hyp2.tree and hyp.tree is not hyp2.tree print(idx)
def run(): asdl_text = open('asdl/lang/py/py_asdl.txt').read() grammar = ASDLGrammar.from_text(asdl_text) annot_file = 'data/django/all.anno' code_file = 'data/django/all.code' transition_system = PythonTransitionSystem(grammar) for idx, (src_query, tgt_code) in enumerate(zip(open(annot_file), open(code_file))): src_query = src_query.strip() tgt_code = tgt_code.strip() query_tokens, tgt_canonical_code, str_map = Django.canonicalize_example(src_query, tgt_code) python_ast = ast.parse(tgt_canonical_code).body[0] gold_source = astor.to_source(python_ast) tgt_ast = python_ast_to_asdl_ast(python_ast, grammar) tgt_actions = transition_system.get_actions(tgt_ast) # sanity check hyp = Hypothesis() hyp2 = Hypothesis() for action in tgt_actions: assert action.__class__ in transition_system.get_valid_continuation_types(hyp) if isinstance(action, ApplyRuleAction): assert action.production in transition_system.get_valid_continuating_productions(hyp) hyp = hyp.clone_and_apply_action(action) hyp2.apply_action(action) src_from_hyp = astor.to_source(asdl_ast_to_python_ast(hyp.tree, grammar)) assert src_from_hyp == gold_source assert hyp.tree == hyp2.tree and hyp.tree is not hyp2.tree print(idx)
def post_process_hypothesis(self, hyp, meta_info, utterance=None): """traverse the AST and replace slot ids with original strings""" slot2str_map = {v: k for k, v in meta_info.items()} hyp_ast = asdl_ast_to_python_ast(hyp.tree, self.transition_system.grammar) replace_string_ast_nodes(hyp_ast, slot2str_map) hyp.code = astor.to_source(hyp_ast).strip()
def ast_to_surface_code(self, asdl_ast): py_ast = asdl_ast_to_python_ast(asdl_ast, self.grammar) code = astor.to_source(py_ast).strip() if code.endswith(':'): code += ' pass' # first make sure the hypothesis code is parsable by `ast` # sometimes, the parser generates syntactically invalid surface code: # e.g., slot_0.1() # ast.parse(code) return code
def parse_django_dataset(annot_file, code_file, asdl_file_path, max_query_len=70, vocab_freq_cutoff=10): asdl_text = open(asdl_file_path).read() grammar = ASDLGrammar.from_text(asdl_text) transition_system = PythonTransitionSystem(grammar) loaded_examples = [] from components.vocab import Vocab, VocabEntry from components.dataset import Example for idx, (src_query, tgt_code) in enumerate(zip(open(annot_file), open(code_file))): src_query = src_query.strip() tgt_code = tgt_code.strip() src_query_tokens, tgt_canonical_code, str_map = Django.canonicalize_example( src_query, tgt_code) python_ast = ast.parse(tgt_canonical_code).body[0] gold_source = astor.to_source(python_ast).strip() tgt_ast = python_ast_to_asdl_ast(python_ast, grammar) tgt_actions = transition_system.get_actions(tgt_ast) # print('+' * 60) # print('Example: %d' % idx) # print('Source: %s' % ' '.join(src_query_tokens)) # if str_map: # print('Original String Map:') # for str_literal, str_repr in str_map.items(): # print('\t%s: %s' % (str_literal, str_repr)) # print('Code:\n%s' % gold_source) # print('Actions:') # sanity check try: hyp = Hypothesis() for t, action in enumerate(tgt_actions): # assert action.__class__ in transition_system.get_valid_continuation_types(hyp) # if isinstance(action, ApplyRuleAction): # assert action.production in transition_system.get_valid_continuating_productions(hyp) p_t = -1 f_t = None if hyp.frontier_node: p_t = hyp.frontier_node.created_time f_t = hyp.frontier_field.field.__repr__(plain=True) # print('\t[%d] %s, frontier field: %s, parent: %d' % (t, action, f_t, p_t)) hyp = hyp.clone_and_apply_action(action) assert hyp.frontier_node is None and hyp.frontier_field is None src_from_hyp = astor.to_source( asdl_ast_to_python_ast(hyp.tree, grammar)).strip() assert src_from_hyp == gold_source # print('+' * 60) except: continue loaded_examples.append({ 'src_query_tokens': src_query_tokens, 'tgt_canonical_code': gold_source, 'tgt_ast': tgt_ast, 'tgt_actions': tgt_actions, 'raw_code': tgt_code, 'str_map': str_map }) # print('first pass, processed %d' % idx, file=sys.stderr) train_examples = [] dev_examples = [] test_examples = [] action_len = [] for idx, e in enumerate(loaded_examples): src_query_tokens = e['src_query_tokens'][:max_query_len] tgt_actions = e['tgt_actions'] tgt_action_infos = get_action_infos(src_query_tokens, tgt_actions) example = Example(idx=idx, src_sent=src_query_tokens, tgt_actions=tgt_action_infos, tgt_code=e['tgt_canonical_code'], tgt_ast=e['tgt_ast'], meta={ 'raw_code': e['raw_code'], 'str_map': e['str_map'] }) # print('second pass, processed %d' % idx, file=sys.stderr) action_len.append(len(tgt_action_infos)) # train, valid, test split if 0 <= idx < 16000: train_examples.append(example) elif 16000 <= idx < 17000: dev_examples.append(example) else: test_examples.append(example) print('Max action len: %d' % max(action_len), file=sys.stderr) print('Avg action len: %d' % np.average(action_len), file=sys.stderr) print('Actions larger than 100: %d' % len(list(filter(lambda x: x > 100, action_len))), file=sys.stderr) src_vocab = VocabEntry.from_corpus( [e.src_sent for e in train_examples], size=5000, freq_cutoff=vocab_freq_cutoff) primitive_tokens = [ map( lambda a: a.action.token, filter(lambda a: isinstance(a.action, GenTokenAction), e.tgt_actions)) for e in train_examples ] primitive_vocab = VocabEntry.from_corpus(primitive_tokens, size=5000, freq_cutoff=vocab_freq_cutoff) assert '_STR:0_' in primitive_vocab # generate vocabulary for the code tokens! code_tokens = [ tokenize_code(e.tgt_code, mode='decoder') for e in train_examples ] code_vocab = VocabEntry.from_corpus(code_tokens, size=5000, freq_cutoff=vocab_freq_cutoff) vocab = Vocab(source=src_vocab, primitive=primitive_vocab, code=code_vocab) print('generated vocabulary %s' % repr(vocab), file=sys.stderr) return (train_examples, dev_examples, test_examples), vocab
def ast_to_surface_code(self, asdl_ast): py_ast = asdl_ast_to_python_ast(asdl_ast, self.grammar) code = astor.to_source(py_ast).strip() return code
def parse_django_dataset(annot_file, code_file, asdl_file_path, max_query_len=70, vocab_freq_cutoff=10): asdl_text = open(asdl_file_path).read() grammar = ASDLGrammar.from_text(asdl_text) transition_system = PythonTransitionSystem(grammar) loaded_examples = [] from components.vocab import Vocab, VocabEntry from components.dataset import Example for idx, (src_query, tgt_code) in enumerate(zip(open(annot_file), open(code_file))): src_query = src_query.strip() tgt_code = tgt_code.strip() src_query_tokens, tgt_canonical_code, str_map = Django.canonicalize_example(src_query, tgt_code) python_ast = ast.parse(tgt_canonical_code).body[0] gold_source = astor.to_source(python_ast).strip() tgt_ast = python_ast_to_asdl_ast(python_ast, grammar) tgt_actions = transition_system.get_actions(tgt_ast) print('+' * 60) print('Example: %d' % idx) print('Source: %s' % ' '.join(src_query_tokens)) if str_map: print('Original String Map:') for str_literal, str_repr in str_map.items(): print('\t%s: %s' % (str_literal, str_repr)) print('Code:\n%s' % gold_source) print('Actions:') # sanity check hyp = Hypothesis() for t, action in enumerate(tgt_actions): assert action.__class__ in transition_system.get_valid_continuation_types(hyp) if isinstance(action, ApplyRuleAction): assert action.production in transition_system.get_valid_continuating_productions(hyp) p_t = -1 f_t = None if hyp.frontier_node: p_t = hyp.frontier_node.created_time f_t = hyp.frontier_field.field.__repr__(plain=True) print('\t[%d] %s, frontier field: %s, parent: %d' % (t, action, f_t, p_t)) hyp = hyp.clone_and_apply_action(action) assert hyp.frontier_node is None and hyp.frontier_field is None src_from_hyp = astor.to_source(asdl_ast_to_python_ast(hyp.tree, grammar)).strip() assert src_from_hyp == gold_source print('+' * 60) loaded_examples.append({'src_query_tokens': src_query_tokens, 'tgt_canonical_code': gold_source, 'tgt_ast': tgt_ast, 'tgt_actions': tgt_actions, 'raw_code': tgt_code, 'str_map': str_map}) # print('first pass, processed %d' % idx, file=sys.stderr) train_examples = [] dev_examples = [] test_examples = [] action_len = [] for idx, e in enumerate(loaded_examples): src_query_tokens = e['src_query_tokens'][:max_query_len] tgt_actions = e['tgt_actions'] tgt_action_infos = get_action_infos(src_query_tokens, tgt_actions) example = Example(idx=idx, src_sent=src_query_tokens, tgt_actions=tgt_action_infos, tgt_code=e['tgt_canonical_code'], tgt_ast=e['tgt_ast'], meta={'raw_code': e['raw_code'], 'str_map': e['str_map']}) # print('second pass, processed %d' % idx, file=sys.stderr) action_len.append(len(tgt_action_infos)) # train, valid, test split if 0 <= idx < 16000: train_examples.append(example) elif 16000 <= idx < 17000: dev_examples.append(example) else: test_examples.append(example) print('Max action len: %d' % max(action_len), file=sys.stderr) print('Avg action len: %d' % np.average(action_len), file=sys.stderr) print('Actions larger than 100: %d' % len(list(filter(lambda x: x > 100, action_len))), file=sys.stderr) src_vocab = VocabEntry.from_corpus([e.src_sent for e in train_examples], size=5000, freq_cutoff=vocab_freq_cutoff) primitive_tokens = [map(lambda a: a.action.token, filter(lambda a: isinstance(a.action, GenTokenAction), e.tgt_actions)) for e in train_examples] primitive_vocab = VocabEntry.from_corpus(primitive_tokens, size=5000, freq_cutoff=vocab_freq_cutoff) assert '_STR:0_' in primitive_vocab # generate vocabulary for the code tokens! code_tokens = [tokenize_code(e.tgt_code, mode='decoder') for e in train_examples] code_vocab = VocabEntry.from_corpus(code_tokens, size=5000, freq_cutoff=vocab_freq_cutoff) vocab = Vocab(source=src_vocab, primitive=primitive_vocab, code=code_vocab) print('generated vocabulary %s' % repr(vocab), file=sys.stderr) return (train_examples, dev_examples, test_examples), vocab
def ast_to_surface_code(self, asdl_ast): py_ast = asdl_ast_to_python_ast(asdl_ast, self.grammar) code = astor.to_source(py_ast).strip() return code
def post_process_hypothesis(self, hyp, meta_info, utterance=None): """traverse the AST and replace slot ids with original strings""" hyp_ast = asdl_ast_to_python_ast(hyp.tree, self.transition_system.grammar) code_from_hyp = astor.to_source(hyp_ast).strip() hyp.code = decanonicalize_code(code_from_hyp, meta_info)