def run():
asdl_text = open('asdl/lang/py/py_asdl.txt').read()
grammar = ASDLGrammar.from_text(asdl_text)
annot_file = 'data/django/all.anno'
code_file = 'data/django/all.code'
transition_system = PythonTransitionSystem(grammar)
for idx, (src_query, tgt_code) in enumerate(zip(open(annot_file), open(code_file))):
src_query = src_query.strip()
tgt_code = tgt_code.strip()
query_tokens, tgt_canonical_code, str_map = Django.canonicalize_example(src_query, tgt_code)
python_ast = ast.parse(tgt_canonical_code).body[0]
gold_source = astor.to_source(python_ast)
tgt_ast = python_ast_to_asdl_ast(python_ast, grammar)
tgt_actions = transition_system.get_actions(tgt_ast)
# sanity check
hyp = Hypothesis()
hyp2 = Hypothesis()
for action in tgt_actions:
assert action.__class__ in transition_system.get_valid_continuation_types(hyp)
if isinstance(action, ApplyRuleAction):
assert action.production in transition_system.get_valid_continuating_productions(hyp)
hyp = hyp.clone_and_apply_action(action)
hyp2.apply_action(action)
src_from_hyp = astor.to_source(asdl_ast_to_python_ast(hyp.tree, grammar))
assert src_from_hyp == gold_source
assert hyp.tree == hyp2.tree and hyp.tree is not hyp2.tree
print(idx)
def run():
asdl_text = open('asdl/lang/py/py_asdl.txt').read()
grammar = ASDLGrammar.from_text(asdl_text)
annot_file = 'data/django/all.anno'
code_file = 'data/django/all.code'
transition_system = PythonTransitionSystem(grammar)
for idx, (src_query, tgt_code) in enumerate(zip(open(annot_file), open(code_file))):
src_query = src_query.strip()
tgt_code = tgt_code.strip()
query_tokens, tgt_canonical_code, str_map = Django.canonicalize_example(src_query, tgt_code)
python_ast = ast.parse(tgt_canonical_code).body[0]
gold_source = astor.to_source(python_ast)
tgt_ast = python_ast_to_asdl_ast(python_ast, grammar)
tgt_actions = transition_system.get_actions(tgt_ast)
# sanity check
hyp = Hypothesis()
hyp2 = Hypothesis()
for action in tgt_actions:
assert action.__class__ in transition_system.get_valid_continuation_types(hyp)
if isinstance(action, ApplyRuleAction):
assert action.production in transition_system.get_valid_continuating_productions(hyp)
hyp = hyp.clone_and_apply_action(action)
hyp2.apply_action(action)
src_from_hyp = astor.to_source(asdl_ast_to_python_ast(hyp.tree, grammar))
assert src_from_hyp == gold_source
assert hyp.tree == hyp2.tree and hyp.tree is not hyp2.tree
print(idx)
def post_process_hypothesis(self, hyp, meta_info, utterance=None):
"""traverse the AST and replace slot ids with original strings"""
slot2str_map = {v: k for k, v in meta_info.items()}
hyp_ast = asdl_ast_to_python_ast(hyp.tree,
self.transition_system.grammar)
replace_string_ast_nodes(hyp_ast, slot2str_map)
hyp.code = astor.to_source(hyp_ast).strip()
def ast_to_surface_code(self, asdl_ast):
py_ast = asdl_ast_to_python_ast(asdl_ast, self.grammar)
code = astor.to_source(py_ast).strip()
if code.endswith(':'):
code += ' pass'
# first make sure the hypothesis code is parsable by `ast`
# sometimes, the parser generates syntactically invalid surface code:
# e.g., slot_0.1()
# ast.parse(code)
return code
def parse_django_dataset(annot_file,
code_file,
asdl_file_path,
max_query_len=70,
vocab_freq_cutoff=10):
asdl_text = open(asdl_file_path).read()
grammar = ASDLGrammar.from_text(asdl_text)
transition_system = PythonTransitionSystem(grammar)
loaded_examples = []
from components.vocab import Vocab, VocabEntry
from components.dataset import Example
for idx, (src_query,
tgt_code) in enumerate(zip(open(annot_file),
open(code_file))):
src_query = src_query.strip()
tgt_code = tgt_code.strip()
src_query_tokens, tgt_canonical_code, str_map = Django.canonicalize_example(
src_query, tgt_code)
python_ast = ast.parse(tgt_canonical_code).body[0]
gold_source = astor.to_source(python_ast).strip()
tgt_ast = python_ast_to_asdl_ast(python_ast, grammar)
tgt_actions = transition_system.get_actions(tgt_ast)
# print('+' * 60)
# print('Example: %d' % idx)
# print('Source: %s' % ' '.join(src_query_tokens))
# if str_map:
# print('Original String Map:')
# for str_literal, str_repr in str_map.items():
# print('\t%s: %s' % (str_literal, str_repr))
# print('Code:\n%s' % gold_source)
# print('Actions:')
# sanity check
try:
hyp = Hypothesis()
for t, action in enumerate(tgt_actions):
# assert action.__class__ in transition_system.get_valid_continuation_types(hyp)
# if isinstance(action, ApplyRuleAction):
# assert action.production in transition_system.get_valid_continuating_productions(hyp)
p_t = -1
f_t = None
if hyp.frontier_node:
p_t = hyp.frontier_node.created_time
f_t = hyp.frontier_field.field.__repr__(plain=True)
# print('\t[%d] %s, frontier field: %s, parent: %d' % (t, action, f_t, p_t))
hyp = hyp.clone_and_apply_action(action)
assert hyp.frontier_node is None and hyp.frontier_field is None
src_from_hyp = astor.to_source(
asdl_ast_to_python_ast(hyp.tree, grammar)).strip()
assert src_from_hyp == gold_source
# print('+' * 60)
except:
continue
loaded_examples.append({
'src_query_tokens': src_query_tokens,
'tgt_canonical_code': gold_source,
'tgt_ast': tgt_ast,
'tgt_actions': tgt_actions,
'raw_code': tgt_code,
'str_map': str_map
})
# print('first pass, processed %d' % idx, file=sys.stderr)
train_examples = []
dev_examples = []
test_examples = []
action_len = []
for idx, e in enumerate(loaded_examples):
src_query_tokens = e['src_query_tokens'][:max_query_len]
tgt_actions = e['tgt_actions']
tgt_action_infos = get_action_infos(src_query_tokens, tgt_actions)
example = Example(idx=idx,
src_sent=src_query_tokens,
tgt_actions=tgt_action_infos,
tgt_code=e['tgt_canonical_code'],
tgt_ast=e['tgt_ast'],
meta={
'raw_code': e['raw_code'],
'str_map': e['str_map']
})
# print('second pass, processed %d' % idx, file=sys.stderr)
action_len.append(len(tgt_action_infos))
# train, valid, test split
if 0 <= idx < 16000:
train_examples.append(example)
elif 16000 <= idx < 17000:
dev_examples.append(example)
else:
test_examples.append(example)
print('Max action len: %d' % max(action_len), file=sys.stderr)
print('Avg action len: %d' % np.average(action_len), file=sys.stderr)
print('Actions larger than 100: %d' %
len(list(filter(lambda x: x > 100, action_len))),
file=sys.stderr)
src_vocab = VocabEntry.from_corpus(
[e.src_sent for e in train_examples],
size=5000,
freq_cutoff=vocab_freq_cutoff)
primitive_tokens = [
map(
lambda a: a.action.token,
filter(lambda a: isinstance(a.action, GenTokenAction),
e.tgt_actions)) for e in train_examples
]
primitive_vocab = VocabEntry.from_corpus(primitive_tokens,
size=5000,
freq_cutoff=vocab_freq_cutoff)
assert '_STR:0_' in primitive_vocab
# generate vocabulary for the code tokens!
code_tokens = [
tokenize_code(e.tgt_code, mode='decoder') for e in train_examples
]
code_vocab = VocabEntry.from_corpus(code_tokens,
size=5000,
freq_cutoff=vocab_freq_cutoff)
vocab = Vocab(source=src_vocab,
primitive=primitive_vocab,
code=code_vocab)
print('generated vocabulary %s' % repr(vocab), file=sys.stderr)
return (train_examples, dev_examples, test_examples), vocab
def ast_to_surface_code(self, asdl_ast):
py_ast = asdl_ast_to_python_ast(asdl_ast, self.grammar)
code = astor.to_source(py_ast).strip()
return code
def parse_django_dataset(annot_file, code_file, asdl_file_path, max_query_len=70, vocab_freq_cutoff=10):
asdl_text = open(asdl_file_path).read()
grammar = ASDLGrammar.from_text(asdl_text)
transition_system = PythonTransitionSystem(grammar)
loaded_examples = []
from components.vocab import Vocab, VocabEntry
from components.dataset import Example
for idx, (src_query, tgt_code) in enumerate(zip(open(annot_file), open(code_file))):
src_query = src_query.strip()
tgt_code = tgt_code.strip()
src_query_tokens, tgt_canonical_code, str_map = Django.canonicalize_example(src_query, tgt_code)
python_ast = ast.parse(tgt_canonical_code).body[0]
gold_source = astor.to_source(python_ast).strip()
tgt_ast = python_ast_to_asdl_ast(python_ast, grammar)
tgt_actions = transition_system.get_actions(tgt_ast)
print('+' * 60)
print('Example: %d' % idx)
print('Source: %s' % ' '.join(src_query_tokens))
if str_map:
print('Original String Map:')
for str_literal, str_repr in str_map.items():
print('\t%s: %s' % (str_literal, str_repr))
print('Code:\n%s' % gold_source)
print('Actions:')
# sanity check
hyp = Hypothesis()
for t, action in enumerate(tgt_actions):
assert action.__class__ in transition_system.get_valid_continuation_types(hyp)
if isinstance(action, ApplyRuleAction):
assert action.production in transition_system.get_valid_continuating_productions(hyp)
p_t = -1
f_t = None
if hyp.frontier_node:
p_t = hyp.frontier_node.created_time
f_t = hyp.frontier_field.field.__repr__(plain=True)
print('\t[%d] %s, frontier field: %s, parent: %d' % (t, action, f_t, p_t))
hyp = hyp.clone_and_apply_action(action)
assert hyp.frontier_node is None and hyp.frontier_field is None
src_from_hyp = astor.to_source(asdl_ast_to_python_ast(hyp.tree, grammar)).strip()
assert src_from_hyp == gold_source
print('+' * 60)
loaded_examples.append({'src_query_tokens': src_query_tokens,
'tgt_canonical_code': gold_source,
'tgt_ast': tgt_ast,
'tgt_actions': tgt_actions,
'raw_code': tgt_code, 'str_map': str_map})
# print('first pass, processed %d' % idx, file=sys.stderr)
train_examples = []
dev_examples = []
test_examples = []
action_len = []
for idx, e in enumerate(loaded_examples):
src_query_tokens = e['src_query_tokens'][:max_query_len]
tgt_actions = e['tgt_actions']
tgt_action_infos = get_action_infos(src_query_tokens, tgt_actions)
example = Example(idx=idx,
src_sent=src_query_tokens,
tgt_actions=tgt_action_infos,
tgt_code=e['tgt_canonical_code'],
tgt_ast=e['tgt_ast'],
meta={'raw_code': e['raw_code'], 'str_map': e['str_map']})
# print('second pass, processed %d' % idx, file=sys.stderr)
action_len.append(len(tgt_action_infos))
# train, valid, test split
if 0 <= idx < 16000:
train_examples.append(example)
elif 16000 <= idx < 17000:
dev_examples.append(example)
else:
test_examples.append(example)
print('Max action len: %d' % max(action_len), file=sys.stderr)
print('Avg action len: %d' % np.average(action_len), file=sys.stderr)
print('Actions larger than 100: %d' % len(list(filter(lambda x: x > 100, action_len))), file=sys.stderr)
src_vocab = VocabEntry.from_corpus([e.src_sent for e in train_examples], size=5000, freq_cutoff=vocab_freq_cutoff)
primitive_tokens = [map(lambda a: a.action.token,
filter(lambda a: isinstance(a.action, GenTokenAction), e.tgt_actions))
for e in train_examples]
primitive_vocab = VocabEntry.from_corpus(primitive_tokens, size=5000, freq_cutoff=vocab_freq_cutoff)
assert '_STR:0_' in primitive_vocab
# generate vocabulary for the code tokens!
code_tokens = [tokenize_code(e.tgt_code, mode='decoder') for e in train_examples]
code_vocab = VocabEntry.from_corpus(code_tokens, size=5000, freq_cutoff=vocab_freq_cutoff)
vocab = Vocab(source=src_vocab, primitive=primitive_vocab, code=code_vocab)
print('generated vocabulary %s' % repr(vocab), file=sys.stderr)
return (train_examples, dev_examples, test_examples), vocab
def ast_to_surface_code(self, asdl_ast):
py_ast = asdl_ast_to_python_ast(asdl_ast, self.grammar)
code = astor.to_source(py_ast).strip()
return code
def post_process_hypothesis(self, hyp, meta_info, utterance=None):
"""traverse the AST and replace slot ids with original strings"""
hyp_ast = asdl_ast_to_python_ast(hyp.tree, self.transition_system.grammar)
code_from_hyp = astor.to_source(hyp_ast).strip()
hyp.code = decanonicalize_code(code_from_hyp, meta_info)
