def decode_python_dataset(model, dataset, verbose=True):
from lang.py.parse import decode_tree_to_python_ast
# if verbose:
# logging.info('decoding [%s] set, num. examples: %d', dataset.name, dataset.count)
decode_results = []
cum_num = 0
for example in dataset.examples:
# import pdb;pdb.set_trace()
cand_list = model.decode(example, dataset.grammar, dataset.terminal_vocab,
beam_size=config.beam_size, max_time_step=config.decode_max_time_step)
exg_decode_results = []
for cid, cand in enumerate(cand_list[:10]):
try:
ast_tree = decode_tree_to_python_ast(cand.tree)
code = astor.to_source(ast_tree)
exg_decode_results.append((cid, cand, ast_tree, code))
except:
if verbose:
print "Exception in converting tree to code:"
print '-' * 60
print 'raw_id: %d, beam pos: %d' % (example.raw_id, cid)
traceback.print_exc(file=sys.stdout)
print '-' * 60
cum_num += 1
if cum_num % 50 == 0 and verbose:
print '%d examples so far ...' % cum_num
decode_results.append(exg_decode_results)
return decode_results
def decode_python_dataset(model, dataset, verbose=True):
from lang.py.parse import decode_tree_to_python_ast
if verbose:
logging.info('decoding [%s] set, num. examples: %d', dataset.name, dataset.count)
decode_results = []
cum_num = 0
for example in dataset.examples:
cand_list = model.decode(example, dataset.grammar, dataset.terminal_vocab,
beam_size=config.beam_size, max_time_step=config.decode_max_time_step)
exg_decode_results = []
for cid, cand in enumerate(cand_list[:10]):
try:
ast_tree = decode_tree_to_python_ast(cand.tree)
code = astor.to_source(ast_tree)
exg_decode_results.append((cid, cand, ast_tree, code))
except:
if verbose:
print "Exception in converting tree to code:"
print '-' * 60
print 'raw_id: %d, beam pos: %d' % (example.raw_id, cid)
traceback.print_exc(file=sys.stdout)
print '-' * 60
cum_num += 1
if cum_num % 50 == 0 and verbose:
print '%d examples so far ...' % cum_num
decode_results.append(exg_decode_results)
return decode_results
def handle(clientsocket):
# while 1:
cmd = clientsocket.recv(MAX_LENGTH)
print cmd
if args.mode == 'dataset':
try:
example_id = int(cmd)
example = [
e for e in test_data.examples if e.raw_id == example_id
][0]
except:
print 'something went wrong ...'
# continue
elif args.mode == 'new':
# we play with new examples!
query, str_map = canonicalize_query(cmd)
vocab = train_data.annot_vocab
query_tokens = query.split(' ')
query_tokens_data = [query_to_data(query, vocab)]
example = namedtuple('example',
['query', 'data'])(query=query_tokens,
data=query_tokens_data)
if hasattr(example, 'parse_tree'):
print 'gold parse tree:'
print example.parse_tree
cand_list = model.decode(example,
train_data.grammar,
train_data.terminal_vocab,
beam_size=args.beam_size,
max_time_step=args.decode_max_time_step,
log=True)
has_grammar_error = any(
[c for c in cand_list if c.has_grammar_error])
print 'has_grammar_error: ', has_grammar_error
for cid, cand in enumerate(cand_list[:5]):
print '*' * 60
print 'cand #%d, score: %f' % (cid, cand.score)
try:
ast_tree = decode_tree_to_python_ast(cand.tree)
code = astor.to_source(ast_tree)
print 'code: ', code
print 'decode log: ', cand.log
except:
print "Exception in converting tree to code:"
print '-' * 60
print 'raw_id: %d, beam pos: %d' % (example.raw_id, cid)
traceback.print_exc(file=sys.stdout)
print '-' * 60
finally:
print '* parse tree *'
print cand.tree.__repr__()
print 'n_timestep: %d' % cand.n_timestep
print 'ast size: %d' % cand.tree.size
print '*' * 60
def decode_python_dataset(model,
train_data,
dev_data,
test_data,
dataset_type="test_data",
retrieval=False,
verbose=True):
dataset = eval(dataset_type)
from lang.py.parse import decode_tree_to_python_ast
if verbose:
logging.info('decoding [%s] set, num. examples: %d', dataset.name,
dataset.count)
decode_results = []
cum_num = 0
for example in dataset.examples:
print 'raw_id: %d' % example.raw_id
if retrieval:
ngrams = retrieve_translation_pieces(train_data, example)
cand_list = model.decode_with_retrieval(
example,
dataset.grammar,
dataset.terminal_vocab,
ngrams,
beam_size=config.beam_size,
max_time_step=config.decode_max_time_step,
log=True)
else:
cand_list = model.decode(example,
dataset.grammar,
dataset.terminal_vocab,
beam_size=config.beam_size,
max_time_step=config.decode_max_time_step)
exg_decode_results = []
for cid, cand in enumerate(cand_list[:10]):
try:
ast_tree = decode_tree_to_python_ast(cand.tree)
code = astor.to_source(ast_tree)
exg_decode_results.append((cid, cand, ast_tree, code))
except:
if verbose:
print "Exception in converting tree to code:"
print '-' * 60
print 'raw_id: %d, beam pos: %d' % (example.raw_id, cid)
traceback.print_exc(file=sys.stdout)
print '-' * 60
cum_num += 1
if cum_num % 50 == 0 and verbose:
print '%d examples so far ...' % cum_num
decode_results.append(exg_decode_results)
return decode_results
def decode_python_dataset(model, dataset, verbose=True):
from lang.py.parse import decode_tree_to_python_ast
if verbose:
logging.info('decoding [%s] set, num. examples: %d', dataset.name,
dataset.count)
decode_results = []
cum_num = 0
for example in dataset.examples:
cand_list = model.decode(example,
dataset.grammar,
dataset.terminal_vocab,
beam_size=config.beam_size,
max_time_step=config.decode_max_time_step)
print(len(cand_list))
exg_decode_results = []
#print(example.query)
for cid, cand in enumerate(cand_list[:10]):
#print('come in')
try:
code = ''
#print(cand.tree)
ast_tree = decode_tree_to_python_ast(cand.tree)
#print(ast_tree)
#print('------------------------')
for at in ast_tree.children:
code += astor.to_source(at)
#print(cid, cand)
exg_decode_results.append((cid, cand, ast_tree, code))
except:
if verbose:
print("Exception in converting tree to code:")
print('-' * 60)
print('raw_id: %d, beam pos: %d' % (example.raw_id, cid))
traceback.print_exc(file=sys.stdout)
print('-' * 60)
cum_num += 1
if cum_num % 50 == 0 and verbose:
print('%d examples so far ...' % cum_num)
decode_results.append(exg_decode_results)
return decode_results
def write_to_code_file(mode, data, path_to_load, path_to_export,
path_raw_code):
g = data.grammar
nt = {v: reverse_typename(k) for k, v in g.node_type_to_id.items()}
#print(nt,g.node_type_to_id)
v = data.terminal_vocab
raw = []
with open(path_raw_code, 'r') as f:
for line in f:
raw.append(line[:-1])
with open(path_to_load, 'r') as f:
l = json.load(f, encoding='utf8')
l_code = []
for i in range(len(l)):
# print(raw[i])
try:
t = ASTNode.from_dict(l[i], nt, v)
ast_tree = parse.decode_tree_to_python_ast(t)
code = astor.to_source(ast_tree)[:-1]
real_code = parse.de_canonicalize_code(code, raw[i])
if (mode == "hs"):
real_code = " ".join(parse.tokenize_code_adv(
real_code, True)).replace("\n", "#NEWLINE#").replace(
"#NEWLINE# ", "").replace("#INDENT# ", "")
real_code = " ".join(parse.tokenize_code_adv(real_code, False))
#print(real_code,raw[i])
l_code.append(real_code)
except:
print "Tree %d impossible to parse" % (i)
l_code.append("")
with open(path_to_export, 'w') as f:
for c in l_code:
f.write(c + "\n")
train_data.terminal_vocab,
beam_size=args.beam_size,
max_time_step=args.decode_max_time_step,
log=True)
has_grammar_error = any(
[c for c in cand_list if c.has_grammar_error])
print 'has_grammar_error: ', has_grammar_error
query_temp, str_m = canonicalize_query(cmd)
# print 'Query',query_temp
for cid, cand in enumerate(cand_list[:5]):
print '*' * 60
print 'cand #%d, score: %f' % (cid, cand.score)
try:
ast_tree = decode_tree_to_python_ast(cand.tree)
code = astor.to_source(ast_tree)
print 'code: ', code
print 'decode log: ', cand.log
except:
print "Exception in converting tree to code:"
print '-' * 60
print 'raw_id: %d, beam pos: %d' % (example.raw_id, cid)
traceback.print_exc(file=sys.stdout)
print '-' * 60
finally:
print '* parse tree *'
print cand.tree.__repr__()
print 'n_timestep: %d' % cand.n_timestep
print 'ast size: %d' % cand.tree.size
return cand_list
if __name__ == '__main__':
print 'run in interactive mode'
while True:
query = raw_input('input a query: ')
cand_list = decode_query(query)
# output top 5 candidates
for cid, cand in enumerate(cand_list[:5]):
print '*' * 60
print 'cand #%d, score: %f' % (cid, cand.score)
try:
ast_tree = decode_tree_to_python_ast(cand.tree)
code = astor.to_source(ast_tree)
print 'code: ', code
print 'decode log: ', cand.log
except:
print "Exception in converting tree to code:"
print '-' * 60
print 'raw_id: %d, beam pos: %d' % (example.raw_id, cid)
traceback.print_exc(file=sys.stdout)
print '-' * 60
finally:
print '* parse tree *'
print cand.tree.__repr__()
print 'n_timestep: %d' % cand.n_timestep
print 'ast size: %d' % cand.tree.size
print '*' * 60
def evaluate_seq2tree_sample_file(sample_file, id_file, dataset):
from lang.py.parse import tokenize_code, de_canonicalize_code
import ast, astor
import traceback
from lang.py.seq2tree_exp import seq2tree_repr_to_ast_tree, merge_broken_value_nodes
from lang.py.parse import decode_tree_to_python_ast
f_sample = open(sample_file)
line_id_to_raw_id = OrderedDict()
raw_id_to_eid = OrderedDict()
for i, line in enumerate(open(id_file)):
raw_id = int(line.strip())
line_id_to_raw_id[i] = raw_id
for eid in range(len(dataset.examples)):
raw_id_to_eid[dataset.examples[eid].raw_id] = eid
rare_word_map = defaultdict(dict)
if config.seq2tree_rareword_map:
logging.info('use rare word map')
for i, line in enumerate(open(config.seq2tree_rareword_map)):
line = line.strip()
if line:
for e in line.split(' '):
d = e.split(':', 1)
rare_word_map[i][int(d[0])] = d[1]
cum_bleu = 0.0
cum_acc = 0.0
sm = SmoothingFunction()
convert_error_num = 0
for i in range(len(line_id_to_raw_id)):
# print 'working on %d' % i
ref_repr = f_sample.readline().strip()
predict_repr = f_sample.readline().strip()
predict_repr = predict_repr.replace(
'<U>', 'str{}{unk}') # .replace('( )', '( str{}{unk} )')
f_sample.readline()
# if ' ( ) ' in ref_repr:
# print i, ref_repr
if i in rare_word_map:
for unk_id, w in rare_word_map[i].iteritems():
ref_repr = ref_repr.replace(' str{}{unk_%s} ' % unk_id,
' str{}{%s} ' % w)
predict_repr = predict_repr.replace(' str{}{unk_%s} ' % unk_id,
' str{}{%s} ' % w)
try:
parse_tree = seq2tree_repr_to_ast_tree(predict_repr)
merge_broken_value_nodes(parse_tree)
except:
print('error when converting:')
print(predict_repr)
convert_error_num += 1
continue
raw_id = line_id_to_raw_id[i]
eid = raw_id_to_eid[raw_id]
example = dataset.examples[eid]
ref_code = example.code
ref_ast_tree = ast.parse(ref_code).body[0]
refer_source = astor.to_source(ref_ast_tree).strip()
refer_tokens = tokenize_code(refer_source)
try:
ast_tree = decode_tree_to_python_ast(parse_tree)
code = astor.to_source(ast_tree).strip()
except:
print("Exception in converting tree to code:")
print('-' * 60)
print('line id: %d' % i)
traceback.print_exc(file=sys.stdout)
print('-' * 60)
convert_error_num += 1
continue
if config.data_type == 'django':
ref_code_for_bleu = example.meta_data['raw_code']
pred_code_for_bleu = de_canonicalize_code(
code, example.meta_data['raw_code'])
# convert canonicalized code to raw code
for literal, place_holder in example.meta_data[
'str_map'].iteritems():
pred_code_for_bleu = pred_code_for_bleu.replace(
'\'' + place_holder + '\'', literal)
elif config.data_type == 'hs':
ref_code_for_bleu = ref_code
pred_code_for_bleu = code
# we apply Ling Wang's trick when evaluating BLEU scores
refer_tokens_for_bleu = tokenize_for_bleu_eval(ref_code_for_bleu)
pred_tokens_for_bleu = tokenize_for_bleu_eval(pred_code_for_bleu)
predict_tokens = tokenize_code(code)
# if ref_repr == predict_repr:
if predict_tokens == refer_tokens:
cum_acc += 1
ngram_weights = [0.25] * min(4, len(refer_tokens_for_bleu))
bleu_score = sentence_bleu([refer_tokens_for_bleu],
pred_tokens_for_bleu,
weights=ngram_weights,
smoothing_function=sm.method3)
cum_bleu += bleu_score
cum_bleu /= len(line_id_to_raw_id)
cum_acc /= len(line_id_to_raw_id)
logging.info('nun. examples: %d', len(line_id_to_raw_id))
logging.info('num. errors when converting repr to tree: %d',
convert_error_num)
logging.info('ratio of grammatically incorrect trees: %f',
convert_error_num / float(len(line_id_to_raw_id)))
logging.info('sentence level bleu: %f', cum_bleu)
logging.info('accuracy: %f', cum_acc)
return reverse_typename(t[:-1])
else:
return vars(ast)[t]
if __name__ == '__main__':
flag = "hs"
path_to_load = "../data/exp/results/test_hs_10_iter.json"
if flag == "django":
train_data, dev_data, test_data = deserialize_from_file("../../django.cleaned.dataset.freq5.par_info.refact.space_only.bin")
elif flag == "hs":
train_data, dev_data, test_data = deserialize_from_file("../../hs.freq3.pre_suf.unary_closure.bin")
data = test_data
g = data.grammar
nt = {v:reverse_typename(k) for k,v in g.node_type_to_id.items()}
#print(nt,g.node_type_to_id)
v = data.terminal_vocab
results = []
with open(path_to_load,'r') as f:
l = json.load(f, encoding='utf8')
for i in range(len(l)):
t = ASTNode.from_dict(l[i], nt,v)
ast_tree = parse.decode_tree_to_python_ast(t)
results.append(ast_tree)
evaluate_decode_results(flag, test_data, results, verbose=True)
def evaluate_seq2tree_sample_file(sample_file, id_file, dataset):
from lang.py.parse import tokenize_code, de_canonicalize_code
import ast, astor
import traceback
from lang.py.seq2tree_exp import seq2tree_repr_to_ast_tree, merge_broken_value_nodes
from lang.py.parse import decode_tree_to_python_ast
f_sample = open(sample_file)
line_id_to_raw_id = OrderedDict()
raw_id_to_eid = OrderedDict()
for i, line in enumerate(open(id_file)):
raw_id = int(line.strip())
line_id_to_raw_id[i] = raw_id
for eid in range(len(dataset.examples)):
raw_id_to_eid[dataset.examples[eid].raw_id] = eid
rare_word_map = defaultdict(dict)
if config.seq2tree_rareword_map:
logging.info('use rare word map')
for i, line in enumerate(open(config.seq2tree_rareword_map)):
line = line.strip()
if line:
for e in line.split(' '):
d = e.split(':', 1)
rare_word_map[i][int(d[0])] = d[1]
cum_bleu = 0.0
cum_acc = 0.0
sm = SmoothingFunction()
convert_error_num = 0
for i in range(len(line_id_to_raw_id)):
# print 'working on %d' % i
ref_repr = f_sample.readline().strip()
predict_repr = f_sample.readline().strip()
predict_repr = predict_repr.replace('<U>', 'str{}{unk}') # .replace('( )', '( str{}{unk} )')
f_sample.readline()
# if ' ( ) ' in ref_repr:
# print i, ref_repr
if i in rare_word_map:
for unk_id, w in rare_word_map[i].iteritems():
ref_repr = ref_repr.replace(' str{}{unk_%s} ' % unk_id, ' str{}{%s} ' % w)
predict_repr = predict_repr.replace(' str{}{unk_%s} ' % unk_id, ' str{}{%s} ' % w)
try:
parse_tree = seq2tree_repr_to_ast_tree(predict_repr)
merge_broken_value_nodes(parse_tree)
except:
print 'error when converting:'
print predict_repr
convert_error_num += 1
continue
raw_id = line_id_to_raw_id[i]
eid = raw_id_to_eid[raw_id]
example = dataset.examples[eid]
ref_code = example.code
ref_ast_tree = ast.parse(ref_code).body[0]
refer_source = astor.to_source(ref_ast_tree).strip()
refer_tokens = tokenize_code(refer_source)
try:
ast_tree = decode_tree_to_python_ast(parse_tree)
code = astor.to_source(ast_tree).strip()
except:
print "Exception in converting tree to code:"
print '-' * 60
print 'line id: %d' % i
traceback.print_exc(file=sys.stdout)
print '-' * 60
convert_error_num += 1
continue
if config.data_type == 'django':
ref_code_for_bleu = example.meta_data['raw_code']
pred_code_for_bleu = de_canonicalize_code(code, example.meta_data['raw_code'])
# convert canonicalized code to raw code
for literal, place_holder in example.meta_data['str_map'].iteritems():
pred_code_for_bleu = pred_code_for_bleu.replace('\'' + place_holder + '\'', literal)
elif config.data_type == 'hs':
ref_code_for_bleu = ref_code
pred_code_for_bleu = code
# we apply Ling Wang's trick when evaluating BLEU scores
refer_tokens_for_bleu = tokenize_for_bleu_eval(ref_code_for_bleu)
pred_tokens_for_bleu = tokenize_for_bleu_eval(pred_code_for_bleu)
predict_tokens = tokenize_code(code)
# if ref_repr == predict_repr:
if predict_tokens == refer_tokens:
cum_acc += 1
ngram_weights = [0.25] * min(4, len(refer_tokens_for_bleu))
bleu_score = sentence_bleu([refer_tokens_for_bleu], pred_tokens_for_bleu, weights=ngram_weights,
smoothing_function=sm.method3)
cum_bleu += bleu_score
cum_bleu /= len(line_id_to_raw_id)
cum_acc /= len(line_id_to_raw_id)
logging.info('nun. examples: %d', len(line_id_to_raw_id))
logging.info('num. errors when converting repr to tree: %d', convert_error_num)
logging.info('ratio of grammatically incorrect trees: %f', convert_error_num / float(len(line_id_to_raw_id)))
logging.info('sentence level bleu: %f', cum_bleu)
logging.info('accuracy: %f', cum_acc)
