def hello():
pred_json = json.loads(request.args.get('pred'))
gold_json = json.loads(request.args.get('gold'))
dataset = request.args.get('dataset')
engine = DBEngine(os.path.join(DATABASE_PATH, "{}.db".format(dataset)))
exact_match = []
grades = []
for lp, ls in tqdm(zip(pred_json, gold_json), total=len(gold_json)):
eg = ls
ep = lp
qg = Query.from_dict(eg['sql'])
gold = engine.execute_query(eg['table_id'], qg, lower=True)
pred = ep['error']
qp = None
if not ep['error']:
try:
qp = Query.from_dict(ep['query'])
pred = engine.execute_query(eg['table_id'], qp, lower=True)
except Exception as e:
pred = repr(e)
correct = pred == gold
match = qp == qg
grades.append(correct)
exact_match.append(match)
ex_accuracy = sum(grades) / len(grades)
lf_accuracy = sum(exact_match) / len(exact_match)
return json.dumps({"ex_accuracy": ex_accuracy, "lf_accuracy": lf_accuracy})
def eval_one_qelos(db_file, pred_file, source_file):
engine = DBEngine(db_file)
exact_match = []
with open(source_file) as fs, open(pred_file) as fp:
grades = []
for ls, lp in tqdm(zip(fs, fp), total=count_lines(source_file)):
eg = json.loads(ls)
ep = json.loads(lp)
qg = Query.from_dict(eg['sql'])
gold = engine.execute_query(eg['table_id'], qg, lower=True)
qp = None
try:
qp = Query.from_dict(ep)
pred = engine.execute_query(eg['table_id'], qp, lower=True)
except Exception as e:
pred = repr(e)
correct = pred == gold
match = qp == qg
grades.append(correct)
exact_match.append(match)
result = {
'ex_accuracy': sum(grades) / len(grades),
'lf_accuracy': sum(exact_match) / len(exact_match),
}
return result
def main(argv):
del argv # Unused.
db_file = join(FLAGS.data_root, FLAGS.db_file)
parsed_std_sql_file = join(FLAGS.data_root, FLAGS.parsed_std_sql_file)
parsed_pred_sql_file = join(FLAGS.data_root, FLAGS.parsed_pred_sql_file)
engine = DBEngine(db_file)
exact_match = []
with open(parsed_std_sql_file) as fs, open(parsed_pred_sql_file) as fp:
grades = []
for ls, lp in tqdm(zip(fs, fp),
total=count_lines(parsed_std_sql_file)):
eg = json.loads(ls)
ep = json.loads(lp)
try:
qg = Query.from_dict(eg['sql'])
gold = engine.execute_query(eg['table_id'], qg, lower=True)
except Exception as e:
gold = repr(e)
#pred = ep['error']
qp = None
#if not ep['error']:
if True:
try:
qp = Query.from_dict(ep['sql'])
pred = engine.execute_query(eg['table_id'], qp, lower=True)
except Exception as e:
pred = repr(e)
correct = pred == gold
match = qp == qg
if pred == gold and qp != qg:
print(qp)
print(qg)
grades.append(correct)
exact_match.append(match)
print(
json.dumps(
{
'ex_accuracy': sum(grades) / len(grades),
'lf_accuracy': sum(exact_match) / len(exact_match),
},
indent=2))
def main(anno_file_name, col_headers, raw_args=None, verbose=True):
parser = argparse.ArgumentParser(description='evaluate.py')
opts.translate_opts(parser)
opt = parser.parse_args(raw_args)
torch.cuda.set_device(opt.gpu)
opt.db_file = os.path.join(opt.data_path, '{}.db'.format(opt.split))
opt.pre_word_vecs = os.path.join(opt.data_path, 'embedding')
dummy_parser = argparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
opts.train_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
opt.anno = anno_file_name
engine = DBEngine(opt.db_file)
js_list = table.IO.read_anno_json(opt.anno)
prev_best = (None, None)
sql_query = []
for fn_model in glob.glob(opt.model_path):
opt.model = fn_model
translator = Translator(opt, dummy_opt.__dict__)
data = table.IO.TableDataset(js_list, translator.fields, None, False)
test_data = table.IO.OrderedIterator(dataset=data,
device=opt.gpu,
batch_size=opt.batch_size,
train=False,
sort=True,
sort_within_batch=False)
# inference
r_list = []
for batch in test_data:
r_list += translator.translate(batch)
r_list.sort(key=lambda x: x.idx)
pred = r_list[-1]
sql_pred = {
'agg': pred.agg,
'sel': pred.sel,
'conds': pred.recover_cond_to_gloss(js_list[-1])
}
if verbose:
print('\n sql_pred: ', sql_pred, '\n')
print('\n col_headers: ', col_headers, '\n')
sql_query = Query(sql_pred['sel'], sql_pred['agg'], sql_pred['conds'])
try:
ans_pred = engine.execute_query(js_list[-1]['table_id'],
Query.from_dict(sql_pred),
lower=True,
verbose=verbose)
except Exception as e:
ans_pred = None
return sql_query.get_complete_query(col_headers), ans_pred
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from lib.query import Query
from lib.dbengine import DBEngine
import pytorch
if __name__ == '__main__':
for split in ['train', 'dev', 'test']:
print('checking {}'.format(split))
engine = DBEngine('data/{}.db'.format(split))
n_lines = 0
with open('data/{}.jsonl'.format(split)) as f:
for l in f:
n_lines += 1
with open('data/{}.jsonl'.format(split)) as f:
for l in tqdm(f, total=n_lines):
d = json.loads(l)
query = Query.from_dict(d['sql'])
# make sure it's executable
result = engine.execute_query(d['table_id'], query)
if result:
for a, b, c in d['sql']['conds']:
if str(c).lower() not in d['question'].lower():
raise Exception(
'Could not find condition {} in question {} for query {}'
.format(c, d['question'], query))
else:
raise Exception(
'Query {} did not execute to a valid result'.format(
query))
parser.add_argument('--topk', type=int, default=3, help='k of top_k')
args = parser.parse_args()
engine = DBEngine(args.db_file)
temp = []
with open(args.source_file) as fs, open(args.pred_file) as fp:
grades = []
exact_match = []
for ls, lp in tqdm(zip(fs, fp), total=count_lines(args.source_file)):
eg = json.loads(ls)
qg = Query.from_dict(eg['sql'], ordered=args.ordered)
gold = engine.execute_query(eg['table_id'], qg, lower=True)
pred_topk = []
qp_topk = []
ep = json.loads(lp)
pred = ep.get('error', None)
qp = None
for i in range(args.topk):
if not ep.get('error', None):
try:
if ep['query'][str(i)]['conds'] == [[]]:
ep['query'][str(i)]['conds'] = []
qp = Query.from_dict(ep['query'][str(i)], ordered=args.ordered)
parser.add_argument("--pred_file")
parser.add_argument("--ordered", action='store_true')
args = parser.parse_args()
engine = DBEngine(args.db_file)
ex_acc_list = []
lf_acc_list = []
with open(args.source_file) as sf, open(args.pred_file) as pf:
for source_line, pred_line in tqdm(zip(sf, pf), total=count_lines(args.source_file)):
# line별 정답과 예측 샘플 가져오기
gold_example = json.loads(source_line)
pred_example = json.loads(pred_line)
# 정답 샘플 lf, ex 구하기
lf_gold_query = Query.from_dict(gold_example['sql'], ordered=args.ordered)
ex_gold = engine.execute_query(gold_example['table_id'], lf_gold_query, lower=True)
# error가 아닌 경우 예측 샘플 lf, ex 구하기
lf_pred_query = None
ex_pred = pred_example.get('error', None)
if not ex_pred:
try:
lf_pred_query = Query.from_dict(pred_example['query'], ordered=args.ordered)
ex_pred = engine.execute_query(gold_example['table_id'], lf_pred_query, lower=True)
except Exception as e:
ex_pred = repr(e)
# lf, ex의 gold, pred 매칭결과 구하기
ex_acc_list.append(ex_pred == ex_gold)
lf_acc_list.append(lf_pred_query == lf_gold_query) # query의 __eq__를 호출
print('ex_accuracy {}\n lf_accuracy {}'.format(
