def test_program_on_IO(e, IO, schema_args, executor_):
"""
run executor
"""
stats = executor.evaluate_code(e, schema_args, IO, executor_)
#print(stats['tests-executed'], stats['tests-passed'])
return stats['tests-executed'] == stats['tests-passed']
def get_stats_from_code(args):
res, example, executor_ = args
if len(example.tests) == 0:
return None
if executor_ is not None:
stats = executor.evaluate_code(
res.code_tree if res.code_tree else res.code_sequence,
example.schema.args, example.tests, executor_)
stats['exact-code-match'] = is_same_code(example, res)
stats['correct-program'] = int(
stats['tests-executed'] == stats['tests-passed'])
else:
stats = {
'tests-executed': 0,
'tests-passed': 0,
'result-none': 0,
'syntax-error': 0,
'runtime-exception': 0,
'exceptions': []
}
stats['correct-program'] = stats['exact-code-match'] = is_same_code(
example, res)
stats['bleu'] = compute_bleu(example, res)
stats['example'] = example.to_dict()
stats['res'] = res.to_dict() if hasattr(res, 'to_dict') else res
return stats
def get_best_code(args):
example, codes, executor_ = args
for code in codes:
stats = executor.evaluate_code(code, example.schema.args,
example.input_tests, executor_)
ok = (stats['tests-executed'] == stats['tests-passed'])
if ok:
return code
return codes[0]