def test_calling():
ctx = Context('data')
prob = ctx.get_problem_startswith('496994bd')
assert isinstance(prob, Problem)
assert isinstance(prob.data[0], Example)
q, a, is_test = prob.data[0].data
assert isinstance(q, Question)
assert isinstance(a, Answer)
assert isinstance(is_test, bool)
bop_fun = BopForward.bopforward_pic_all_as_pat
assert isinstance(bop_fun, Function)
assert isinstance(bop_fun.arg_types, list)
assert bop_fun.ret_type == Block.type_pattern
ff = bop_fun.data
assert callable(ff)
ret = ff(q)
assert isinstance(ret, Block)
assert ret.type == Block.type_pattern
assert isinstance(ret.data, tuple)
assert len(ret.data) == 2
def test_successful_end_to_end():
f = Field(Example)
assert f.from_kind == Question
assert f.to_kind == Answer
assert 'get_question' in f.opcode
assert 'tests_verify_answer' in f.opcode
assert '2pat_merge_as_pic' in f.opcode
source = Source(('get_question',
'pic_all_as_pat',
'pat_flip_up_down',
'get_question',
'pic_all_as_pat',
'2pat_merge_as_pic',
'tests_verify_answer'))
code = f.compile(source)
assert isinstance(code, Code)
assert len(code.data) == len(source.data)
ctx = Context('data')
prb = ctx.get_problem_startswith('496994bd')
for example in prb:
ret = f.run(code, example, peek_answer=True)
assert isinstance(ret, Block)
assert ret.type == Block.type_integer
assert isinstance(ret.data, int)
assert ret.data == 1
def test_successful_end_to_end():
ctx = Context('data')
prb = ctx.get_problem_startswith('496994bd')
code = Code(
(Bebop.bebop_get_question, BopForward.bopforward_pic_all_as_pat,
BopForward.bopforward_pat_flip_up_down, Bebop.bebop_get_question,
BopForward.bopforward_pic_all_as_pat,
BopForward.bopforward_2pat_merge_as_pic))
for example in prb:
q, a, _ = example.data
core = Core(code)
core.register['question'] = q
*_, last_block = core
assert core.all_right
assert len(core.stack) == 1
assert isinstance(last_block, Block)
assert last_block.type == Block.type_picture
assert np.array_equal(last_block.data, a.data)
def test_CodeGen():
context = Context('data')
codegen = CodeGen.CodeGen('./code_base/basis_code.jcb',
'./code_base/basis_code.evd', Example)
problem = context.get_problem_startswith(
'917bccba.json') # Hard enough to be unsolved
codegen.demo_q = []
codegen.demo_a = []
codegen.question = []
for example in problem:
q, a, is_test = example.data
if is_test:
codegen.question.append(q)
else:
codegen.demo_q.append(q)
codegen.demo_a.append(a)
root = MctsNode()
new_moves = codegen.new_moves(root)
visits = 0
rewards = 0
for code_item, prior, reward, eval in new_moves:
assert isinstance(code_item, tuple)
assert prior > 0 and prior <= 2
assert reward > 0 and reward <= 1
pr = codegen.predict_rewards(np.stack([eval], axis=0))
assert pr[0] == reward
child = MctsNode(code_item, prior, reward, root)
assert isinstance(child, MctsNode)
visits += 1
rewards += reward
root.visits += visits
root.reward += REWARD_DISCOUNT * rewards
node = root
while not node.is_leaf():
node = node.select_child()
new_moves = codegen.new_moves(node)
visits = 0
rewards = 0
for code_item, prior, reward, eval in new_moves:
assert isinstance(code_item, tuple)
assert prior > 0 and prior <= 2
assert reward > 0 and reward <= 1
pr = codegen.predict_rewards(np.stack([eval], axis=0))
assert pr[0] == reward
child = MctsNode(code_item, prior, reward, root)
assert isinstance(child, MctsNode)
visits += 1
rewards += reward
def run_experiment(experiment_path, mcts=None):
"""
Runs the MCTS.run_search() over all the problems in the experiment:
- saves the best codes
- verifies the solution and saves the result
Both result files are updated at each search iteration. You can tail -f the summary to see the progress
Full details are stored in experiment_path + '/result_details.json' as a dictionary where the key is the problem name and the
value is another dictionary with:
(Computed by MCTS)
'source' : A list of (maximum) NUM_TOP_SOLUTIONS as source code to help serialization
'evaluation' : A list of (maximum) NUM_TOP_SOLUTIONS CodeEval np.array.tolist() evaluations for each snippet
'elapsed' : A list of (maximum) NUM_TOP_SOLUTIONS the second each solution was found
'num_walks' : A list of (maximum) NUM_TOP_SOLUTIONS the walk number at which the solution was found
'prediction' : A LoL for each question in the problem and (maximum) NUM_TOP_SOLUTIONS the guess as an np.array.tolist()
'tot_elapsed' : The total number of seconds the search used
'tot_walks' : The total number of walks the search did
'stopped_on' : Either 'time' or 'found' the condition that stopped the search
(Computed by run_experiment)
'correct' : A list for each question of (int) number of correct guesses for the question
Summary details are stored in experiment_path + '/result_summary.txt' as a text file (and printed to stdout) as:
'Total number of problems : %4i of %4i'
'Total number of questions : %4i'
'Total number of correct solutions : %4i (%3.1f%%)'
'Total running time : %4i minutes'
''
'Correct solutions in code base : %4i (%3.1f%%)'
'New correct solutions : %4i (%3.1f%%)'
'Failed in code base : %s'
'Found and wrong : %s'
'New correct found : %s'
"""
mcts = mcts if mcts is not None else MCTS(
'./code_base/basis_code.jcb', './code_base/basis_code.evd',
Example)
context = Context('data')
with open(experiment_path + '/config.json', 'r') as f:
conf = json.load(f)
filenames = conf['solved'] + conf['not_solved']
num_prob_todo = len(filenames)
num_prob_done = 0
tot_num_questions = 0
tot_correct_sol = 0
tot_running_sec = 0
correct_in_cdb = 0
correct_new = 0
failed_in_codebase = []
found_and_wrong = []
found_new = []
details = {}
for filename in filenames:
problem = context.get_problem_startswith(filename)
answers = []
for example in problem:
_, answer, is_test = example.data
if is_test:
answers.append(answer)
results = mcts.run_search(problem, conf['stop_rlz'])
results['correct'] = []
correct = 0
found = False
for eval, preds in zip(results['evaluation'],
results['prediction']):
if eval[IDX_PIC_REACH_MIN] == EVAL_FULL_MATCH:
found = True
for pred, answer in zip(preds, answers):
if np.array_equal(answer.data,
np.array(pred, dtype=np.int32)):
correct += 1
results['correct'].append(correct)
if correct > 0:
tot_correct_sol += 1
if filename in conf['solved']:
correct_in_cdb += 1
else:
correct_new += 1
found_new.append(filename)
else:
if filename in conf['solved']:
failed_in_codebase.append(filename)
if found:
found_and_wrong.append(filename)
tot_num_questions += len(answers)
details[filename] = results
num_prob_done += 1
tot_running_sec += results['tot_elapsed']
summary = []
summary.append(
'Total number of problems : %4i (%3.1f%%)' %
(num_prob_done, 100 * num_prob_done / num_prob_todo))
summary.append('Total number of questions : %4i' %
tot_num_questions)
summary.append(
'Total number of correct solutions : %4i (%3.1f%%)' %
(tot_correct_sol, 100 * tot_correct_sol / tot_num_questions))
summary.append('Total running time : %4i minutes' %
int(tot_running_sec / 60))
summary.append('')
summary.append(
'Correct solutions in code base : %4i (%3.1f%%)' %
(correct_in_cdb, 100 * correct_in_cdb / tot_num_questions))
summary.append(
'New correct solutions : %4i (%3.1f%%)' %
(correct_new, 100 * correct_new / tot_num_questions))
summary.append('Failed in code base : %s' %
', '.join(failed_in_codebase))
summary.append('Found and wrong : %s' %
', '.join(found_and_wrong))
summary.append('New correct found : %s' %
', '.join(found_new))
with open(experiment_path + '/result_details.json', 'w') as f:
json.dump(details, f)
with open(experiment_path + '/result_summary.txt', 'w') as f:
f.writelines(["%s\n" % line for line in summary])
return summary
def test_Multicore():
# '1b2d62fb.json' : Source(('get_question',
# 'pic_fork_on_v_axis_as_pics',
# 'pics_as_2pic',
# '2pic_maximum',
# '(0, 8)',
# 'swap_top2',
# 'pic_intp_swap_colors',
# '(9, 0)',
# 'swap_top2',
# 'pic_intp_swap_colors')),
ctx = Context('data')
prob = ctx.get_problem_startswith('1b2d62fb')
eval = CodeEval(Example)
eval.demo_q = []
eval.demo_a = []
eval.question = []
for example in prob:
q, a, is_test = example.data
if is_test:
eval.question.append(q)
else:
eval.demo_q.append(q)
eval.demo_a.append(a)
c11 = eval.compile(
Source(('get_question', 'pic_fork_on_v_axis_as_pics', 'pics_as_2pic',
'2pic_maximum')))
c12 = eval.compile(
Source(('get_question', 'pic_fork_on_h_axis_as_pics', 'pics_as_2pic',
'2pic_maximum')))
c13 = eval.compile(
Source(('get_question', 'pic_fork_on_v_axis_as_pics', 'pics_as_2pic',
'2pic_xor_masks_to_1')))
c21 = eval.compile(Source(('(0, 8)', 'swap_top2', 'pic_intp_swap_colors')))
c22 = eval.compile(Source(('(5, 4)', 'swap_top2', 'pic_intp_swap_colors')))
c23 = eval.compile(
Source(('(5, 4)', 'swap_top2', 'pic_intp_select_columns')))
c31 = eval.compile(Source(('(9, 0)', 'swap_top2', 'pic_intp_swap_colors')))
c32 = eval.compile(Source(('(5, 4)', 'swap_top2', 'pic_intp_recolor')))
c33 = eval.compile(Source(('pic_transpose', )))
assert c11.type == Block.type_no_error and c12.type == Block.type_no_error and c13.type == Block.type_no_error
assert c21.type == Block.type_no_error and c22.type == Block.type_no_error and c23.type == Block.type_no_error
assert c31.type == Block.type_no_error and c32.type == Block.type_no_error and c33.type == Block.type_no_error
eval.multicore_clear()
ret = eval.multicore_run_all(c11)
assert ret.type == Block.type_no_error
for pl in eval.multicore_state['pic_lists']:
assert len(pl) == 1
ret = eval.multicore_run_all(c21)
assert ret.type == Block.type_no_error
for pl in eval.multicore_state['pic_lists']:
assert len(pl) == 2
ret = eval.multicore_run_all(c31)
assert ret.type == Block.type_no_error
for pic_list, example in zip(eval.multicore_state['pic_lists'], prob):
assert len(pic_list) == 3
_, answer, _ = example.data
assert np.array_equal(pic_list[2].data, answer.data)
eval.multicore_clear()
root = eval.multicore_copy_state(eval.multicore_state)
ret = eval.multicore_run_all(c13)
assert ret.type == Block.type_no_error
ret = eval.multicore_run_all(c21)
assert ret.type == Block.type_no_error
for pl in eval.multicore_state['pic_lists']:
assert len(pl) == 2
eval.multicore_set_state(root)
for pl in eval.multicore_state['pic_lists']:
assert len(pl) == 0
ret = eval.multicore_run_all(c12)
assert ret.type == Block.type_no_error
ret = eval.multicore_run_all(c23)
assert ret.type == Block.type_no_error
for pl in eval.multicore_state['pic_lists']:
assert len(pl) == 2
eval.multicore_set_state(root)
ret = eval.multicore_run_all(c11)
for pl in eval.multicore_state['pic_lists']:
assert len(pl) == 1
lev_1 = eval.multicore_copy_state(eval.multicore_state)
ret = eval.multicore_run_all(c23)
assert ret.type == Block.type_no_error
ret = eval.multicore_run_all(c33)
assert ret.type == Block.type_no_error
eval.multicore_set_state(lev_1)
ret = eval.multicore_run_all(c22)
assert ret.type == Block.type_no_error
ret = eval.multicore_run_all(c31)
assert ret.type == Block.type_no_error
eval.multicore_set_state(lev_1)
ret = eval.multicore_run_all(c21)
lev_2 = eval.multicore_copy_state(eval.multicore_state)
ret = eval.multicore_run_all(c33)
assert ret.type == Block.type_no_error
eval.multicore_set_state(lev_2)
ret = eval.multicore_run_all(c32)
assert ret.type == Block.type_no_error
eval.multicore_set_state(lev_2)
ret = eval.multicore_run_all(c31)
assert ret.type == Block.type_no_error
pic_lists = eval.multicore_state['pic_lists']
for pic_list, example in zip(pic_lists, prob):
assert len(pic_list) == 3
_, answer, _ = example.data
assert np.array_equal(pic_list[2].data, answer.data)
def test_CodeEval():
# '1b2d62fb.json' : Source(('get_question',
# 'pic_fork_on_v_axis_as_pics',
# 'pics_as_2pic',
# '2pic_maximum',
# '(0, 8)',
# 'swap_top2',
# 'pic_intp_swap_colors',
# '(9, 0)',
# 'swap_top2',
# 'pic_intp_swap_colors')),
ctx = Context('data')
prob = ctx.get_problem_startswith('1b2d62fb')
eval = CodeEval(Example)
with pytest.raises(TypeError): # self.multicore_state is None
eval.eval_code()
eval.demo_q = []
eval.demo_a = []
eval.question = []
eval.multicore_clear() # Builds a state without cores
with pytest.raises(LookupError): # self.multicore_state is None
eval.eval_code()
for example in prob:
q, a, is_test = example.data
if is_test:
eval.question.append(q)
else:
eval.demo_q.append(q)
eval.demo_a.append(a)
eval.multicore_clear()
with pytest.raises(LookupError): # Runs an eval without a picture
eval.eval_code()
c1 = eval.compile(
Source(('get_question', 'pic_fork_on_v_axis_as_pics', 'pics_as_2pic',
'2pic_maximum')))
c2 = eval.compile(Source(('(0, 8)', 'swap_top2', 'pic_intp_swap_colors')))
c3 = eval.compile(Source(('(9, 0)', 'swap_top2', 'pic_intp_swap_colors')))
ret = eval.multicore_run_all(c1)
assert ret.type == Block.type_no_error
vec = eval.eval_code()
assert vec[IDX_PIC_REACH_MIN] == 0 and vec[IDX_PIC_REACH_MAX] == 0
assert vec[IDX_PAT_REACH_MIN] == 0 and vec[IDX_PAT_REACH_MAX] == 0
assert vec[IDX_PIC_BETTER_MEAN] == EVAL_WRONG_SHAPE and vec[
IDX_PIC_WORSE_MEAN] == EVAL_WRONG_SHAPE
ret = eval.multicore_run_all(c2)
assert ret.type == Block.type_no_error
vec = eval.eval_code()
assert 0 < vec[IDX_PIC_REACH_MIN] and vec[IDX_PIC_REACH_MIN] < vec[
IDX_PIC_REACH_MEAN]
assert vec[IDX_PIC_REACH_MEAN] < vec[IDX_PIC_REACH_MAX] and vec[
IDX_PIC_REACH_MAX] < 1
assert vec[IDX_PIC_REACH_MIN] == vec[IDX_PAT_REACH_MIN] and vec[
IDX_PIC_REACH_MEAN] == vec[IDX_PAT_REACH_MEAN]
assert vec[IDX_PIC_WORSE_MAX] == 0 and vec[IDX_PAT_WORSE_MAX] == 0
assert 0 < vec[IDX_PIC_BETTER_MIN] and vec[IDX_PIC_BETTER_MIN] < vec[
IDX_PIC_BETTER_MEAN]
assert vec[IDX_PIC_BETTER_MEAN] < vec[IDX_PIC_BETTER_MAX] and vec[
IDX_PIC_BETTER_MAX] < 1
assert vec[IDX_PIC_BETTER_MIN] == vec[IDX_PAT_BETTER_MIN] and vec[
IDX_PIC_BETTER_MEAN] == vec[IDX_PAT_BETTER_MEAN]
ret = eval.multicore_run_all(c3)
assert ret.type == Block.type_no_error
vec = eval.eval_code()
assert vec[IDX_PIC_REACH_MIN] == EVAL_FULL_MATCH
assert vec[IDX_PAT_REACH_MIN] == EVAL_FULL_MATCH
assert vec[IDX_PIC_WORSE_MIN] == 0 and vec[IDX_PAT_WORSE_MIN] == 0
assert vec[IDX_PIC_WORSE_MEAN] == 0 and vec[IDX_PAT_WORSE_MEAN] == 0
assert vec[IDX_PIC_WORSE_MAX] == 0 and vec[IDX_PAT_WORSE_MAX] == 0
assert 0 < vec[IDX_PIC_BETTER_MIN] and vec[IDX_PIC_BETTER_MIN] < vec[
IDX_PIC_BETTER_MEAN]
assert vec[IDX_PIC_BETTER_MEAN] < vec[IDX_PIC_BETTER_MAX] and vec[
IDX_PIC_BETTER_MAX] < 1
eval.multicore_clear()
c1 = eval.compile(Source(('get_question', 'pic_transpose')))
ret = eval.multicore_run_all(c1)
assert ret.type == Block.type_no_error
vec = eval.eval_code()
for vv in vec.tolist():
assert vv == EVAL_WRONG_SHAPE
def test_MCTS():
context = Context('data')
mcts = MCTS('./code_base/basis_code.jcb', './code_base/basis_code.evd',
Example)
stop_rlz = {
'broken_threshold': 0.1,
'max_broken_walks': 50,
'max_elapsed_sec': 2,
'min_num_walks': 5,
'stop_num_full_matches': 1
}
prob1a = context.get_problem_startswith('ed36ccf7') # One item
prob1b = context.get_problem_startswith('007bbfb7') # One item
prob2 = context.get_problem_startswith('0692e18c') # Two items
probX = context.get_problem_startswith('c6e1b8da') # Nightmare task
answer = None
for example in prob1a:
_, a, is_test = example.data
if is_test:
answer = a
ret = mcts.run_search(prob1a, stop_rlz)
assert ret['stopped_on'] == 'found' and ret[
'tot_walks'] == 5 and ret['tot_elapsed'] < 10
N = 0
for src, evl, ela, n_w, prd in zip(ret['source'], ret['evaluation'],
ret['elapsed'], ret['num_walks'],
ret['prediction']):
code = mcts.compile(Source(src))
assert code.type == Block.type_no_error
assert evl[IDX_PIC_REACH_MIN] == EVAL_FULL_MATCH
assert ela < 10
assert n_w <= 5
for pic in prd:
assert np.array_equal(np.array(pic, dtype=np.int32), answer.data)
N += 1
assert N == 3
ret = mcts.run_search(prob1b, stop_rlz)
assert ret['stopped_on'] == 'found' and ret[
'tot_walks'] == 5 and ret['tot_elapsed'] < 10
answer = None
for example in prob2:
_, a, is_test = example.data
if is_test:
answer = a
root = MctsNode()
ret = mcts.run_search(prob2, stop_rlz, root=root)
assert ret['tot_walks'] < 100 and ret['tot_elapsed'] < 10
N = 0
for src, evl, ela, n_w, prd in zip(ret['source'], ret['evaluation'],
ret['elapsed'], ret['num_walks'],
ret['prediction']):
code = mcts.compile(Source(src))
assert code.type == Block.type_no_error
assert ela < 10
assert n_w <= 100
N += 1 * (evl[IDX_PIC_REACH_MIN] == EVAL_FULL_MATCH)
assert ret['stopped_on'] == 'time' or N > 0
root.print(field=mcts)
last = root.children[-1]
last.print()
ret = mcts.run_search(probX, stop_rlz)
assert ret['stopped_on'] == 'time' and ret['tot_walks'] < 100 and ret[
'tot_elapsed'] < 10
stop_rlz['broken_threshold'] = 1
stop_rlz['max_broken_walks'] = 4
ret = mcts.run_search(probX, stop_rlz)
assert ret['stopped_on'] == 'lost' and ret['tot_walks'] < 100 and ret[
'tot_elapsed'] < 10
