def test_experiments():
random.seed(2001)
stop_rlz = {
'broken_threshold' : 0.1,
'max_broken_walks' : 50,
'max_elapsed_sec' : 2,
'min_num_walks' : 30,
'stop_num_full_matches' : 3
}
x_path = Search.build_experiment(15, 30, stop_rlz)
with open(x_path + '/config.json', 'r') as f:
conf = json.load(f)
assert(len(conf['solved']) == 15)
assert(len(conf['not_solved']) == 15)
assert(conf['stop_rlz']['max_elapsed_sec'] == 2)
assert(conf['stop_rlz']['stop_num_full_matches'] == 3)
random.seed(2001)
Search.run_experiment(x_path, mcts=MockMCTS())
with open(x_path + '/result_details.json', 'r') as f:
results = json.load(f)
name = conf['solved'][0]
assert isinstance(results[name]['correct'][0], int)
with open(x_path + '/result_summary.txt', 'r') as f:
summary = f.read().splitlines()
assert summary[0].startswith('Total number of problems')
assert summary[1].startswith('Total number of questions')
assert summary[2].startswith('Total number of correct solutions')
assert summary[3].startswith('Total running time')
assert summary[5].startswith('Correct solutions in code base')
assert summary[6].startswith('New correct solutions')
assert summary[7].startswith('Failed in code base')
assert summary[8].startswith('Found and wrong')
assert summary[9].startswith('New correct found')
shutil.rmtree(x_path)
from Search import Search
stop_rlz = {
'broken_threshold': 0.08,
'max_broken_walks': 50,
'max_elapsed_sec': 12,
'min_num_walks': 50,
'stop_num_full_matches': 1
}
x_path = Search.build_experiment(100, 100, stop_rlz)
# x_path = './experiments/2020-05-24_180322'
print(x_path)
res = Search.run_experiment(x_path)
print('\n'.join(res))
