def test_experiments(): random.seed(2001) stop_rlz = { 'broken_threshold' : 0.1, 'max_broken_walks' : 50, 'max_elapsed_sec' : 2, 'min_num_walks' : 30, 'stop_num_full_matches' : 3 } x_path = Search.build_experiment(15, 30, stop_rlz) with open(x_path + '/config.json', 'r') as f: conf = json.load(f) assert(len(conf['solved']) == 15) assert(len(conf['not_solved']) == 15) assert(conf['stop_rlz']['max_elapsed_sec'] == 2) assert(conf['stop_rlz']['stop_num_full_matches'] == 3) random.seed(2001) Search.run_experiment(x_path, mcts=MockMCTS()) with open(x_path + '/result_details.json', 'r') as f: results = json.load(f) name = conf['solved'][0] assert isinstance(results[name]['correct'][0], int) with open(x_path + '/result_summary.txt', 'r') as f: summary = f.read().splitlines() assert summary[0].startswith('Total number of problems') assert summary[1].startswith('Total number of questions') assert summary[2].startswith('Total number of correct solutions') assert summary[3].startswith('Total running time') assert summary[5].startswith('Correct solutions in code base') assert summary[6].startswith('New correct solutions') assert summary[7].startswith('Failed in code base') assert summary[8].startswith('Found and wrong') assert summary[9].startswith('New correct found') shutil.rmtree(x_path)
from Search import Search stop_rlz = { 'broken_threshold': 0.08, 'max_broken_walks': 50, 'max_elapsed_sec': 12, 'min_num_walks': 50, 'stop_num_full_matches': 1 } x_path = Search.build_experiment(100, 100, stop_rlz) # x_path = './experiments/2020-05-24_180322' print(x_path) res = Search.run_experiment(x_path) print('\n'.join(res))