def test_add_entry(self, mock_stats): tl = TrajLogger(output_dir='./tmp_test_folder', stats=mock_stats) test_config = {'param_a': 0.5, 'param_b': 1, 'param_c': 'value'} mock_stats.ta_time_used = .5 mock_stats.get_used_wallclock_time = self.mocked_get_used_wallclock_time mock_stats.ta_runs = 1 tl.add_entry(0.9, 1, test_config) self.assertTrue(os.path.exists('tmp_test_folder/traj_old.csv')) self.assertTrue(os.path.exists('tmp_test_folder/traj_aclib2.json')) with open('tmp_test_folder/traj_old.csv') as to: data = to.read().split('\n')[:-1] header = data[0].split(',') self.assertEquals(header[0], '"CPU Time Used"') self.assertEquals(header[-1], '"Configuration..."') data = list(map(lambda x: x.split(', '), data[1:])) frmt_str = '%1.6f' self.assertEquals(frmt_str % .5, data[0][0]) self.assertEquals(frmt_str % .9, data[0][1]) self.assertEquals(frmt_str % 0.5, data[0][-4]) with open('tmp_test_folder/traj_aclib2.json') as js: json_dict = json.load(js) self.assertEquals(json_dict['cpu_time'], .5) self.assertEquals(json_dict['cost'], 0.9) self.assertEquals(len(json_dict['incumbent']), 3) self.assertTrue("param_a='0.5'" in json_dict['incumbent']) # And finally, test the list that's added to the trajectory class self.assertEqual( tl.trajectory[0], TrajEntry(0.9, 1, { 'param_c': 'value', 'param_b': 1, 'param_a': 0.5 }, 1, 0.5, 1)) # Test named-tuple-access: self.assertEqual(tl.trajectory[0].train_perf, 0.9) self.assertEqual(tl.trajectory[0].incumbent_id, 1) self.assertEqual(tl.trajectory[0].ta_runs, 1) self.assertEqual(tl.trajectory[0].ta_time_used, 0.5) self.assertEqual(tl.trajectory[0].wallclock_time, 1) self.assertEqual(len(tl.trajectory), 1)
def test_add_entries(self, mock_stats): # Mock stats mock_stats.ta_time_used = .5 mock_stats.get_used_wallclock_time = self.mocked_get_used_wallclock_time mock_stats.finished_ta_runs = 1 with tempfile.TemporaryDirectory() as tmpdir: tl = TrajLogger(output_dir=tmpdir, stats=mock_stats) # Add some entries tl.add_entry(0.9, 1, self.test_config, 0) mock_stats.ta_runs = 2 mock_stats.ta_time_used = 0 tl.add_entry(1.3, 1, self.test_config, 10) mock_stats.ta_time_used = 0 tl.add_entry(0.7, 2, Configuration(self.cs, dict(self.test_config.get_dictionary(), **{'param_a': 0.})), 10) # Test the list that's added to the trajectory class self.assertEqual(tl.trajectory[0], TrajEntry(0.9, 1, self.test_config, 1, 0.5, 1, 0)) # Test named-tuple-access: self.assertEqual(tl.trajectory[0].train_perf, 0.9) self.assertEqual(tl.trajectory[0].incumbent_id, 1) self.assertEqual(tl.trajectory[0].ta_runs, 1) self.assertEqual(tl.trajectory[0].ta_time_used, 0.5) self.assertEqual(tl.trajectory[0].wallclock_time, 1) self.assertEqual(tl.trajectory[0].budget, 0) self.assertEqual(len(tl.trajectory), 3) # Check if the trajectories are generated for fn in ['traj_old.csv', 'traj_aclib2.json', 'traj.json']: self.assertTrue(os.path.exists(os.path.join(tmpdir, fn))) # Load trajectories with open(os.path.join(tmpdir, 'traj_old.csv')) as to: data = to.read().split('\n') with open(os.path.join(tmpdir, 'traj_aclib2.json')) as js_aclib: json_dicts_aclib2 = [json.loads(line) for line in js_aclib.read().splitlines()] with open(os.path.join(tmpdir, 'traj.json')) as js: json_dicts_alljson = [json.loads(line) for line in js.read().splitlines()] # Check old format header = data[0].split(',') self.assertEqual(header[0], '"CPU Time Used"') self.assertEqual(header[-1], '"Configuration..."') data = list(map(lambda x: x.split(', '), data[1:])) frmt_str = '%1.6f' self.assertEqual(frmt_str % 0.5, data[0][0]) self.assertEqual(frmt_str % 0.9, data[0][1]) self.assertEqual(frmt_str % 0.5, data[0][4]) self.assertEqual(frmt_str % 0, data[1][0]) self.assertEqual(frmt_str % 1.3, data[1][1]) self.assertEqual(frmt_str % 2, data[1][4]) self.assertEqual(frmt_str % 0, data[2][0]) self.assertEqual(frmt_str % .7, data[2][1]) self.assertEqual(frmt_str % 3, data[2][4]) # Check aclib2-format self.assertEqual(json_dicts_aclib2[0]['cpu_time'], .5) self.assertEqual(json_dicts_aclib2[0]['cost'], 0.9) self.assertEqual(len(json_dicts_aclib2[0]['incumbent']), 4) self.assertTrue("param_a='0.5'" in json_dicts_aclib2[0]['incumbent']) self.assertTrue("param_a='0.0'" in json_dicts_aclib2[2]['incumbent']) # Check alljson-format self.assertEqual(json_dicts_alljson[0]['cpu_time'], .5) self.assertEqual(json_dicts_alljson[0]['cost'], 0.9) self.assertEqual(len(json_dicts_alljson[0]['incumbent']), 4) self.assertTrue(json_dicts_alljson[0]["incumbent"]["param_a"] == 0.5) self.assertTrue(json_dicts_alljson[2]["incumbent"]["param_a"] == 0.0) self.assertEqual(json_dicts_alljson[0]['budget'], 0) self.assertEqual(json_dicts_alljson[2]['budget'], 10)
def test_add_entries_multi_objectives(self, mock_stats): # Mock stats mock_stats.ta_time_used = .5 mock_stats.get_used_wallclock_time = self.mocked_get_used_wallclock_time mock_stats.finished_ta_runs = 1 num_obj = 2 with tempfile.TemporaryDirectory() as tmpdir: tl = TrajLogger(output_dir=tmpdir, stats=mock_stats) # Add some entries tl.add_entry([0.9, 0.8], 1, self.test_config, 0) # Test the list that's added to the trajectory class self.assertEqual( tl.trajectory[0], TrajEntry([0.9, 0.8], 1, self.test_config, 1, 0.5, 1, 0)) # Test named-tuple-access: self.assertEqual(tl.trajectory[0].train_perf, [0.9, 0.8]) self.assertEqual(len(tl.trajectory), 1) # Check if the trajectories are generated for fn in ['traj_old.csv', 'traj_aclib2.json', 'traj.json']: self.assertTrue(os.path.exists(os.path.join(tmpdir, fn))) # Load trajectories with open(os.path.join(tmpdir, 'traj_old.csv')) as to: data = to.read().split('\n') with open(os.path.join(tmpdir, 'traj_aclib2.json')) as js_aclib: json_dicts_aclib2 = [ json.loads(line) for line in js_aclib.read().splitlines() ] with open(os.path.join(tmpdir, 'traj.json')) as js: json_dicts_alljson = [ json.loads(line) for line in js.read().splitlines() ] # Check old format header = data[0].split(',') self.assertEqual(header[0], '"CPU Time Used"') self.assertEqual(header[-1], '"Configuration..."') data = list(map(lambda x: x.split(', '), data[1:])) data[0][1] = ', '.join(data[0][1:1 + num_obj]) del data[0][1 + 1:1 + num_obj] frmt_str = '%1.6f' self.assertEqual(frmt_str % 0.5, data[0][0]) self.assertEqual(f'[{0.9}, {0.8}]', data[0][1]) self.assertEqual(frmt_str % 0.5, data[0][4]) # Check aclib2-format self.assertEqual(json_dicts_aclib2[0]['cpu_time'], .5) self.assertEqual(json_dicts_aclib2[0]['cost'], [0.9, 0.8]) self.assertEqual(len(json_dicts_aclib2[0]['incumbent']), 4) self.assertTrue("param_a='0.5'" in json_dicts_aclib2[0]['incumbent']) # Check alljson-format self.assertEqual(json_dicts_alljson[0]['cpu_time'], .5) self.assertEqual(json_dicts_alljson[0]['cost'], [0.9, 0.8]) self.assertEqual(len(json_dicts_alljson[0]['incumbent']), 4) self.assertTrue(json_dicts_alljson[0]["incumbent"]["param_a"] == 0.5) self.assertEqual(json_dicts_alljson[0]['budget'], 0)