def test_add_entry(self, mock_stats):
tl = TrajLogger(output_dir='./tmp_test_folder', stats=mock_stats)
test_config = {'param_a': 0.5, 'param_b': 1, 'param_c': 'value'}
mock_stats.ta_time_used = .5
mock_stats.get_used_wallclock_time = self.mocked_get_used_wallclock_time
mock_stats.ta_runs = 1
tl.add_entry(0.9, 1, test_config)
self.assertTrue(os.path.exists('tmp_test_folder/traj_old.csv'))
self.assertTrue(os.path.exists('tmp_test_folder/traj_aclib2.json'))
with open('tmp_test_folder/traj_old.csv') as to:
data = to.read().split('\n')[:-1]
header = data[0].split(',')
self.assertEquals(header[0], '"CPU Time Used"')
self.assertEquals(header[-1], '"Configuration..."')
data = list(map(lambda x: x.split(', '), data[1:]))
frmt_str = '%1.6f'
self.assertEquals(frmt_str % .5, data[0][0])
self.assertEquals(frmt_str % .9, data[0][1])
self.assertEquals(frmt_str % 0.5, data[0][-4])
with open('tmp_test_folder/traj_aclib2.json') as js:
json_dict = json.load(js)
self.assertEquals(json_dict['cpu_time'], .5)
self.assertEquals(json_dict['cost'], 0.9)
self.assertEquals(len(json_dict['incumbent']), 3)
self.assertTrue("param_a='0.5'" in json_dict['incumbent'])
# And finally, test the list that's added to the trajectory class
self.assertEqual(
tl.trajectory[0],
TrajEntry(0.9, 1, {
'param_c': 'value',
'param_b': 1,
'param_a': 0.5
}, 1, 0.5, 1))
# Test named-tuple-access:
self.assertEqual(tl.trajectory[0].train_perf, 0.9)
self.assertEqual(tl.trajectory[0].incumbent_id, 1)
self.assertEqual(tl.trajectory[0].ta_runs, 1)
self.assertEqual(tl.trajectory[0].ta_time_used, 0.5)
self.assertEqual(tl.trajectory[0].wallclock_time, 1)
self.assertEqual(len(tl.trajectory), 1)
def test_add_entries(self, mock_stats):
# Mock stats
mock_stats.ta_time_used = .5
mock_stats.get_used_wallclock_time = self.mocked_get_used_wallclock_time
mock_stats.finished_ta_runs = 1
with tempfile.TemporaryDirectory() as tmpdir:
tl = TrajLogger(output_dir=tmpdir, stats=mock_stats)
# Add some entries
tl.add_entry(0.9, 1, self.test_config, 0)
mock_stats.ta_runs = 2
mock_stats.ta_time_used = 0
tl.add_entry(1.3, 1, self.test_config, 10)
mock_stats.ta_time_used = 0
tl.add_entry(0.7, 2, Configuration(self.cs, dict(self.test_config.get_dictionary(), **{'param_a': 0.})), 10)
# Test the list that's added to the trajectory class
self.assertEqual(tl.trajectory[0], TrajEntry(0.9, 1, self.test_config, 1, 0.5, 1, 0))
# Test named-tuple-access:
self.assertEqual(tl.trajectory[0].train_perf, 0.9)
self.assertEqual(tl.trajectory[0].incumbent_id, 1)
self.assertEqual(tl.trajectory[0].ta_runs, 1)
self.assertEqual(tl.trajectory[0].ta_time_used, 0.5)
self.assertEqual(tl.trajectory[0].wallclock_time, 1)
self.assertEqual(tl.trajectory[0].budget, 0)
self.assertEqual(len(tl.trajectory), 3)
# Check if the trajectories are generated
for fn in ['traj_old.csv', 'traj_aclib2.json', 'traj.json']:
self.assertTrue(os.path.exists(os.path.join(tmpdir, fn)))
# Load trajectories
with open(os.path.join(tmpdir, 'traj_old.csv')) as to:
data = to.read().split('\n')
with open(os.path.join(tmpdir, 'traj_aclib2.json')) as js_aclib:
json_dicts_aclib2 = [json.loads(line) for line in js_aclib.read().splitlines()]
with open(os.path.join(tmpdir, 'traj.json')) as js:
json_dicts_alljson = [json.loads(line) for line in js.read().splitlines()]
# Check old format
header = data[0].split(',')
self.assertEqual(header[0], '"CPU Time Used"')
self.assertEqual(header[-1], '"Configuration..."')
data = list(map(lambda x: x.split(', '), data[1:]))
frmt_str = '%1.6f'
self.assertEqual(frmt_str % 0.5, data[0][0])
self.assertEqual(frmt_str % 0.9, data[0][1])
self.assertEqual(frmt_str % 0.5, data[0][4])
self.assertEqual(frmt_str % 0, data[1][0])
self.assertEqual(frmt_str % 1.3, data[1][1])
self.assertEqual(frmt_str % 2, data[1][4])
self.assertEqual(frmt_str % 0, data[2][0])
self.assertEqual(frmt_str % .7, data[2][1])
self.assertEqual(frmt_str % 3, data[2][4])
# Check aclib2-format
self.assertEqual(json_dicts_aclib2[0]['cpu_time'], .5)
self.assertEqual(json_dicts_aclib2[0]['cost'], 0.9)
self.assertEqual(len(json_dicts_aclib2[0]['incumbent']), 4)
self.assertTrue("param_a='0.5'" in json_dicts_aclib2[0]['incumbent'])
self.assertTrue("param_a='0.0'" in json_dicts_aclib2[2]['incumbent'])
# Check alljson-format
self.assertEqual(json_dicts_alljson[0]['cpu_time'], .5)
self.assertEqual(json_dicts_alljson[0]['cost'], 0.9)
self.assertEqual(len(json_dicts_alljson[0]['incumbent']), 4)
self.assertTrue(json_dicts_alljson[0]["incumbent"]["param_a"] == 0.5)
self.assertTrue(json_dicts_alljson[2]["incumbent"]["param_a"] == 0.0)
self.assertEqual(json_dicts_alljson[0]['budget'], 0)
self.assertEqual(json_dicts_alljson[2]['budget'], 10)
def test_add_entries_multi_objectives(self, mock_stats):
# Mock stats
mock_stats.ta_time_used = .5
mock_stats.get_used_wallclock_time = self.mocked_get_used_wallclock_time
mock_stats.finished_ta_runs = 1
num_obj = 2
with tempfile.TemporaryDirectory() as tmpdir:
tl = TrajLogger(output_dir=tmpdir, stats=mock_stats)
# Add some entries
tl.add_entry([0.9, 0.8], 1, self.test_config, 0)
# Test the list that's added to the trajectory class
self.assertEqual(
tl.trajectory[0],
TrajEntry([0.9, 0.8], 1, self.test_config, 1, 0.5, 1, 0))
# Test named-tuple-access:
self.assertEqual(tl.trajectory[0].train_perf, [0.9, 0.8])
self.assertEqual(len(tl.trajectory), 1)
# Check if the trajectories are generated
for fn in ['traj_old.csv', 'traj_aclib2.json', 'traj.json']:
self.assertTrue(os.path.exists(os.path.join(tmpdir, fn)))
# Load trajectories
with open(os.path.join(tmpdir, 'traj_old.csv')) as to:
data = to.read().split('\n')
with open(os.path.join(tmpdir, 'traj_aclib2.json')) as js_aclib:
json_dicts_aclib2 = [
json.loads(line) for line in js_aclib.read().splitlines()
]
with open(os.path.join(tmpdir, 'traj.json')) as js:
json_dicts_alljson = [
json.loads(line) for line in js.read().splitlines()
]
# Check old format
header = data[0].split(',')
self.assertEqual(header[0], '"CPU Time Used"')
self.assertEqual(header[-1], '"Configuration..."')
data = list(map(lambda x: x.split(', '), data[1:]))
data[0][1] = ', '.join(data[0][1:1 + num_obj])
del data[0][1 + 1:1 + num_obj]
frmt_str = '%1.6f'
self.assertEqual(frmt_str % 0.5, data[0][0])
self.assertEqual(f'[{0.9}, {0.8}]', data[0][1])
self.assertEqual(frmt_str % 0.5, data[0][4])
# Check aclib2-format
self.assertEqual(json_dicts_aclib2[0]['cpu_time'], .5)
self.assertEqual(json_dicts_aclib2[0]['cost'], [0.9, 0.8])
self.assertEqual(len(json_dicts_aclib2[0]['incumbent']), 4)
self.assertTrue("param_a='0.5'" in json_dicts_aclib2[0]['incumbent'])
# Check alljson-format
self.assertEqual(json_dicts_alljson[0]['cpu_time'], .5)
self.assertEqual(json_dicts_alljson[0]['cost'], [0.9, 0.8])
self.assertEqual(len(json_dicts_alljson[0]['incumbent']), 4)
self.assertTrue(json_dicts_alljson[0]["incumbent"]["param_a"] == 0.5)
self.assertEqual(json_dicts_alljson[0]['budget'], 0)