def test_passed_runhistory(self):
''' test if passed runhistory is in resulting runhistory '''
scen = Scenario(self.scen_fn,
cmd_args={'run_obj':'quality',
'instances' : self.train_insts,
'test_instances': self.test_insts})
scen.instance_specific = self.inst_specs
validator = Validator(scen, self.trajectory, self.rng)
# Add a few runs and check, if they are correctly processed
old_configs = [entry["incumbent"] for entry in self.trajectory]
old_rh = RunHistory(average_cost)
seeds = [127 for i in range(int(len(old_configs)/2))]
seeds[-1] = 126 # Test instance_seed-structure in validation
for config in old_configs[:int(len(old_configs)/2)]:
old_rh.add(config, 1, 1, StatusType.SUCCESS, instance_id='0',
seed=seeds[old_configs.index(config)])
configs = validator._get_configs('all')
insts = validator._get_instances('train')
runs_w_rh = validator.get_runs(configs, insts, repetitions=2,
runhistory=old_rh)
runs_wo_rh = validator.get_runs(configs, insts, repetitions=2)
self.assertEqual(len(runs_w_rh[0]), len(runs_wo_rh[0]) - 4)
self.assertEqual(len(runs_w_rh[1].data), 4)
self.assertEqual(len(runs_wo_rh[1].data), 0)
def test_passed_runhistory(self):
''' test if passed runhistory is in resulting runhistory '''
self.scen.train_insts = self.train_insts
self.scen.test_insts = self.test_insts
validator = Validator(self.scen, self.trajectory, self.output_rh,
self.rng)
# Add a few runs and check, if they are correctly processed
old_configs = [entry["incumbent"] for entry in self.trajectory]
old_rh = RunHistory(average_cost)
for config in old_configs[:int(len(old_configs) / 2)]:
old_rh.add(config,
1,
1,
StatusType.SUCCESS,
instance_id='0',
seed=127)
configs = validator._get_configs('all')
insts = validator._get_instances('train')
runs_w_rh = validator.get_runs(configs,
insts,
repetitions=2,
runhistory=old_rh)
runs_wo_rh = validator.get_runs(configs, insts, repetitions=2)
self.assertEqual(len(runs_w_rh), len(runs_wo_rh) - 4)
def test_get_runs(self):
''' test if the runs are generated as expected '''
scen = Scenario(self.scen_fn,
cmd_args={'run_obj':'quality',
'instances' : self.train_insts,
'test_instances': self.test_insts})
scen.instance_specific = self.inst_specs
validator = Validator(scen, self.trajectory, self.rng)
# Get multiple configs
expected = [Run(inst_specs='three', seed=1608637542, inst='3', config='config1'),
Run(inst_specs='three', seed=1608637542, inst='3', config='config2'),
Run(inst_specs='three', seed=1935803228, inst='3', config='config1'),
Run(inst_specs='three', seed=1935803228, inst='3', config='config2'),
Run(inst_specs='four', seed=996406378, inst='4', config='config1'),
Run(inst_specs='four', seed=996406378, inst='4', config='config2'),
Run(inst_specs='four', seed=423734972, inst='4', config='config1'),
Run(inst_specs='four', seed=423734972, inst='4', config='config2'),
Run(inst_specs='five', seed=670094950, inst='5', config='config1'),
Run(inst_specs='five', seed=670094950, inst='5', config='config2'),
Run(inst_specs='five', seed=669991378, inst='5', config='config1'),
Run(inst_specs='five', seed=669991378, inst='5', config='config2')]
runs = validator.get_runs(['config1', 'config2'], scen.test_insts, repetitions=2)
self.assertEqual(runs[0], expected)
# Only train
expected = [Run(inst_specs='null', seed=249467210, inst='0', config='config1'),
Run(inst_specs='null', seed=1572714583, inst='0', config='config1'),
Run(inst_specs='one', seed=434285667, inst='1', config='config1'),
Run(inst_specs='one', seed=893664919, inst='1', config='config1'),
Run(inst_specs='two', seed=88409749, inst='2', config='config1'),
Run(inst_specs='two', seed=2018247425, inst='2', config='config1')]
runs = validator.get_runs(['config1'], scen.train_insts, repetitions=2)
self.assertEqual(runs[0], expected)
# Test and train
expected = [Run(inst='0', seed=1427830251, config='config1', inst_specs='null' ),
Run(inst='1', seed=911989541, config='config1', inst_specs='one' ),
Run(inst='2', seed=780932287, config='config1', inst_specs='two' ),
Run(inst='3', seed=787716372, config='config1', inst_specs='three'),
Run(inst='4', seed=1306710475, config='config1', inst_specs='four' ),
Run(inst='5', seed=106328085, config='config1', inst_specs='five' )]
insts = self.train_insts
insts.extend(self.test_insts)
runs = validator.get_runs(['config1'], insts, repetitions=1)
self.assertEqual(runs[0], expected)
def test_passed_runhistory_no_insts(self):
''' test passed runhistory, without instances '''
scen = Scenario(self.scen_fn,
cmd_args={'run_obj':'quality'})
scen.instance_specific = self.inst_specs
validator = Validator(scen, self.trajectory, self.rng)
# Add a few runs and check, if they are correctly processed
old_configs = [entry["incumbent"] for entry in self.trajectory]
old_rh = RunHistory(average_cost)
for config in old_configs[:int(len(old_configs)/2)]:
old_rh.add(config, 1, 1, StatusType.SUCCESS, seed=127)
configs = validator._get_configs('all')
insts = validator._get_instances('train')
runs_w_rh = validator.get_runs(configs, insts, repetitions=2,
runhistory=old_rh)
runs_wo_rh = validator.get_runs(configs, insts, repetitions=2)
self.assertEqual(len(runs_w_rh[0]), len(runs_wo_rh[0]) - 4)
self.assertEqual(len(runs_w_rh[1].data), 4)
self.assertEqual(len(runs_wo_rh[1].data), 0)
def test_get_runs(self):
''' test if the runs are generated as expected '''
scen = Scenario(self.scen_fn,
cmd_args={
'run_obj': 'quality',
'instances': self.train_insts,
'test_instances': self.test_insts
})
scen.instance_specific = self.inst_specs
validator = Validator(scen, self.trajectory, self.rng)
# Get multiple configs
expected = [
Run(inst_specs='three',
seed=1608637542,
inst='3',
config='config1'),
Run(inst_specs='three',
seed=1608637542,
inst='3',
config='config2'),
Run(inst_specs='three',
seed=1273642419,
inst='3',
config='config1'),
Run(inst_specs='three',
seed=1273642419,
inst='3',
config='config2'),
Run(inst_specs='four', seed=1935803228, inst='4',
config='config1'),
Run(inst_specs='four', seed=1935803228, inst='4',
config='config2'),
Run(inst_specs='four', seed=787846414, inst='4', config='config1'),
Run(inst_specs='four', seed=787846414, inst='4', config='config2'),
Run(inst_specs='five', seed=996406378, inst='5', config='config1'),
Run(inst_specs='five', seed=996406378, inst='5', config='config2'),
Run(inst_specs='five', seed=1201263687, inst='5',
config='config1'),
Run(inst_specs='five', seed=1201263687, inst='5', config='config2')
]
runs = validator.get_runs(['config1', 'config2'],
scen.test_insts,
repetitions=2)
self.assertEqual(runs[0], expected)
# Only train
expected = [
Run(inst_specs='null', seed=423734972, inst='0', config='config1'),
Run(inst_specs='null', seed=415968276, inst='0', config='config1'),
Run(inst_specs='one', seed=670094950, inst='1', config='config1'),
Run(inst_specs='one', seed=1914837113, inst='1', config='config1'),
Run(inst_specs='two', seed=669991378, inst='2', config='config1'),
Run(inst_specs='two', seed=429389014, inst='2', config='config1')
]
runs = validator.get_runs(['config1'], scen.train_insts, repetitions=2)
self.assertEqual(runs[0], expected)
# Test and train
expected = [
Run(inst='0', seed=249467210, config='config1', inst_specs='null'),
Run(inst='1', seed=1972458954, config='config1', inst_specs='one'),
Run(inst='2', seed=1572714583, config='config1', inst_specs='two'),
Run(inst='3',
seed=1433267572,
config='config1',
inst_specs='three'),
Run(inst='4', seed=434285667, config='config1', inst_specs='four'),
Run(inst='5', seed=613608295, config='config1', inst_specs='five')
]
insts = self.train_insts
insts.extend(self.test_insts)
runs = validator.get_runs(['config1'], insts, repetitions=1)
self.assertEqual(runs[0], expected)
def test_get_runs(self):
''' test if the runs are generated as expected '''
self.scen.train_insts = self.train_insts
self.scen.test_insts = self.test_insts
validator = Validator(self.scen, self.trajectory, self.output_rh,
self.rng)
# Get multiple configs
expected = [{
'inst_specs': 'three',
'seed': 1608637542,
'inst': '3',
'config': 'config1'
}, {
'inst_specs': 'three',
'seed': 1608637542,
'inst': '3',
'config': 'config2'
}, {
'inst_specs': 'three',
'seed': 1273642419,
'inst': '3',
'config': 'config1'
}, {
'inst_specs': 'three',
'seed': 1273642419,
'inst': '3',
'config': 'config2'
}, {
'inst_specs': 'four',
'seed': 1935803228,
'inst': '4',
'config': 'config1'
}, {
'inst_specs': 'four',
'seed': 1935803228,
'inst': '4',
'config': 'config2'
}, {
'inst_specs': 'four',
'seed': 787846414,
'inst': '4',
'config': 'config1'
}, {
'inst_specs': 'four',
'seed': 787846414,
'inst': '4',
'config': 'config2'
}, {
'inst_specs': 'five',
'seed': 996406378,
'inst': '5',
'config': 'config1'
}, {
'inst_specs': 'five',
'seed': 996406378,
'inst': '5',
'config': 'config2'
}, {
'inst_specs': 'five',
'seed': 1201263687,
'inst': '5',
'config': 'config1'
}, {
'inst_specs': 'five',
'seed': 1201263687,
'inst': '5',
'config': 'config2'
}]
runs = validator.get_runs(['config1', 'config2'],
self.scen.test_insts,
repetitions=2)
self.assertEqual(runs, expected)
# Only train
expected = [{
'inst_specs': 'null',
'seed': 423734972,
'inst': '0',
'config': 'config1'
}, {
'inst_specs': 'null',
'seed': 415968276,
'inst': '0',
'config': 'config1'
}, {
'inst_specs': 'one',
'seed': 670094950,
'inst': '1',
'config': 'config1'
}, {
'inst_specs': 'one',
'seed': 1914837113,
'inst': '1',
'config': 'config1'
}, {
'inst_specs': 'two',
'seed': 669991378,
'inst': '2',
'config': 'config1'
}, {
'inst_specs': 'two',
'seed': 429389014,
'inst': '2',
'config': 'config1'
}]
runs = validator.get_runs(['config1'],
self.scen.train_insts,
repetitions=2)
self.assertEqual(runs, expected)
# Test and train
expected = [{
'inst': '0',
'seed': 249467210,
'config': 'config1',
'inst_specs': 'null'
}, {
'inst': '1',
'seed': 1972458954,
'config': 'config1',
'inst_specs': 'one'
}, {
'inst': '2',
'seed': 1572714583,
'config': 'config1',
'inst_specs': 'two'
}, {
'inst': '3',
'seed': 1433267572,
'config': 'config1',
'inst_specs': 'three'
}, {
'inst': '4',
'seed': 434285667,
'config': 'config1',
'inst_specs': 'four'
}, {
'inst': '5',
'seed': 613608295,
'config': 'config1',
'inst_specs': 'five'
}]
insts = self.train_insts
insts.extend(self.test_insts)
runs = validator.get_runs(['config1'], insts, repetitions=1)
self.assertEqual(runs, expected)
