def test_pass_tae(self): scen = Scenario(self.scen_fn, cmd_options={'run_obj': 'quality'}) tae = ExecuteTARunOld(ta=scen.ta, stats=self.stats) validator = Validator(scen, self.trajectory) rh_mock = mock.Mock() with mock.patch.object( Validator, "_validate_parallel", return_value=[ mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), mock.MagicMock() ], ) as validate_parallel_mock: with mock.patch.object( Validator, "_get_runs", return_value=[[ mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock() ], rh_mock], ): validator.validate(tae=tae) self.assertIs(validate_parallel_mock.call_args[0][0], tae) self.assertEqual(rh_mock.add.call_count, 4)
def validate(self, config_mode='inc', instance_mode='train+test', repetitions=1, n_jobs=-1, backend='threading'): """Create validator-object and run validation, using scenario-information, runhistory from smbo and tae_runner from intensify Parameters ---------- config_mode: string what configurations to validate from [def, inc, def+inc, time, all], time means evaluation at timesteps 2^-4, 2^-3, 2^-2, 2^-1, 2^0, 2^1, ... instance_mode: string what instances to use for validation, from [train, test, train+test] repetitions: int number of repetitions in nondeterministic algorithms (in deterministic will be fixed to 1) n_jobs: int number of parallel processes used by joblib Returns ------- runhistory: RunHistory runhistory containing all specified runs """ traj_fn = os.path.join(self.scenario.output_dir, "traj_aclib2.json") trajectory = TrajLogger.read_traj_aclib_format(fn=traj_fn, cs=self.scenario.cs) new_rh_path = os.path.join(self.scenario.output_dir, "validated_runhistory.json") validator = Validator(self.scenario, trajectory, new_rh_path, self.rng) new_rh = validator.validate(config_mode, instance_mode, repetitions, n_jobs, backend, self.runhistory, self.intensifier.tae_runner) return new_rh
def test_parallel(self): ''' test parallel ''' scen = Scenario(self.scen_fn, cmd_options={'run_obj': 'quality'}) validator = Validator(scen, self.trajectory, self.rng) validator.validate(config_mode='all', instance_mode='train+test', n_jobs=-1)
def test_get_runs_capped(self): ''' test if capped, crashed and aborted runs are ignored during rh-recovery ''' scen = Scenario(self.scen_fn, cmd_options={'run_obj': 'quality', 'instances': ['0']}) validator = Validator(scen, self.trajectory, self.rng) # Get runhistory old_configs = [Configuration(scen.cs, values={'x1': i, 'x2': i}) for i in range(1, 7)] old_rh = RunHistory() old_rh.add(old_configs[0], 1, 1, StatusType.SUCCESS, instance_id='0', seed=0) old_rh.add(old_configs[1], 1, 1, StatusType.TIMEOUT, instance_id='0', seed=0) old_rh.add(old_configs[2], 1, 1, StatusType.CRASHED, instance_id='0', seed=0) old_rh.add(old_configs[3], 1, 1, StatusType.ABORT, instance_id='0', seed=0) old_rh.add(old_configs[4], 1, 1, StatusType.MEMOUT, instance_id='0', seed=0) old_rh.add(old_configs[5], 1, 1, StatusType.CAPPED, instance_id='0', seed=0) # Get multiple configs expected = [_Run(inst_specs='0', seed=0, inst='0', config=old_configs[2]), _Run(inst_specs='0', seed=0, inst='0', config=old_configs[3]), _Run(inst_specs='0', seed=0, inst='0', config=old_configs[5])] runs = validator._get_runs(old_configs, ['0'], repetitions=1, runhistory=old_rh) self.assertEqual(runs[0], expected)
def test_inst_no_feat(self): ''' test if scenarios are treated correctly if no features are specified.''' scen = Scenario(self.scen_fn, cmd_options={ 'run_obj': 'quality', 'train_insts': self.train_insts, 'test_insts': self.test_insts }) self.assertTrue(scen.feature_array is None) self.assertEqual(len(scen.feature_dict), 0) scen.instance_specific = self.inst_specs validator = Validator(scen, self.trajectory, self.rng) # Add a few runs and check, if they are correctly processed old_configs = [entry["incumbent"] for entry in self.trajectory] old_rh = RunHistory() for config in old_configs[:int(len(old_configs) / 2)]: old_rh.add(config, 1, 1, StatusType.SUCCESS, instance_id='0', seed=127) rh = validator.validate_epm('all', 'train+test', 1, old_rh) self.assertEqual(len(old_rh.get_all_configs()), 4) self.assertEqual(len(rh.get_all_configs()), 10)
def optimize(optimizer, scenario, trajectory=None): then = time.time() best_conf = optimizer.optimize() print(best_conf) print('training time:', time.time() - then) traj_logger = TrajLogger(None, Stats(scenario)) trajectory = trajectory or traj_logger.read_traj_aclib_format( "smac-output/run_1/traj_aclib2.json", scenario.cs) validator = Validator(scenario, trajectory, rng=np.random.RandomState(42)) # evaluate on test instances and calculate cpu time then = time.time() runhis_dev = validator.validate(config_mode="def", instance_mode="test") runhis_inc = validator.validate(config_mode="inc", instance_mode="test") print('validating time:', time.time() - then) default_conf = runhis_dev.ids_config[1] incumbent_conf = runhis_inc.ids_config[1] dev_vals = get_instance_costs_for_config(runhis_dev, default_conf) inc_vals = get_instance_costs_for_config(runhis_inc, incumbent_conf) # ###### Filter runs for plotting ####### dev_x = [] inc_x = [] for key in set(dev_vals.keys()) & set(inc_vals.keys()): dev_x.append(dev_vals[key]) inc_x.append(inc_vals[key]) # print(dev_vals) # print(inc_vals) print(dev_x) print(inc_x) print('PAR10:', np.mean(inc_x), '/', np.mean(dev_x)) max_x = 1000.0 par1er = lambda xx: np.mean([(x / 10 if x == max_x else x) for x in xx]) print('PAR1 :', par1er(inc_x), '/', par1er(dev_x)) to_counter = lambda xx: len([x for x in xx if x == max_x]) print('TOs :', to_counter(inc_x), '/', to_counter(dev_x)) print('wins :', len([i for i in range(len(dev_x)) if dev_x[i] > inc_x[i]]), '/', len(dev_x)) fig, ax = plt.subplots() ax.scatter(dev_x, inc_x, marker="x") ax.set_xlabel("Default Configuration") ax.set_ylabel("Incumbent Configuration") lims = [ np.min([ax.get_xlim(), ax.get_ylim()]), np.max([ax.get_xlim(), ax.get_ylim()]) ] ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0) # ax.set_xlim(lims) # ax.set_ylim(lims) ax.set_xscale('log') ax.set_yscale('log') fig.savefig("fig-smac.png")
def test_parallel(self): ''' test parallel ''' validator = Validator(self.scen, self.trajectory, self.output_rh, self.rng) validator.validate(config_mode='all', instance_mode='train+test', n_jobs=-1)
def test_pass_tae(self): scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality'}) tae = ExecuteTARunOld(ta=scen.ta) validator = Validator(scen, self.trajectory) with mock.patch.object(Validator, "_validate_parallel", return_value=[(1,2,3,4)]): self.assertEqual(1, len(validator.validate(tae=tae).data))
def _init_pimp_and_validator(self, rh, alternative_output_dir=None): """Create ParameterImportance-object and use it's trained model for validation and further predictions We pass validated runhistory, so that the returned model will be based on as much information as possible Parameters ---------- rh: RunHistory runhistory used to build EPM alternative_output_dir: str e.g. for budgets we want pimp to use an alternative output-dir (subfolders per budget) """ self.logger.debug( "Using '%s' as output for pimp", alternative_output_dir if alternative_output_dir else self.output_dir) self.pimp = Importance( scenario=copy.deepcopy(self.scenario), runhistory=rh, incumbent=self.default, # Inject correct incumbent later parameters_to_evaluate=4, save_folder=alternative_output_dir if alternative_output_dir else self.output_dir, seed=self.rng.randint(1, 100000), max_sample_size=self.pimp_max_samples, fANOVA_pairwise=self.fanova_pairwise, preprocess=False) self.model = self.pimp.model # Validator (initialize without trajectory) self.validator = Validator(self.scenario, None, None) self.validator.epm = self.model
def _init_pimp_and_validator( self, alternative_output_dir=None, ): """Create ParameterImportance-object and use it's trained model for validation and further predictions. We pass a combined (original + validated) runhistory, so that the returned model will be based on as much information as possible Parameters ---------- alternative_output_dir: str e.g. for budgets we want pimp to use an alternative output-dir (subfolders per budget) """ self.logger.debug( "Using '%s' as output for pimp", alternative_output_dir if alternative_output_dir else self.output_dir) self.pimp = Importance( scenario=copy.deepcopy(self.scenario), runhistory=self.combined_runhistory, incumbent=self.incumbent if self.incumbent else self.default, save_folder=alternative_output_dir if alternative_output_dir is not None else self.output_dir, seed=self.rng.randint(1, 100000), max_sample_size=self.options['fANOVA'].getint("pimp_max_samples"), fANOVA_pairwise=self.options['fANOVA'].getboolean( "fanova_pairwise"), preprocess=False, verbose=1, # disable progressbars ) # Validator (initialize without trajectory) self.validator = Validator(self.scenario, None, None) self.validator.epm = self.pimp.model
def validate(self, config_mode: typing.Union[str, typing.List[Configuration]] = 'inc', instance_mode: typing.Union[str, typing.List[str]] = 'train+test', repetitions: int = 1, use_epm: bool = False, n_jobs: int = -1, backend: str = 'threading') -> RunHistory: """Create validator-object and run validation, using scenario-information, runhistory from smbo and tae_runner from intensify Parameters ---------- config_mode: str or list<Configuration> string or directly a list of Configuration str from [def, inc, def+inc, wallclock_time, cpu_time, all] time evaluates at cpu- or wallclock-timesteps of: [max_time/2^0, max_time/2^1, max_time/2^3, ..., default] with max_time being the highest recorded time instance_mode: string what instances to use for validation, from [train, test, train+test] repetitions: int number of repetitions in nondeterministic algorithms (in deterministic will be fixed to 1) use_epm: bool whether to use an EPM instead of evaluating all runs with the TAE n_jobs: int number of parallel processes used by joblib Returns ------- runhistory: RunHistory runhistory containing all specified runs """ if isinstance(config_mode, str): assert self.scenario.output_dir_for_this_run is not None # Please mypy traj_fn = os.path.join(self.scenario.output_dir_for_this_run, "traj_aclib2.json") trajectory = ( TrajLogger.read_traj_aclib_format(fn=traj_fn, cs=self.config_space) ) # type: typing.Optional[typing.List[typing.Dict[str, typing.Union[float, int, Configuration]]]] else: trajectory = None if self.scenario.output_dir_for_this_run: new_rh_path = os.path.join(self.scenario.output_dir_for_this_run, "validated_runhistory.json") # type: typing.Optional[str] # noqa E501 else: new_rh_path = None validator = Validator(self.scenario, trajectory, self.rng) if use_epm: new_rh = validator.validate_epm(config_mode=config_mode, instance_mode=instance_mode, repetitions=repetitions, runhistory=self.runhistory, output_fn=new_rh_path) else: new_rh = validator.validate(config_mode, instance_mode, repetitions, n_jobs, backend, self.runhistory, self.intensifier.tae_runner, output_fn=new_rh_path) return new_rh
def test_rng(self): scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality'}) validator = Validator(scen, self.trajectory, 42) self.assertTrue(isinstance(validator.rng, np.random.RandomState)) validator = Validator(scen, self.trajectory) self.assertTrue(isinstance(validator.rng, np.random.RandomState)) validator = Validator(scen, self.trajectory, np.random.RandomState()) self.assertTrue(isinstance(validator.rng, np.random.RandomState))
def test_no_feature_dict(self): scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality'}) scen.feature_array = None validator = Validator(scen, self.trajectory) old_rh = RunHistory(average_cost) for config in [e["incumbent"] for e in self.trajectory]: old_rh.add(config, 1, 1, StatusType.SUCCESS, instance_id='0', seed=127) validator.validate_epm(runhistory=old_rh)
def test_validate_no_insts(self): ''' no instances ''' scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality'}) validator = Validator(scen, self.trajectory, self.rng) rh = validator.validate(config_mode='def+inc', instance_mode='train', repetitions=3, output_fn=self.output_rh) self.assertEqual(len(rh.get_all_configs()), 2) self.assertEqual(sum([len(rh.get_runs_for_config(c)) for c in rh.get_all_configs()]), 6)
def test_objective_runtime(self): ''' test if everything is ok with objective runtime (imputing!) ''' scen = Scenario(self.scen_fn, cmd_args={'run_obj' : 'runtime', 'cutoff_time' : 5}) validator = Validator(scen, self.trajectory, self.rng) old_configs = [entry["incumbent"] for entry in self.trajectory] old_rh = RunHistory(average_cost) for config in old_configs[:int(len(old_configs)/2)]: old_rh.add(config, 1, 1, StatusType.SUCCESS, instance_id='0') validator.validate_epm('all', 'train', 1, old_rh)
def test_validate_no_insts(self): ''' no instances ''' validator = Validator(self.scen, self.trajectory, self.output_rh, self.rng) rh = validator.validate(config_mode='def+inc', instance_mode='train', repetitions=3) self.assertEqual(len(rh.get_all_configs()), 2) self.assertEqual( sum([len(rh.get_runs_for_config(c)) for c in rh.get_all_configs()]), 6)
def test_validate_deterministic(self): ''' deterministic ta ''' self.scen.deterministic = True self.scen.train_insts = self.train_insts validator = Validator(self.scen, self.trajectory, self.output_rh, self.rng) rh = validator.validate(config_mode='def+inc', instance_mode='train', repetitions=3) self.assertEqual(len(rh.get_all_configs()), 2) self.assertEqual( sum([len(rh.get_runs_for_config(c)) for c in rh.get_all_configs()]), 6)
def test_validate_deterministic(self): ''' deterministic ta ''' scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality', 'instances' : self.train_insts, 'deterministic': True}) scen.instance_specific = self.inst_specs validator = Validator(scen, self.trajectory, self.rng) rh = validator.validate(config_mode='def+inc', instance_mode='train', repetitions=3) self.assertEqual(len(rh.get_all_configs()), 2) self.assertEqual(sum([len(rh.get_runs_for_config(c)) for c in rh.get_all_configs()]), 6)
def test_get_runs(self): ''' test if the runs are generated as expected ''' scen = Scenario(self.scen_fn, cmd_options={'run_obj': 'quality', 'train_insts': self.train_insts, 'test_insts': self.test_insts}) scen.instance_specific = self.inst_specs validator = Validator(scen, self.trajectory, self.rng) # Get multiple configs self.maxDiff = None expected = [_Run(config='config1', inst='3', seed=1608637542, inst_specs='three'), _Run(config='config2', inst='3', seed=1608637542, inst_specs='three'), _Run(config='config1', inst='3', seed=1273642419, inst_specs='three'), _Run(config='config2', inst='3', seed=1273642419, inst_specs='three'), _Run(config='config1', inst='4', seed=1935803228, inst_specs='four'), _Run(config='config2', inst='4', seed=1935803228, inst_specs='four'), _Run(config='config1', inst='4', seed=787846414, inst_specs='four'), _Run(config='config2', inst='4', seed=787846414, inst_specs='four'), _Run(config='config1', inst='5', seed=996406378, inst_specs='five'), _Run(config='config2', inst='5', seed=996406378, inst_specs='five'), _Run(config='config1', inst='5', seed=1201263687, inst_specs='five'), _Run(config='config2', inst='5', seed=1201263687, inst_specs='five')] runs = validator._get_runs(['config1', 'config2'], scen.test_insts, repetitions=2) self.assertEqual(runs[0], expected) # Only train expected = [_Run(config='config1', inst='0', seed=423734972, inst_specs='null'), _Run(config='config1', inst='0', seed=415968276, inst_specs='null'), _Run(config='config1', inst='1', seed=670094950, inst_specs='one'), _Run(config='config1', inst='1', seed=1914837113, inst_specs='one'), _Run(config='config1', inst='2', seed=669991378, inst_specs='two'), _Run(config='config1', inst='2', seed=429389014, inst_specs='two')] runs = validator._get_runs(['config1'], scen.train_insts, repetitions=2) self.assertEqual(runs[0], expected) # Test and train expected = [_Run(config='config1', inst='0', seed=249467210, inst_specs='null'), _Run(config='config1', inst='1', seed=1972458954, inst_specs='one'), _Run(config='config1', inst='2', seed=1572714583, inst_specs='two'), _Run(config='config1', inst='3', seed=1433267572, inst_specs='three'), _Run(config='config1', inst='4', seed=434285667, inst_specs='four'), _Run(config='config1', inst='5', seed=613608295, inst_specs='five')] insts = self.train_insts insts.extend(self.test_insts) runs = validator._get_runs(['config1'], insts, repetitions=1) self.assertEqual(runs[0], expected)
def test_get_runs_capped(self): ''' test if capped, crashed and aborted runs are ignored during rh-recovery ''' scen = Scenario(self.scen_fn, cmd_options={ 'run_obj': 'quality', 'instances': ['0'] }) validator = Validator(scen, self.trajectory, self.rng) # Get runhistory old_configs = [ 'config1', 'config2', 'config3', 'config4', 'config5', 'config6' ] old_rh = RunHistory(average_cost) old_rh.add('config1', 1, 1, StatusType.SUCCESS, instance_id='0', seed=0) old_rh.add('config2', 1, 1, StatusType.TIMEOUT, instance_id='0', seed=0) old_rh.add('config3', 1, 1, StatusType.CRASHED, instance_id='0', seed=0) old_rh.add('config4', 1, 1, StatusType.ABORT, instance_id='0', seed=0) old_rh.add('config5', 1, 1, StatusType.MEMOUT, instance_id='0', seed=0) old_rh.add('config6', 1, 1, StatusType.CAPPED, instance_id='0', seed=0) # Get multiple configs expected = [ _Run(inst_specs='0', seed=0, inst='0', config='config3'), _Run(inst_specs='0', seed=0, inst='0', config='config4'), _Run(inst_specs='0', seed=0, inst='0', config='config6') ] runs = validator._get_runs(old_configs, ['0'], repetitions=1, runhistory=old_rh) self.assertEqual(runs[0], expected)
def test_get_runs(self): ''' test if the runs are generated as expected ''' scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality', 'instances' : self.train_insts, 'test_instances': self.test_insts}) scen.instance_specific = self.inst_specs validator = Validator(scen, self.trajectory, self.rng) # Get multiple configs expected = [Run(inst_specs='three', seed=1608637542, inst='3', config='config1'), Run(inst_specs='three', seed=1608637542, inst='3', config='config2'), Run(inst_specs='three', seed=1935803228, inst='3', config='config1'), Run(inst_specs='three', seed=1935803228, inst='3', config='config2'), Run(inst_specs='four', seed=996406378, inst='4', config='config1'), Run(inst_specs='four', seed=996406378, inst='4', config='config2'), Run(inst_specs='four', seed=423734972, inst='4', config='config1'), Run(inst_specs='four', seed=423734972, inst='4', config='config2'), Run(inst_specs='five', seed=670094950, inst='5', config='config1'), Run(inst_specs='five', seed=670094950, inst='5', config='config2'), Run(inst_specs='five', seed=669991378, inst='5', config='config1'), Run(inst_specs='five', seed=669991378, inst='5', config='config2')] runs = validator.get_runs(['config1', 'config2'], scen.test_insts, repetitions=2) self.assertEqual(runs[0], expected) # Only train expected = [Run(inst_specs='null', seed=249467210, inst='0', config='config1'), Run(inst_specs='null', seed=1572714583, inst='0', config='config1'), Run(inst_specs='one', seed=434285667, inst='1', config='config1'), Run(inst_specs='one', seed=893664919, inst='1', config='config1'), Run(inst_specs='two', seed=88409749, inst='2', config='config1'), Run(inst_specs='two', seed=2018247425, inst='2', config='config1')] runs = validator.get_runs(['config1'], scen.train_insts, repetitions=2) self.assertEqual(runs[0], expected) # Test and train expected = [Run(inst='0', seed=1427830251, config='config1', inst_specs='null' ), Run(inst='1', seed=911989541, config='config1', inst_specs='one' ), Run(inst='2', seed=780932287, config='config1', inst_specs='two' ), Run(inst='3', seed=787716372, config='config1', inst_specs='three'), Run(inst='4', seed=1306710475, config='config1', inst_specs='four' ), Run(inst='5', seed=106328085, config='config1', inst_specs='five' )] insts = self.train_insts insts.extend(self.test_insts) runs = validator.get_runs(['config1'], insts, repetitions=1) self.assertEqual(runs[0], expected)
def test_validate_epm(self): ''' test using epm to validate ''' scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality', 'instances' : self.train_insts, 'test_instances': self.test_insts, 'features': self.feature_dict}) scen.instance_specific = self.inst_specs validator = Validator(scen, self.trajectory, self.rng) # Add a few runs and check, if they are correctly processed old_configs = [entry["incumbent"] for entry in self.trajectory] old_rh = RunHistory(average_cost) for config in old_configs[:int(len(old_configs)/2)]: old_rh.add(config, 1, 1, StatusType.SUCCESS, instance_id='0', seed=127) validator.validate_epm('all', 'train', 1, old_rh)
def test_epm_reuse_rf(self): """ if no runhistory is passed to epm, but there was a model trained before, that model should be reused! (if reuse_epm flag is set) """ scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality'}) scen.feature_array = None validator = Validator(scen, self.trajectory) old_rh = RunHistory(average_cost) for config in [e["incumbent"] for e in self.trajectory]: old_rh.add(config, 1, 1, StatusType.SUCCESS, instance_id='0', seed=127) self.assertTrue(isinstance(validator.validate_epm(runhistory=old_rh), RunHistory)) self.assertTrue(isinstance(validator.validate_epm( output_fn="test/test_files/validation/"), RunHistory)) self.assertRaises(ValueError, validator.validate_epm, reuse_epm=False)
def test_validate_no_insts(self): ''' no instances ''' scen = Scenario(self.scen_fn, cmd_options={ 'run_obj': 'quality', 'save-instantly': False, 'deterministic': False, }) validator = Validator(scen, self.trajectory, self.rng) rh = validator.validate(config_mode='def+inc', instance_mode='train', repetitions=3, output_fn=self.output_rh) self.assertEqual(len(rh.get_all_configs()), 2) self.assertEqual( sum([ len(rh.get_runs_for_config(c, only_max_observed_budget=True)) for c in rh.get_all_configs() ]), 6)
def test_get_configs(self): scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality'}) validator = Validator(scen, self.trajectory, self.rng) self.assertEqual(1, len(validator._get_configs("def"))) self.assertEqual(1, len(validator._get_configs("inc"))) self.assertEqual(2, len(validator._get_configs("def+inc"))) self.assertEqual(7, len(validator._get_configs("wallclock_time"))) self.assertEqual(8, len(validator._get_configs("cpu_time"))) self.assertEqual(10, len(validator._get_configs("all"))) # Using maxtime validator.scen.wallclock_limit = 65 validator.scen.algo_runs_timelimit = 33 self.assertEqual(8, len(validator._get_configs("wallclock_time"))) self.assertEqual(9, len(validator._get_configs("cpu_time"))) # Exceptions self.assertRaises(ValueError, validator._get_configs, "notanoption") self.assertRaises(ValueError, validator._get_instances, "notanoption")
def test_validate(self): ''' test validation ''' self.scen.train_insts = self.train_insts self.scen.test_insts = self.test_insts validator = Validator(self.scen, self.trajectory, self.output_rh, self.rng) # Test basic usage rh = validator.validate(config_mode='def', instance_mode='test', repetitions=3) self.assertEqual(len(rh.get_all_configs()), 1) self.assertEqual(len(rh.get_runs_for_config(rh.get_all_configs()[0])), 9) rh = validator.validate(config_mode='inc', instance_mode='train+test') self.assertEqual(len(rh.get_all_configs()), 1) self.assertEqual(len(rh.get_runs_for_config(rh.get_all_configs()[0])), 6) rh = validator.validate(config_mode='time', instance_mode='train') self.assertEqual(len(rh.get_all_configs()), 9) self.assertEqual( sum([len(rh.get_runs_for_config(c)) for c in rh.get_all_configs()]), 27) # Test with backend multiprocessing rh = validator.validate(config_mode='def', instance_mode='test', repetitions=3, backend='multiprocessing') self.assertEqual(len(rh.get_all_configs()), 1) self.assertEqual(len(rh.get_runs_for_config(rh.get_all_configs()[0])), 9)
def test_passed_runhistory(self): ''' test if passed runhistory is in resulting runhistory ''' self.scen.train_insts = self.train_insts self.scen.test_insts = self.test_insts validator = Validator(self.scen, self.trajectory, self.output_rh, self.rng) # Add a few runs and check, if they are correctly processed old_configs = [entry["incumbent"] for entry in self.trajectory] old_rh = RunHistory(average_cost) for config in old_configs[:int(len(old_configs) / 2)]: old_rh.add(config, 1, 1, StatusType.SUCCESS, instance_id='0', seed=127) configs = validator._get_configs('all') insts = validator._get_instances('train') runs_w_rh = validator.get_runs(configs, insts, repetitions=2, runhistory=old_rh) runs_wo_rh = validator.get_runs(configs, insts, repetitions=2) self.assertEqual(len(runs_w_rh), len(runs_wo_rh) - 4)
def test_validate(self): ''' test validation ''' scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality', 'instances' : self.train_insts, 'test_instances': self.test_insts}) scen.instance_specific = self.inst_specs validator = Validator(scen, self.trajectory, self.rng) # Test basic usage rh = validator.validate(config_mode='def', instance_mode='test', repetitions=3) self.assertEqual(len(rh.get_all_configs()), 1) self.assertEqual(len(rh.get_runs_for_config(rh.get_all_configs()[0])), 9) rh = validator.validate(config_mode='inc', instance_mode='train+test') self.assertEqual(len(rh.get_all_configs()), 1) self.assertEqual(len(rh.get_runs_for_config(rh.get_all_configs()[0])), 6) rh = validator.validate(config_mode='wallclock_time', instance_mode='train') self.assertEqual(len(rh.get_all_configs()), 7) self.assertEqual(sum([len(rh.get_runs_for_config(c)) for c in rh.get_all_configs()]), 21) # Test with backend multiprocessing rh = validator.validate(config_mode='def', instance_mode='test', repetitions=3, backend='multiprocessing') self.assertEqual(len(rh.get_all_configs()), 1) self.assertEqual(len(rh.get_runs_for_config(rh.get_all_configs()[0])), 9)
def test_passed_runhistory_deterministic(self): ''' test if passed runhistory is in resulting runhistory ''' scen = Scenario(self.scen_fn, cmd_options={ 'run_obj': 'quality', 'train_insts': self.train_insts, 'deterministic': True }) scen.instance_specific = self.inst_specs validator = Validator(scen, self.trajectory, self.rng) # Add a few runs and check, if they are correctly processed old_configs = [entry["incumbent"] for entry in self.trajectory] old_rh = RunHistory() for config in old_configs[:int(len(old_configs) / 2)]: old_rh.add(config, 1, 1, StatusType.SUCCESS, instance_id='0') configs = validator._get_configs('all') insts = validator._get_instances('train') runs_w_rh = validator._get_runs(configs, insts, repetitions=2, runhistory=old_rh) runs_wo_rh = validator._get_runs(configs, insts, repetitions=2) self.assertEqual(len(runs_w_rh[0]), len(runs_wo_rh[0]) - 4) self.assertEqual(len(runs_w_rh[1].data), 4) self.assertEqual(len(runs_wo_rh[1].data), 0)
def test_passed_runhistory(self): ''' test if passed runhistory is in resulting runhistory ''' scen = Scenario(self.scen_fn, cmd_args={'run_obj':'quality', 'instances' : self.train_insts, 'test_instances': self.test_insts}) scen.instance_specific = self.inst_specs validator = Validator(scen, self.trajectory, self.rng) # Add a few runs and check, if they are correctly processed old_configs = [entry["incumbent"] for entry in self.trajectory] old_rh = RunHistory(average_cost) seeds = [127 for i in range(int(len(old_configs)/2))] seeds[-1] = 126 # Test instance_seed-structure in validation for config in old_configs[:int(len(old_configs)/2)]: old_rh.add(config, 1, 1, StatusType.SUCCESS, instance_id='0', seed=seeds[old_configs.index(config)]) configs = validator._get_configs('all') insts = validator._get_instances('train') runs_w_rh = validator.get_runs(configs, insts, repetitions=2, runhistory=old_rh) runs_wo_rh = validator.get_runs(configs, insts, repetitions=2) self.assertEqual(len(runs_w_rh[0]), len(runs_wo_rh[0]) - 4) self.assertEqual(len(runs_w_rh[1].data), 4) self.assertEqual(len(runs_wo_rh[1].data), 0)