def test_race_challenger_2(self): """ Makes sure that a racing configuration with better performance, that is capped, doesn't substitute the incumbent. """ def target(x): time.sleep(1.5) return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], ) self.rh.add( config=self.config1, cost=0.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_2(self): ''' test _race_challenger with adaptive capping ''' def target(x): time.sleep(1.5) return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config1)
def test_run(self): """Makes sure that we are able to run a configuration and return the expected values/types""" # We use the funcdict as a mechanism to test SerialRunner runner = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj='quality') self.assertIsInstance(runner, SerialRunner) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) # submit runs! then get the value runner.submit_run(run_info) run_values = runner.get_finished_runs() self.assertEqual(len(run_values), 1) self.assertIsInstance(run_values, list) self.assertIsInstance(run_values[0][0], RunInfo) self.assertIsInstance(run_values[0][1], RunValue) self.assertEqual(run_values[0][1].cost, 4) self.assertEqual(run_values[0][1].status, StatusType.SUCCESS)
def test_add_inc_run_nondet(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data)
def test_add_inc_run_det(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data)
def test_race_challenger(self): ''' test _race_challenger without adaptive capping ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) self.assertEqual(inc, self.config2)
def test_memout(self): def fill_memory(*args): a = np.random.random_sample((10000, 10000)).astype(np.float64) return np.sum(a) taf = ExecuteTAFuncDict(ta=fill_memory, stats=self.stats, memory_limit=1024) rval = taf.run(config=None) platform = os.getenv('TRAVIS_OS_NAME') if platform is None: platform = {'linux': 'linux', 'darwin': 'osx'}.get(sys.platform) print(platform, sys.platform) if platform == 'linux': self.assertEqual(rval[0], StatusType.MEMOUT) self.assertEqual(rval[1], 2147483647.0) self.assertGreaterEqual(rval[2], 0.0) self.assertEqual(rval[3], dict()) elif platform == 'osx': self.assertEqual(rval[0], StatusType.SUCCESS) else: raise ValueError('Test cannot be performed on platform %s' % sys.platform)
def test_add_inc_run_det(self): """ test _add_inc_run() """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run # as it is the first evaluation of this intensifier self.assertEqual(intensifier.num_run, 0) self.assertEqual(intensifier.num_chall_run, 0)
def test_race_challenger_large_blocked_seed(self): """ test _race_challenger whether seeds are blocked for challenger runs """ def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=False) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10)
def test_timeout(self): def run_over_time(*args): time.sleep(5) taf = ExecuteTAFuncDict(ta=run_over_time, stats=self.stats) rval = taf.run(config=None, cutoff=1) self.assertEqual(rval[0], StatusType.TIMEOUT) self.assertEqual(rval[1], 2147483647.0) self.assertGreaterEqual(rval[2], 0.0) self.assertEqual(rval[3], dict())
def test_fail_silent(self): def function(*args): return taf = ExecuteTAFuncDict(ta=function, stats=self.stats) rval = taf.run(config=None, cutoff=1) self.assertEqual(rval[0], StatusType.CRASHED) self.assertEqual(rval[1], 2147483647.0) self.assertGreaterEqual(rval[2], 0.0) self.assertEqual(rval[3], dict())
def test_serial_runs(self): # We use the funcdict as a mechanism to test SerialRunner runner = ExecuteTAFuncDict(ta=target_delayed, stats=self.stats, run_obj='quality') self.assertIsInstance(runner, SerialRunner) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) run_info = RunInfo(config=3, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) run_values = runner.get_finished_runs() self.assertEqual(len(run_values), 2) # To make sure runs launched serially, we just make sure that the end time of # a run is later than the other # Results are returned in left to right self.assertLessEqual(int(run_values[1][1].endtime), int(run_values[0][1].starttime)) # No wait time in serial runs! start = time.time() runner.wait() # The run takes a second, so 0.5 is sufficient self.assertLess(time.time() - start, 0.5) pass
def test_run(self): target = lambda x: x**2 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) rval = taf.run(config=2) self.assertFalse(taf._accepts_instance) self.assertFalse(taf._accepts_seed) self.assertEqual(rval[0], StatusType.SUCCESS) self.assertEqual(rval[1], 4) self.assertGreaterEqual(rval[2], 0.0) self.assertEqual(rval[3], dict()) target = lambda x, seed: (x**2, {'key': seed}) taf = ExecuteTAFuncDict(ta=target, stats=self.stats) rval = taf.run(config=2, instance='test') self.assertFalse(taf._accepts_instance) self.assertTrue(taf._accepts_seed) self.assertEqual(rval[0], StatusType.SUCCESS) self.assertEqual(rval[1], 4) self.assertGreaterEqual(rval[2], 0.0) self.assertEqual(rval[3], {'key': 12345}) target = lambda x, seed, instance: (x**2, { 'key': seed, 'instance': instance }) taf = ExecuteTAFuncDict(ta=target, stats=self.stats) rval = taf.run(config=2, instance='test') self.assertTrue(taf._accepts_instance) self.assertTrue(taf._accepts_seed) self.assertEqual(rval[0], StatusType.SUCCESS) self.assertEqual(rval[1], 4) self.assertGreaterEqual(rval[2], 0.0) self.assertEqual(rval[3], {'key': 12345, 'instance': 'test'})
def test_race_challenger_1(self): """ Makes sure that a racing configuration with better performance, is selected as incumbent No adaptive capping """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) intensifier.N = 1 inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", cutoff=cutoff, seed=seed, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_3(self): ''' test _race_challenger with adaptive capping on a previously capped configuration ''' def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime", par_factor=1) taf.runhistory = self.rh intensifier = Intensifier( tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1]) self.rh.add(config=self.config1, cost=0.5, time=.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None) # give config2 a second chance inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.runs_per_config[2], 2)
def test_eval_challenger_3(self): """ test eval_challenger for a resumed SMAC run (first run with incumbent) """ def target(x): return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=False, always_race_against=None, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # adding run for incumbent configuration self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) # intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage intent_, run_info = intensifier.get_next_run( challengers=[self.config2], incumbent=self.config1, run_history=self.rh, chooser=None) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 2)
def test_num_workers(self): """Make sure we can properly return the number of workers""" # We use the funcdict as a mechanism to test Runner runner = ExecuteTAFuncDict(ta=target_delayed, stats=self.stats, run_obj='quality') runner = DaskParallelRunner(runner, n_workers=2) self.assertEqual(runner.num_workers(), 2) # Reduce the number of workers # have to give time for the worker to be killed runner.client.cluster.scale(1) time.sleep(2) self.assertEqual(runner.num_workers(), 1)
def test_timeout_runtime(self): def run_over_time(*args): time.sleep(5) taf = ExecuteTAFuncDict(ta=run_over_time, stats=self.stats, run_obj='runtime', par_factor=11) rval = taf.run(config=None, cutoff=1) self.assertEqual(rval[0], StatusType.TIMEOUT) self.assertGreaterEqual(rval[1], 11) self.assertGreaterEqual(rval[2], 1) self.assertEqual(rval[3], dict())
def test_additional_info_crash_msg(self): """ We want to make sure we catch errors as additional info, and in particular when doing multiprocessing runs, we want to make sure we capture dask exceptions """ def target_nonpickable(x, seed, instance): return x**2, {'key': seed, 'instance': instance} runner = ExecuteTAFuncDict(ta=target_nonpickable, stats=self.stats, run_obj='quality') runner = DaskParallelRunner(runner, n_workers=2) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) runner.wait() run_info, result = runner.get_finished_runs()[0] # Make sure the traceback message is included self.assertIn('traceback', result.additional_info) self.assertIn( # We expect the problem to occur in the run wrapper # So traceback should show this! 'target_nonpickable', result.additional_info['traceback']) # Make sure the error message is included self.assertIn('error', result.additional_info) self.assertIn( 'Can\'t pickle local object', result.additional_info['error'])
def test_eval_challenger(self): """ since hyperband uses eval_challenger and get_next_challenger of the internal successive halving, we don't test these method extensively """ def target(x): return 0.1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Hyperband( tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[None], initial_budget=0.5, max_budget=1, eta=2) self.assertFalse(hasattr(intensifier, 's')) # Testing get_next_challenger - get next configuration config, _ = intensifier.get_next_challenger(challengers=[self.config2, self.config3], chooser=None, run_history=self.rh) self.assertEqual(intensifier.s, intensifier.s_max) self.assertEqual(config, self.config2) # update to the last SH iteration of the given HB stage self.assertEqual(intensifier.s, 1) self.assertEqual(intensifier.s_max, 1) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, seed=0, budget=1) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, seed=0, budget=0.5) self.rh.add(config=self.config3, cost=3, time=2, status=StatusType.SUCCESS, seed=0, budget=0.5) intensifier.sh_intensifier.success_challengers = {self.config2, self.config3} intensifier.sh_intensifier._update_stage(self.rh) # evaluation should change the incumbent to config2 inc, inc_value = intensifier.eval_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.s, 0) self.assertEqual(inc_value, 0.1) self.assertEqual(self.stats.ta_runs, 1) self.assertEqual(list(self.rh.data.keys())[-1][0], self.rh.config_ids[self.config2]) self.assertEqual(self.stats.inc_changed, 1)
def eval_challenger( run_info: RunInfo, taf: ExecuteTAFuncDict, stats: Stats, runhistory: RunHistory, ): """ Wrapper over challenger evaluation SMBO objects handles run history now, but to keep same testing functionality this function is a small wrapper to launch the taf and add it to the history """ # evaluating configuration run_info, result = taf.run_wrapper( run_info=run_info, ) stats.ta_time_used += float(result.time) runhistory.add( config=run_info.config, cost=result.cost, time=result.time, status=result.status, instance_id=run_info.instance, seed=run_info.seed, budget=run_info.budget, ) stats.n_configs = len(runhistory.config_ids) return result
def __init__(self, cost_oracle: typing.Mapping[str, float], tae: typing.Type[SerialRunner] = ExecuteTARunOld, **kwargs: typing.Any) -> None: ''' Constructor Arguments --------- cost_oracle: typing.Mapping[str,float] cost of oracle per instance ''' super().__init__(**kwargs) self.cost_oracle = cost_oracle if tae is ExecuteTARunAClib: self.runner = ExecuteTARunAClib(**kwargs) # type: SerialRunner elif tae is ExecuteTARunOld: self.runner = ExecuteTARunOld(**kwargs) elif tae is ExecuteTAFuncDict: self.runner = ExecuteTAFuncDict(**kwargs) elif tae is ExecuteTAFuncArray: self.runner = ExecuteTAFuncArray(**kwargs) else: raise Exception('TAE not supported')
def setUp(self): self.cs = ConfigurationSpace() self.cs.add_hyperparameter( UniformFloatHyperparameter(name="x1", lower=1, upper=10, default=2)) self.scenario = Scenario({'cs': self.cs, 'run_obj': 'quality'}) self.ta = ExecuteTAFuncDict(lambda x: x["x1"]**2)
def test_inject_stats_and_runhistory_object_to_TAE(self): ta = ExecuteTAFuncDict(lambda x: x**2) self.assertIsNone(ta.stats) self.assertIsNone(ta.runhistory) SMAC(tae_runner=ta, scenario=self.scenario) self.assertIsInstance(ta.stats, Stats) self.assertIsInstance(ta.runhistory, RunHistory)
def test_pass_tae_runner_objective(self): tae = ExecuteTAFuncDict(lambda: 1, run_obj='runtime') self.assertRaisesRegexp(ValueError, "Objective for the target algorithm" " runner and the scenario must be " "the same, but are 'runtime' and " "'quality'", SMAC, tae_runner=tae, scenario=self.scenario)
def test_race_challenger_large_blocked_seed(self): ''' test _race_challenger whether seeds are blocked for challenger runs ''' def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), deterministic=False) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) # tie on first instances and then challenger should always win # and be returned as inc inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds)
def test_get_runhistory_and_trajectory(self): ta = ExecuteTAFuncDict(lambda x: x ** 2) smac = SMAC(tae_runner=ta, scenario=self.scenario) self.assertRaises(ValueError, smac.get_runhistory) self.assertRaises(ValueError, smac.get_trajectory) smac.trajectory = 'dummy' self.assertEqual(smac.get_trajectory(), 'dummy') smac.runhistory = 'dummy' self.assertEqual(smac.get_runhistory(), 'dummy')
def create_worker(self, worker_id) -> Worker: """Function to create a worker. Parameters ---------- worker_id : int A seed for the random generator. It should be the id of the worker. Returns ------- worker : Worker Return a ps-lite worker. """ # 首先指定输出目录 output_dir = self.temp_folder + "worker-output_%s" % ( datetime.datetime.fromtimestamp(time.time()).strftime( '%Y-%m-%d_%H:%M:%S_%f')) # 然后创建scenario scenario_dict = { "cs": self.cs, "run_obj": "quality", "cutoff_time": self.cutoff, "initial_incumbent": "RANDOM", "output_dir": output_dir } scenario = Scenario(scenario_dict) # 统计类 stats = Stats(scenario) # 创建ta函数 ta = CustomizedTA(self.X_train, self.y_train, self.X_valid, self.y_valid) tae_runner = ExecuteTAFuncDict(ta=ta, stats=stats, run_obj=scenario.run_obj, memory_limit=scenario.memory_limit, runhistory=RunHistory( aggregate_func=average_cost), par_factor=scenario.par_factor, cost_for_crash=scenario.cost_for_crash) # logger和rng traj_logger = TrajLogger(output_dir=output_dir, stats=stats) rng = np.random.RandomState(seed=worker_id) # 创建intensifier intensifier = Intensifier(tae_runner=tae_runner, stats=stats, traj_logger=traj_logger, rng=rng, instances=scenario.train_insts) # 最终目的,创建worker并返回 worker = Worker(self.worker_args[worker_id], self.cs, intensifier, worker_id=worker_id) return worker
def setUp(self): self.cs = ConfigurationSpace() self.cs.add_hyperparameter(UniformFloatHyperparameter( name="x1", lower=1, upper=10, default_value=1) ) self.scenario = Scenario({ 'cs': self.cs, 'run_obj': 'quality', 'output_dir': '', 'ta_run_limit': 100, }) self.stats = Stats(scenario=self.scenario) self.rh = RunHistory() self.ta = ExecuteTAFuncDict(lambda x: x["x1"]**2, stats=self.stats, runhistory=self.rh)
def test_inject_dependencies(self): # initialize objects with missing dependencies ta = ExecuteTAFuncDict(lambda x: x**2) rh = RunHistory(aggregate_func=None) acqu_func = EI(model=None) intensifier = Intensifier(tae_runner=None, stats=None, traj_logger=None, rng=np.random.RandomState(), instances=None) init_design = DefaultConfiguration(tae_runner=None, scenario=None, stats=None, traj_logger=None, rng=np.random.RandomState()) rh2epm = RunHistory2EPM4Cost(scenario=self.scenario, num_params=0) rh2epm.scenario = None # assert missing dependencies self.assertIsNone(rh.aggregate_func) self.assertIsNone(acqu_func.model) self.assertIsNone(intensifier.tae_runner) self.assertIsNone(intensifier.stats) self.assertIsNone(intensifier.traj_logger) self.assertIsNone(init_design.tae_runner) self.assertIsNone(init_design.scenario) self.assertIsNone(init_design.stats) self.assertIsNone(init_design.traj_logger) self.assertIsNone(rh2epm.scenario) # initialize smac-object SMAC(scenario=self.scenario, tae_runner=ta, runhistory=rh, intensifier=intensifier, acquisition_function=acqu_func, runhistory2epm=rh2epm, initial_design=init_design) # assert that missing dependencies are injected self.assertIsNotNone(rh.aggregate_func, AbstractAcquisitionFunction) self.assertIsInstance(acqu_func.model, AbstractEPM) self.assertIsInstance(intensifier.tae_runner, ExecuteTARun) self.assertIsInstance(intensifier.stats, Stats) self.assertIsInstance(intensifier.traj_logger, TrajLogger) self.assertIsInstance(init_design.tae_runner, ExecuteTARun) self.assertIsInstance(init_design.scenario, Scenario) self.assertIsInstance(init_design.stats, Stats) self.assertIsInstance(init_design.traj_logger, TrajLogger) self.assertIsInstance(rh2epm.scenario, Scenario)