def test_serial_runs(self): # We use the funcdict as a mechanism to test SerialRunner runner = ExecuteTAFuncDict(ta=target_delayed, stats=self.stats, run_obj='quality') self.assertIsInstance(runner, SerialRunner) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) run_info = RunInfo(config=3, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) run_values = runner.get_finished_runs() self.assertEqual(len(run_values), 2) # To make sure runs launched serially, we just make sure that the end time of # a run is later than the other # Results are returned in left to right self.assertLessEqual(int(run_values[1][1].endtime), int(run_values[0][1].starttime)) # No wait time in serial runs! start = time.time() runner.wait() # The run takes a second, so 0.5 is sufficient self.assertLess(time.time() - start, 0.5) pass
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock): pynisher_mock.return_value = None config = unittest.mock.Mock() config.origin = 'MOCK' config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=self.backend, autosklearn_seed=1, port=self.logger_port, resampling_strategy='holdout', stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, pynisher_context='fork', ) # The following should not fail because abort on first config crashed is false info = ta.run_wrapper( RunInfo(config=config, cutoff=60, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.CRASHED) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float) self.assertEqual( info[1].additional_info, { 'configuration_origin': 'MOCK', 'error': "Result queue is empty", 'exit_status': 0, 'exitcode': 0, 'subprocess_stdout': '', 'subprocess_stderr': '' }) self.stats.submitted_ta_runs += 1 info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.CRASHED) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float) self.assertEqual( info[1].additional_info, { 'configuration_origin': 'MOCK', 'error': "Result queue is empty", 'exit_status': 0, 'exitcode': 0, 'subprocess_stdout': '', 'subprocess_stderr': '' })
def test_eval_with_limits_holdout_timeout_with_results_in_queue(self, pynisher_mock): config = unittest.mock.Mock() config.config_id = 198 def side_effect(**kwargs): queue = kwargs['queue'] queue.put({'status': StatusType.SUCCESS, 'loss': 0.5, 'additional_run_info': {}}) m1 = unittest.mock.Mock() m2 = unittest.mock.Mock() m1.return_value = m2 pynisher_mock.return_value = m1 m2.side_effect = side_effect m2.exit_status = pynisher.TimeoutException m2.wall_clock_time = 30 # Test for a succesful run ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1, port=self.logger_port, resampling_strategy='holdout', stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, pynisher_context='forkserver', ) info = ta.run_wrapper(RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.SUCCESS) self.assertEqual(info[1].cost, 0.5) self.assertIsInstance(info[1].time, float) self.assertNotIn('exitcode', info[1].additional_info) # And a crashed run which is in the queue def side_effect(**kwargs): queue = kwargs['queue'] queue.put({'status': StatusType.CRASHED, 'loss': 2.0, 'additional_run_info': {}}) m2.side_effect = side_effect ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1, port=self.logger_port, resampling_strategy='holdout', stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, pynisher_context='forkserver', ) info = ta.run_wrapper(RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.CRASHED) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float) self.assertNotIn('exitcode', info[1].additional_info)
def run_wrapper( self, run_info: RunInfo, ) -> Tuple[RunInfo, RunValue]: """ wrapper function for ExecuteTARun.run_wrapper() to cap the target algorithm runtime if it would run over the total allowed runtime. Parameters ---------- run_info : RunInfo Object that contains enough information to execute a configuration run in isolation. Returns ------- RunInfo: an object containing the configuration launched RunValue: Contains information about the status/performance of config """ if self.budget_type is None: if run_info.budget != 0: raise ValueError( 'If budget_type is None, budget must be.0, but is %f' % run_info.budget) else: if run_info.budget == 0: run_info = run_info._replace(budget=100) elif run_info.budget <= 0 or run_info.budget > 100: raise ValueError( 'Illegal value for budget, must be >0 and <=100, but is %f' % run_info.budget) if self.budget_type not in ('subsample', 'iterations', 'mixed'): raise ValueError( "Illegal value for budget type, must be one of " "('subsample', 'iterations', 'mixed'), but is : %s" % self.budget_type) remaining_time = self.stats.get_remaing_time_budget() if remaining_time - 5 < run_info.cutoff: run_info = run_info._replace(cutoff=int(remaining_time - 5)) if run_info.cutoff < 1.0: return run_info, RunValue( status=StatusType.STOP, cost=self.worst_possible_result, time=0.0, additional_info={}, starttime=time.time(), endtime=time.time(), ) elif (run_info.cutoff != int(np.ceil(run_info.cutoff)) and not isinstance(run_info.cutoff, int)): run_info = run_info._replace(cutoff=int(np.ceil(run_info.cutoff))) return super().run_wrapper(run_info=run_info)
def test_parallel_runs(self): """Make sure because there are 2 workers, the runs are launched closely in time together""" # We use the funcdict as a mechanism to test Runner runner = ExecuteTAFuncDict(ta=target_delayed, stats=self.stats, run_obj='quality') runner = DaskParallelRunner(runner, n_workers=2) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) run_info = RunInfo(config=3, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) # At this stage, we submitted 2 jobs, that are running in remote # workers. We have to wait for each one of them to complete. The # runner provides a wait() method to do so, yet it can only wait for # a single job to be completed. It does internally via dask wait(<list of futures>) # so we take wait for the first job to complete, and take it out runner.wait() run_values = runner.get_finished_runs() # To be on the safe side, we don't check for: # self.assertEqual(len(run_values), 1) # In the ideal world, two runs were launched which take the same time # so waiting for one means, the second one is completed. But some # overhead might cause it to be delayed. # Above took the first run results and put it on run_values # But for this check we need the second run we submitted # Again, the runs might take slightly different time depending if we have # heterogeneous workers, so calling wait 2 times is a guarantee that 2 # jobs are finished runner.wait() run_values.extend(runner.get_finished_runs()) self.assertEqual(len(run_values), 2) # To make it is parallel, we just make sure that the start of the second # run is earlier than the end of the first run # Results are returned in left to right self.assertLessEqual(int(run_values[0][1].starttime), int(run_values[1][1].endtime))
def test_eval_with_limits_holdout_2(self, eval_houldout_mock): config = unittest.mock.Mock() config.config_id = 198 def side_effect(*args, **kwargs): queue = kwargs['queue'] queue.put({'status': StatusType.SUCCESS, 'loss': 0.5, 'additional_run_info': kwargs['instance']}) eval_houldout_mock.side_effect = side_effect ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1, port=self.logger_port, resampling_strategy='holdout', stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, pynisher_context='fork', ) self.scenario.wallclock_limit = 180 instance = "{'subsample': 30}" info = ta.run_wrapper(RunInfo(config=config, cutoff=30, instance=instance, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.SUCCESS) self.assertEqual(len(info[1].additional_info), 2) self.assertIn('configuration_origin', info[1].additional_info) self.assertEqual(info[1].additional_info['message'], "{'subsample': 30}")
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock): pynisher_mock.side_effect = MemoryError config = unittest.mock.Mock() config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=BackendMock(), seed=1, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, logger_port=self.logger_port, pynisher_context='fork', ) info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.MEMOUT) # For accuracy, worst possible result is MAXINT worst_possible_result = 1 self.assertEqual(info[1].cost, worst_possible_result) self.assertIsInstance(info[1].time, float) self.assertNotIn('exitcode', info[1].additional_info)
def test_run(self): """Makes sure that we are able to run a configuration and return the expected values/types""" # We use the funcdict as a mechanism to test SerialRunner runner = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj='quality') self.assertIsInstance(runner, SerialRunner) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) # submit runs! then get the value runner.submit_run(run_info) run_values = runner.get_finished_runs() self.assertEqual(len(run_values), 1) self.assertIsInstance(run_values, list) self.assertIsInstance(run_values[0][0], RunInfo) self.assertIsInstance(run_values[0][1], RunValue) self.assertEqual(run_values[0][1].cost, 4) self.assertEqual(run_values[0][1].status, StatusType.SUCCESS)
def test_start_tae_return_abort(self, test_run): ''' testing abort ''' # Patch run-function for custom-return test_run.return_value = StatusType.ABORT, 12345.0, 1.2345, {} scen = Scenario( scenario={ 'cs': ConfigurationSpace(), 'run_obj': 'quality', 'output_dir': '', }, cmd_options=None, ) stats = Stats(scen) stats.start_timing() eta = SerialRunner(ta=lambda *args: None, stats=stats) _, run_value = eta.run_wrapper( RunInfo(config=None, instance=1, instance_specific=None, cutoff=30, seed=None, capped=False, budget=0.0)) self.assertEqual(run_value.status, StatusType.ABORT)
def test_eval_with_limits_holdout(self, pynisher_mock): pynisher_mock.side_effect = safe_eval_success_mock config = unittest.mock.Mock() config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=BackendMock(), seed=1, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, logger_port=self.logger_port, pynisher_context='fork', ) info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[0].config.config_id, 198) self.assertEqual(info[1].status, StatusType.SUCCESS, info) self.assertEqual(info[1].cost, 0.5) self.assertIsInstance(info[1].time, float)
def test_exception_in_target_function(self, eval_holdout_mock): config = unittest.mock.Mock() config.config_id = 198 eval_holdout_mock.side_effect = ValueError ta = ExecuteTaFuncWithQueue( backend=BackendMock(), seed=1, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, logger_port=self.logger_port, pynisher_context='fork', ) self.stats.submitted_ta_runs += 1 info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.CRASHED) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float) self.assertEqual(info[1].additional_info['error'], 'ValueError()') self.assertIn('traceback', info[1].additional_info) self.assertNotIn('exitcode', info[1].additional_info)
def test_cutoff_lower_than_remaining_time(self, pynisher_mock): config = unittest.mock.Mock() config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=BackendMock(), seed=1, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, logger_port=self.logger_port, pynisher_context='fork', ) self.stats.ta_runs = 1 ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4) self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'], int)
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock): config = unittest.mock.Mock() config.config_id = 198 m1 = unittest.mock.Mock() m2 = unittest.mock.Mock() m1.return_value = m2 pynisher_mock.return_value = m1 m2.exit_status = pynisher.TimeoutException m2.wall_clock_time = 30 ta = ExecuteTaFuncWithQueue( backend=BackendMock(), seed=1, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, logger_port=self.logger_port, pynisher_context='fork', ) info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.TIMEOUT) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float) self.assertNotIn('exitcode', info[1].additional_info)
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock): pynisher_mock.side_effect = MemoryError config = unittest.mock.Mock() config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=log_loss, cost_for_crash=get_cost_of_crash(log_loss), abort_on_first_run_crash=False, ) info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.MEMOUT) # For logloss, worst possible result is MAXINT worst_possible_result = MAXINT self.assertEqual(info[1].cost, worst_possible_result) self.assertIsInstance(info[1].time, float)
def test_race_challenger_2(self): """ Makes sure that a racing configuration with better performance, that is capped, doesn't substitute the incumbent. """ def target(x): time.sleep(1.5) return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], ) self.rh.add( config=self.config1, cost=0.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1)
def test_additional_info_crash_msg(self): """ We want to make sure we catch errors as additional info, and in particular when doing multiprocessing runs, we want to make sure we capture dask exceptions """ def target_nonpickable(x, seed, instance): return x**2, {'key': seed, 'instance': instance} runner = ExecuteTAFuncDict(ta=target_nonpickable, stats=self.stats, run_obj='quality') runner = DaskParallelRunner(runner, n_workers=2) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) runner.wait() run_info, result = runner.get_finished_runs()[0] # Make sure the traceback message is included self.assertIn('traceback', result.additional_info) self.assertIn( # We expect the problem to occur in the run wrapper # So traceback should show this! 'target_nonpickable', result.additional_info['traceback']) # Make sure the error message is included self.assertIn('error', result.additional_info) self.assertIn( 'Can\'t pickle local object', result.additional_info['error'])
def test_get_next_run_waits_if_no_workers(self): """ In the case all workers are busy, we wait so that we do not saturate the process with configurations that will not finish in time """ intent, run_info = self.intensifier.get_next_run( challengers=[self.config1, self.config2], incumbent=None, run_history=self.rh, num_workers=1, chooser=None, ) # We can get the configuration 1 self.assertEqual(intent, RunInfoIntent.RUN) self.assertEqual(run_info, RunInfo( config=self.config1, instance=1, instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0, )) # We should not get configuration 2 # As there is just 1 worker intent, run_info = self.intensifier.get_next_run( challengers=[self.config2], incumbent=None, run_history=self.rh, num_workers=1, chooser=None, ) self.assertEqual(intent, RunInfoIntent.WAIT) self.assertEqual(run_info, RunInfo( config=None, instance=None, instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0, ))
def test_crashed_cost_value(self, test_run): ''' test cost on crashed runs ''' # Patch run-function for custom-return scen = Scenario(scenario={ 'cs': ConfigurationSpace(), 'run_obj': 'quality' }, cmd_options=None) stats = Stats(scen) stats.start_timing() stats.submitted_ta_runs += 1 # Check quality test_run.return_value = StatusType.CRASHED, np.nan, np.nan, {} eta = SerialRunner(ta=lambda *args: None, stats=stats, run_obj='quality', cost_for_crash=100) run_info, result = eta.run_wrapper( RunInfo(config={}, instance=1, instance_specific="0", cutoff=None, seed=None, capped=False, budget=0.0)) self.assertEqual(100, result.cost) # Check runtime eta = SerialRunner(ta=lambda *args: None, stats=stats, run_obj='runtime', cost_for_crash=10.7) run_info, result = eta.run_wrapper( RunInfo(config={}, instance=1, instance_specific="0", cutoff=20, seed=None, capped=False, budget=0.0)) self.assertEqual(20.0, result.cost)
def mock_get_next_run(**kwargs): config = cs.sample_configuration() all_configs.append(config) return (RunInfoIntent.RUN, RunInfo(config=config, instance=time.time() % 10, instance_specific={}, seed=0, cutoff=None, capped=False, budget=0.0))
def sample_config(self, recursion_depth: int = 5) -> Configuration: # TODO even though initial_incumbent is set to DEFAULT, the first configuration is random # sample next configuration for intensification # Initial design runs are also included in the BO loop now. intent, run_info = self.intensifier.get_next_run( challengers=self.initial_design_configs, incumbent=self.incumbent, chooser=self.epm_chooser, run_history=self.runhistory, repeat_configs=self.intensifier.repeat_configs, num_workers=self.tae_runner.num_workers(), ) # remove config from initial design challengers to not repeat it again self.initial_design_configs = [ c for c in self.initial_design_configs if c != run_info.config ] if intent == RunInfoIntent.SKIP: if recursion_depth > 0: return self.sample_config(recursion_depth - 1) else: self.logger.warning( 'Repeatedly failed to sample configuration. Using random configuration instead.' ) config = self.config_space.sample_configuration() config.origin = 'Random Search' run_info = RunInfo(config, run_info.instance, run_info.instance_specific, run_info.seed, run_info.cutoff, run_info.capped, run_info.budget, run_info.source_id) if run_info.config is None: config = self.config_space.sample_configuration() config.origin = 'Random Search' else: config = run_info.config self.runhistory.add( config=config, cost=float(MAXINT), time=0.0, status=SmacStatus.RUNNING, instance_id=run_info.instance, seed=run_info.seed, budget=run_info.budget, ) self.stats.submitted_ta_runs += 1 self.run_infos[str(config)] = (run_info, time.time()) return config
def test_race_challenger_1(self): """ Makes sure that a racing configuration with better performance, is selected as incumbent No adaptive capping """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) intensifier.N = 1 inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", cutoff=cutoff, seed=seed, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1)
def get_next_run(self, challengers: typing.Optional[typing.List[Configuration]], incumbent: Configuration, chooser: typing.Optional[EPMChooser], run_history: RunHistory, repeat_configs: bool = True, num_workers: int = 1, ) -> typing.Tuple[RunInfoIntent, RunInfo]: """ Selects which challenger to be used. As in a traditional BO loop, we sample from the EPM, which is the next configuration based on the acquisition function. The input data is read from the runhistory. Parameters ---------- challengers : typing.List[Configuration] promising configurations incumbent: Configuration incumbent configuration chooser : smac.optimizer.epm_configuration_chooser.EPMChooser optimizer that generates next configurations to use for racing run_history : smac.runhistory.runhistory.RunHistory stores all runs we ran so far repeat_configs : bool if False, an evaluated configuration will not be generated again num_workers: int the maximum number of workers available at a given time. Returns ------- intent: RunInfoIntent Indicator of how to consume the RunInfo object run_info: RunInfo An object that encapsulates the minimum information to evaluate a configuration """ # We always sample from the configs provided or the EPM challenger = self._next_challenger(challengers=challengers, chooser=chooser, run_history=run_history, repeat_configs=repeat_configs) return RunInfoIntent.RUN, RunInfo( config=challenger, instance=self.instances[-1], instance_specific="0", seed=0 if self.deterministic else self.rs.randint(low=0, high=MAXINT, size=1)[0], cutoff=self.cutoff, capped=False, budget=0.0, )
def test_zero_or_negative_cutoff(self, pynisher_mock): config = unittest.mock.Mock() config.config_id = 198 ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1, port=self.logger_port, resampling_strategy='holdout', stats=self.stats, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, ) self.scenario.wallclock_limit = 5 self.stats.submitted_ta_runs += 1 run_info, run_value = ta.run_wrapper(RunInfo(config=config, cutoff=9, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(run_value.status, StatusType.STOP)
def test_process_results(self): """Ensures that the results are processed by the pertinent intensifer, based on the source id""" scheduler = ParallelScheduler( stats=None, traj_logger=None, instances=[1, 2, 3], rng=np.random.RandomState(12345), deterministic=True, ) scheduler.intensifier_instances = { 0: mock.Mock(), 1: mock.Mock(), 2: mock.Mock(), } run_info = RunInfo( config=None, instance=0, instance_specific="0", cutoff=None, seed=0, capped=False, budget=0.0, source_id=2, ) result = RunValue(cost=1, time=0.5, status=StatusType.SUCCESS, starttime=1, endtime=2, additional_info={}) scheduler.process_results(run_info=run_info, result=result, incumbent=None, run_history=None, time_bound=None) self.assertIsNone( scheduler.intensifier_instances[0].process_results.call_args) self.assertIsNone( scheduler.intensifier_instances[1].process_results.call_args) self.assertEqual( scheduler.intensifier_instances[2].process_results.call_args[1] ['run_info'], run_info)
def _validate_parallel( self, tae: BaseRunner, runs: typing.List[_Run], n_jobs: int, backend: str, runhistory: typing.Optional[RunHistory] = None, ) -> typing.List[RunValue]: """ Validate runs with joblibs Parallel-interface Parameters ---------- tae: BaseRunner tae to be used for validation runs: list<_Run> list with _Run-objects [_Run(config=CONFIG1,inst=INSTANCE1,seed=SEED1,inst_specs=INST_SPECIFICS1), ...] n_jobs: int number of cpus to use for validation (-1 to use all) backend: str what backend to use for parallelization runhistory: RunHistory optional, RunHistory-object to reuse runs Returns ------- run_results: list<tuple(tae-returns)> results as returned by tae """ # Runs with parallel run_results = Parallel(n_jobs=n_jobs, backend=backend)( delayed(_unbound_tae_starter)( tae, runhistory, RunInfo( config=run.config, instance=run.inst, instance_specific="0", seed=run.seed, cutoff=self.scen.cutoff, # type: ignore[attr-defined] # noqa F821 capped=False, budget=0 ) ) for run in runs) return run_results
def test_silent_exception_in_target_function(self): config = unittest.mock.Mock(spec=int) config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=BackendMock(), seed=1, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, logger_port=self.logger_port, pynisher_context='fork', ) ta.pynisher_logger = unittest.mock.Mock() self.stats.submitted_ta_runs += 1 info = ta.run_wrapper( RunInfo(config=config, cutoff=3000, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.CRASHED, msg=str(info[1].additional_info)) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float) self.assertIn(info[1].additional_info['error'], ( """AttributeError("'BackendMock' object has no attribute """ """'save_targets_ensemble'",)""", """AttributeError("'BackendMock' object has no attribute """ """'save_targets_ensemble'")""", """AttributeError('save_targets_ensemble')""" """AttributeError("'BackendMock' object has no attribute """ """'setup_logger'",)""", """AttributeError("'BackendMock' object has no attribute """ """'setup_logger'")""", )) self.assertNotIn('exitcode', info[1].additional_info) self.assertNotIn('exit_status', info[1].additional_info) self.assertNotIn('traceback', info[1])
def test_incorporate_run_results_callback(self, process_results_mock): process_results_mock.return_value = None, None class TestCallback(IncorporateRunResultCallback): def __init__(self): self.num_call = 0 def __call__(self, smbo, run_info, result, time_left) -> None: self.num_call += 1 self.config = run_info.config callback = TestCallback() self.scenario.output_dir = None smac = SMAC4AC(self.scenario) smac.register_callback(callback) self.output_dirs.append(smac.output_dir) smbo = smac.solver config = self.scenario.cs.sample_configuration() run_info = RunInfo( config=config, instance=None, instance_specific=None, seed=1, cutoff=None, capped=False, budget=0.0, source_id=0, ) result = RunValue(1.2345, 2.3456, "status", "starttime", "endtime", "additional_info") time_left = 10 smbo._incorporate_run_results(run_info=run_info, result=result, time_left=time_left) self.assertEqual(callback.num_call, 1) self.assertEqual(callback.config, config)
def test_cutoff_lower_than_remaining_time(self, pynisher_mock): config = unittest.mock.Mock() config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', stats=self.stats, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, ) self.stats.ta_runs = 1 ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4) self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'], int)
def test_get_next_run(self): """ Makes sure that sampling a configuration returns a valid configuration """ intent, run_info = self.intensifier.get_next_run( challengers=[self.config1], incumbent=None, run_history=self.rh, num_workers=1, chooser=None, ) self.assertEqual(intent, RunInfoIntent.RUN) self.assertEqual(run_info, RunInfo( config=self.config1, instance=1, instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0, ))
resampling_strategy='test', memory_limit=memory_lim, disable_file_output=True, logger=logger, stats=stats, scoring_functions=scoring_functions, include=include, metric=automl_arguments['metric'], cost_for_crash=get_cost_of_crash(automl_arguments['metric']), abort_on_first_run_crash=False, ) run_info, run_value = ta.run_wrapper( RunInfo( config=config, instance=None, instance_specific=None, seed=1, cutoff=per_run_time_limit * 3, capped=False, )) if run_value.status == StatusType.SUCCESS: assert len( run_value.additional_info) > 1, run_value.additional_info # print(additional_run_info) validated_trajectory.append( list(entry) + [task_id] + [run_value.additional_info]) print('Finished validating configuration %d/%d' % (i + 1, len(trajectory))) print('Finished to validate configurations')