def test_eval_with_limits_holdout(self, pynisher_mock): pynisher_mock.side_effect = safe_eval_success_mock config = unittest.mock.Mock() config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, ) info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[0].config.config_id, 198) self.assertEqual(info[1].status, StatusType.SUCCESS) self.assertEqual(info[1].cost, 0.5) self.assertIsInstance(info[1].time, float)
def test_eval_with_limits_holdout_2(self, eval_houldout_mock): def side_effect(*args, **kwargs): queue = kwargs['queue'] queue.put({ 'status': StatusType.SUCCESS, 'loss': 0.5, 'additional_run_info': kwargs['instance'] }) eval_houldout_mock.side_effect = side_effect ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy) self.scenario.wallclock_limit = 180 instance = "{'subsample': 30}" info = ta.start(None, cutoff=30, instance=instance) self.assertEqual(info[0], StatusType.SUCCESS) self.assertEqual(info[-1], { 'message': "{'subsample': 30}", 'configuration_origin': 'UNKNOWN' })
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock): config = unittest.mock.Mock() config.config_id = 198 m1 = unittest.mock.Mock() m2 = unittest.mock.Mock() m1.return_value = m2 pynisher_mock.return_value = m1 m2.exit_status = pynisher.TimeoutException m2.wall_clock_time = 30 ta = ExecuteTaFuncWithQueue( backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, ) info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.TIMEOUT) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float)
def test_exception_in_target_function(self, eval_holdout_mock): config = unittest.mock.Mock() config.config_id = 198 eval_holdout_mock.side_effect = ValueError ta = ExecuteTaFuncWithQueue( backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, ) self.stats.submitted_ta_runs += 1 info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.CRASHED) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float) self.assertEqual(info[1].additional_info['error'], 'ValueError()') self.assertIn('traceback', info[1].additional_info)
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock): pynisher_mock.return_value = None ta = ExecuteTaFuncWithQueue( backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, ) # The following should not fail because abort on first config crashed is false info = ta.start(config=None, instance=None, cutoff=60) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float) self.assertEqual(info[3], { 'configuration_origin': 'UNKNOWN', 'error': "Result queue is empty" }) self.stats.ta_runs += 1 info = ta.start(config=None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float) self.assertEqual(info[3], { 'configuration_origin': 'UNKNOWN', 'error': "Result queue is empty" })
def _do_dummy_prediction(self, datamanager, num_run): # When using partial-cv it makes no sense to do dummy predictions if self._resampling_strategy in ['partial-cv', 'partial-cv-iterative-fit']: return num_run self._logger.info("Starting to create dummy predictions.") memory_limit = int(self._ml_memory_limit) scenario_mock = unittest.mock.Mock() scenario_mock.wallclock_limit = self._time_for_task # This stats object is a hack - maybe the SMAC stats object should # already be generated here! stats = Stats(scenario_mock) stats.start_timing() ta = ExecuteTaFuncWithQueue(backend=self._backend, autosklearn_seed=self._seed, resampling_strategy=self._resampling_strategy, initial_num_run=num_run, logger=self._logger, stats=stats, metric=self._metric, memory_limit=memory_limit, disable_file_output=self._disable_evaluator_output, **self._resampling_strategy_arguments) status, cost, runtime, additional_info = \ ta.run(1, cutoff=self._time_for_task) if status == StatusType.SUCCESS: self._logger.info("Finished creating dummy predictions.") else: self._logger.error('Error creating dummy predictions: %s ', str(additional_info)) return ta.num_run
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock): pynisher_mock.return_value = None ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy) self.assertRaisesRegex(FirstRunCrashedException, "First run crashed, abort. Please check your " "setup -- we assume that your " "defaultconfiguration does not crashes. \(To " "deactivate this exception, use the SMAC " "scenario option 'abort_on_first_run_crash'\)", ta.start, config=None, instance=None, cutoff=30) self.stats.ta_runs += 1 info = ta.start(config=None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float) self.assertEqual(info[3], { 'configuration_origin': 'UNKNOWN', 'error': "Result queue is empty" })
def test_cutoff_lower_than_remaining_time(self, pynisher_mock): config = unittest.mock.Mock() config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=self.backend, autosklearn_seed=1, port=self.logger_port, resampling_strategy='holdout', stats=self.stats, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, pynisher_context='forkserver', ) self.stats.ta_runs = 1 ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4) self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'], int)
def _do_dummy_prediction(self, datamanager, num_run): self._logger.info("Starting to create dummy predictions.") # time_limit = int(self._time_for_task / 6.) memory_limit = int(self._ml_memory_limit) ta = ExecuteTaFuncWithQueue( backend=self._backend, autosklearn_seed=self._seed, resampling_strategy=self._resampling_strategy, initial_num_run=num_run, logger=self._logger, **self._resampling_strategy_arguments) status, cost, runtime, additional_info = \ ta.run(1, cutoff=self._time_for_task, memory_limit=memory_limit) if status == StatusType.SUCCESS: self._logger.info("Finished creating dummy predictions.") else: self._logger.error('Error creating dummy predictions:%s ', additional_info) #status, cost, runtime, additional_info = \ # ta.run(2, cutoff=time_limit, memory_limit=memory_limit) #if status == StatusType.SUCCESS: # self._logger.info("Finished creating dummy prediction 2/2.") #else: # self._logger.error('Error creating dummy prediction 2/2 %s', # additional_info) return ta.num_run
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock): pynisher_mock.return_value = None ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy) self.assertRaisesRegex(FirstRunCrashedException, "First run crashed, abort. \(To prevent this, " "toggle the " "'abort_on_first_run_crash'-option!\)", ta.start, config=None, instance=None, cutoff=30) self.stats.ta_runs += 1 info = ta.start(config=None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float) self.assertEqual(info[3], {'error': "Result queue is empty"})
def _do_dummy_prediction(self, datamanager, num_run): self._logger.info("Starting to create dummy predictions.") # time_limit = int(self._time_for_task / 6.) memory_limit = int(self._ml_memory_limit) ta = ExecuteTaFuncWithQueue(backend=self._backend, autosklearn_seed=self._seed, resampling_strategy=self._resampling_strategy, initial_num_run=num_run, logger=self._logger, **self._resampling_strategy_arguments) status, cost, runtime, additional_info = \ ta.run(1, cutoff=self._time_for_task, memory_limit=memory_limit) if status == StatusType.SUCCESS: self._logger.info("Finished creating dummy predictions.") else: self._logger.error('Error creating dummy predictions:%s ', additional_info) #status, cost, runtime, additional_info = \ # ta.run(2, cutoff=time_limit, memory_limit=memory_limit) #if status == StatusType.SUCCESS: # self._logger.info("Finished creating dummy prediction 2/2.") #else: # self._logger.error('Error creating dummy prediction 2/2 %s', # additional_info) return ta.num_run
def test_eval_with_limits_holdout_2(self, eval_houldout_mock): config = unittest.mock.Mock() config.config_id = 198 def side_effect(*args, **kwargs): queue = kwargs['queue'] queue.put({'status': StatusType.SUCCESS, 'loss': 0.5, 'additional_run_info': kwargs['instance']}) eval_houldout_mock.side_effect = side_effect ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1, port=self.logger_port, resampling_strategy='holdout', stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, pynisher_context='fork', ) self.scenario.wallclock_limit = 180 instance = "{'subsample': 30}" info = ta.run_wrapper(RunInfo(config=config, cutoff=30, instance=instance, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.SUCCESS) self.assertEqual(len(info[1].additional_info), 2) self.assertIn('configuration_origin', info[1].additional_info) self.assertEqual(info[1].additional_info['message'], "{'subsample': 30}")
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock): pynisher_mock.side_effect = MemoryError config = unittest.mock.Mock() config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=log_loss, cost_for_crash=get_cost_of_crash(log_loss), abort_on_first_run_crash=False, ) info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.MEMOUT) # For logloss, worst possible result is MAXINT worst_possible_result = MAXINT self.assertEqual(info[1].cost, worst_possible_result) self.assertIsInstance(info[1].time, float)
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock): pynisher_mock.return_value = None config = unittest.mock.Mock() config.origin = 'MOCK' config.config_id = 198 ta = ExecuteTaFuncWithQueue( backend=self.backend, autosklearn_seed=1, port=self.logger_port, resampling_strategy='holdout', stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, pynisher_context='fork', ) # The following should not fail because abort on first config crashed is false info = ta.run_wrapper( RunInfo(config=config, cutoff=60, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.CRASHED) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float) self.assertEqual( info[1].additional_info, { 'configuration_origin': 'MOCK', 'error': "Result queue is empty", 'exit_status': 0, 'exitcode': 0, 'subprocess_stdout': '', 'subprocess_stderr': '' }) self.stats.submitted_ta_runs += 1 info = ta.run_wrapper( RunInfo(config=config, cutoff=30, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.CRASHED) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float) self.assertEqual( info[1].additional_info, { 'configuration_origin': 'MOCK', 'error': "Result queue is empty", 'exit_status': 0, 'exitcode': 0, 'subprocess_stdout': '', 'subprocess_stderr': '' })
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock): pynisher_mock.return_value = None ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger) info = ta.run(None, cutoff=30, memory_limit=3000) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 2.0) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout(self, pynisher_mock): pynisher_mock.side_effect = safe_eval_success_mock ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger) info = ta.run(None, cutoff=30, memory_limit=3000) self.assertEqual(info[0], StatusType.SUCCESS) self.assertEqual(info[1], 0.5) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock): pynisher_mock.side_effect = pynisher.TimeoutException ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger) info = ta.run(None, cutoff=30, memory_limit=3000) self.assertEqual(info[0], StatusType.TIMEOUT) self.assertEqual(info[1], 2.0) self.assertIsInstance(info[2], float)
def test_cutoff_lower_than_remaining_time(self, pynisher_mock): ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072) ta.run(None, cutoff=30) self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4) self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'], int)
def test_zero_or_negative_cutoff(self, pynisher_mock): ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats) self.scenario.wallclock_limit = 5 info = ta.start(None, instance=None, cutoff=10) fixture = (StatusType.ABORT, np.nan, 0, {"misc": "exhausted bugdet -- ABORT"}) self.assertEqual(info, fixture)
def test_cutoff_lower_than_remaining_time(self, pynisher_mock): ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, metric=accuracy) self.stats.ta_runs = 1 ta.start(None, cutoff=30, instance=None) self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4) self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'], int)
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock): pynisher_mock.return_value = None ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger) info = ta.run(None, cutoff=30, memory_limit=3000) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock): pynisher_mock.side_effect = MemoryError ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger) info = ta.run(None, cutoff=30, memory_limit=3000) self.assertEqual(info[0], StatusType.MEMOUT) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock): pynisher_mock.side_effect = pynisher.TimeoutException ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072) info = ta.run(None, cutoff=30) self.assertEqual(info[0], StatusType.TIMEOUT) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock): pynisher_mock.side_effect = MemoryError ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy) info = ta.start(None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.MEMOUT) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_timeout_with_results_in_queue(self, pynisher_mock): def side_effect(**kwargs): queue = kwargs['queue'] queue.put((StatusType.SUCCESS, 0.5, 0.12345, '')) pynisher_mock.side_effect = side_effect ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072) info = ta.run(None, cutoff=30) self.assertEqual(info[0], StatusType.SUCCESS) self.assertEqual(info[1], 0.5) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock): pynisher_mock.side_effect = MemoryError ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=log_loss) info = ta.start(None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.MEMOUT) # For logloss, worst possible result is MAXINT worst_possible_result = MAXINT self.assertEqual(info[1], worst_possible_result) self.assertIsInstance(info[2], float)
def test_exception_in_target_function(self, eval_holdout_mock): eval_holdout_mock.side_effect = ValueError ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy) self.stats.ta_runs += 1 info = ta.start(None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float) self.assertEqual(info[3]['error'], 'ValueError()') self.assertIn('traceback', info[3])
def test_eval_with_limits_holdout(self, eval_houldout_mock): def side_effect(*args, **kwargs): queue = kwargs['queue'] queue.put((StatusType.SUCCESS, 0.5, 0.12345, kwargs['subsample'])) eval_houldout_mock.side_effect = side_effect ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072) self.scenario.wallclock_limit = 180 info = ta.start(None, cutoff=30, instance=None, instance_specific='subsample=30') self.assertEqual(info[0], StatusType.SUCCESS) self.assertEqual(info[-1], 30)
def test_zero_or_negative_cutoff(self, pynisher_mock): config = unittest.mock.Mock() config.config_id = 198 ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1, port=self.logger_port, resampling_strategy='holdout', stats=self.stats, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, ) self.scenario.wallclock_limit = 5 self.stats.submitted_ta_runs += 1 run_info, run_value = ta.run_wrapper(RunInfo(config=config, cutoff=9, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(run_value.status, StatusType.STOP)
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock): m1 = unittest.mock.Mock() m2 = unittest.mock.Mock() m1.return_value = m2 pynisher_mock.return_value = m1 m2.exit_status = pynisher.TimeoutException m2.wall_clock_time = 30 ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy) info = ta.start(config=None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.TIMEOUT) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout(self, pynisher_mock): pynisher_mock.side_effect = safe_eval_success_mock ta = ExecuteTaFuncWithQueue( backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, ) info = ta.start(None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.SUCCESS) self.assertEqual(info[1], 0.5) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_timeout_with_results_in_queue(self, pynisher_mock): def side_effect(**kwargs): queue = kwargs['queue'] queue.put({'status': StatusType.SUCCESS, 'loss': 0.5, 'additional_run_info': {}}) m1 = unittest.mock.Mock() m2 = unittest.mock.Mock() m1.return_value = m2 pynisher_mock.return_value = m1 m2.side_effect = side_effect m2.exit_status = pynisher.TimeoutException m2.wall_clock_time = 30 # Test for a succesful run ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy) info = ta.start(None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.SUCCESS) self.assertEqual(info[1], 0.5) self.assertIsInstance(info[2], float) # And a crashed run which is in the queue def side_effect(**kwargs): queue = kwargs['queue'] queue.put({'status': StatusType.CRASHED, 'loss': 2.0, 'additional_run_info': {}}) m2.side_effect = side_effect ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy) info = ta.start(None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_2(self, eval_houldout_mock): def side_effect(*args, **kwargs): queue = kwargs['queue'] queue.put({'status': StatusType.SUCCESS, 'loss': 0.5, 'additional_run_info': kwargs['instance']}) eval_houldout_mock.side_effect = side_effect ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy) self.scenario.wallclock_limit = 180 instance = "{'subsample': 30}" info = ta.start(None, cutoff=30, instance=instance) self.assertEqual(info[0], StatusType.SUCCESS) self.assertEqual(info[-1], {'message': "{'subsample': 30}", 'configuration_origin': 'UNKNOWN'})
def test_zero_or_negative_cutoff(self, pynisher_mock): ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, metric=accuracy) self.scenario.wallclock_limit = 5 self.stats.ta_runs += 1 self.assertRaises(BudgetExhaustedException, ta.start, None, instance=None, cutoff=9)
def test_silent_exception_in_target_function(self): config = unittest.mock.Mock() config.config_id = 198 delattr(self.backend, 'save_targets_ensemble') ta = ExecuteTaFuncWithQueue( backend=self.backend, port=self.logger_port, autosklearn_seed=1, resampling_strategy='holdout', stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, iterative=False, pynisher_context='fork', ) ta.pynisher_logger = unittest.mock.Mock() self.stats.submitted_ta_runs += 1 info = ta.run_wrapper( RunInfo(config=config, cutoff=3000, instance=None, instance_specific=None, seed=1, capped=False)) self.assertEqual(info[1].status, StatusType.CRASHED, msg=str(info[1].additional_info)) self.assertEqual(info[1].cost, 1.0) self.assertIsInstance(info[1].time, float) self.assertIn( info[1].additional_info['error'], ("""AttributeError("'BackendMock' object has no attribute """ """'save_targets_ensemble'",)""", """AttributeError("'BackendMock' object has no attribute """ """'save_targets_ensemble'")""", """AttributeError('save_targets_ensemble')""")) self.assertNotIn('exitcode', info[1].additional_info) self.assertNotIn('exit_status', info[1].additional_info) self.assertNotIn('traceback', info[1])
def test_eval_with_limits_holdout_timeout_with_results_in_queue( self, pynisher_mock): def side_effect(**kwargs): queue = kwargs['queue'] queue.put({ 'status': StatusType.SUCCESS, 'loss': 0.5, 'additional_run_info': {} }) m1 = unittest.mock.Mock() m2 = unittest.mock.Mock() m1.return_value = m2 pynisher_mock.return_value = m1 m2.side_effect = side_effect m2.exit_status = pynisher.TimeoutException m2.wall_clock_time = 30 # Test for a succesful run ta = ExecuteTaFuncWithQueue( backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, ) info = ta.start(None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.SUCCESS) self.assertEqual(info[1], 0.5) self.assertIsInstance(info[2], float) # And a crashed run which is in the queue def side_effect(**kwargs): queue = kwargs['queue'] queue.put({ 'status': StatusType.CRASHED, 'loss': 2.0, 'additional_run_info': {} }) m2.side_effect = side_effect ta = ExecuteTaFuncWithQueue( backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy, cost_for_crash=get_cost_of_crash(accuracy), abort_on_first_run_crash=False, ) info = ta.start(None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock): pynisher_mock.return_value = None ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger, stats=self.stats, memory_limit=3072, metric=accuracy) self.assertRaisesRegex(FirstRunCrashedException, "First run crashed, abort. Please check your " "setup -- we assume that your " "defaultconfiguration does not crashes. \(To " "deactivate this exception, use the SMAC " "scenario option 'abort_on_first_run_crash'\)", ta.start, config=None, instance=None, cutoff=30) self.stats.ta_runs += 1 info = ta.start(config=None, instance=None, cutoff=30) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 1.0) self.assertIsInstance(info[2], float) self.assertEqual(info[3], {'configuration_origin': 'UNKNOWN', 'error': "Result queue is empty"})
logger = logging.getLogger('Testing:)') stats = Stats( Scenario({ 'cutoff_time': per_run_time_limit * 2, 'run_obj': 'quality', }) ) stats.start_timing() # To avoid the output "first run crashed"... stats.ta_runs += 1 ta = ExecuteTaFuncWithQueue(backend=automl._automl._backend, autosklearn_seed=seed, resampling_strategy='test', memory_limit=memory_limit_factor * automl_arguments['ml_memory_limit'], disable_file_output=True, logger=logger, stats=stats, all_scoring_functions=True, metric=metric) status, cost, runtime, additional_run_info = ta.start( config=config, instance=None, cutoff=per_run_time_limit*3) if status == StatusType.SUCCESS: assert len(additional_run_info) > 1, additional_run_info # print(additional_run_info) validated_trajectory.append(list(entry) + [task_id] + [additional_run_info])