def _do_dummy_prediction(self, datamanager, num_run): self._logger.info("Starting to create dummy predictions.") # time_limit = int(self._time_for_task / 6.) memory_limit = int(self._ml_memory_limit) ta = ExecuteTaFuncWithQueue(backend=self._backend, autosklearn_seed=self._seed, resampling_strategy=self._resampling_strategy, initial_num_run=num_run, logger=self._logger, **self._resampling_strategy_arguments) status, cost, runtime, additional_info = \ ta.run(1, cutoff=self._time_for_task, memory_limit=memory_limit) if status == StatusType.SUCCESS: self._logger.info("Finished creating dummy predictions.") else: self._logger.error('Error creating dummy predictions:%s ', additional_info) #status, cost, runtime, additional_info = \ # ta.run(2, cutoff=time_limit, memory_limit=memory_limit) #if status == StatusType.SUCCESS: # self._logger.info("Finished creating dummy prediction 2/2.") #else: # self._logger.error('Error creating dummy prediction 2/2 %s', # additional_info) return ta.num_run
def _do_dummy_prediction(self, datamanager, num_run): # When using partial-cv it makes no sense to do dummy predictions if self._resampling_strategy in ['partial-cv', 'partial-cv-iterative-fit']: return num_run self._logger.info("Starting to create dummy predictions.") memory_limit = int(self._ml_memory_limit) scenario_mock = unittest.mock.Mock() scenario_mock.wallclock_limit = self._time_for_task # This stats object is a hack - maybe the SMAC stats object should # already be generated here! stats = Stats(scenario_mock) stats.start_timing() ta = ExecuteTaFuncWithQueue(backend=self._backend, autosklearn_seed=self._seed, resampling_strategy=self._resampling_strategy, initial_num_run=num_run, logger=self._logger, stats=stats, metric=self._metric, memory_limit=memory_limit, disable_file_output=self._disable_evaluator_output, **self._resampling_strategy_arguments) status, cost, runtime, additional_info = \ ta.run(1, cutoff=self._time_for_task) if status == StatusType.SUCCESS: self._logger.info("Finished creating dummy predictions.") else: self._logger.error('Error creating dummy predictions: %s ', str(additional_info)) return ta.num_run
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock): pynisher_mock.side_effect = pynisher.TimeoutException ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger) info = ta.run(None, cutoff=30, memory_limit=3000) self.assertEqual(info[0], StatusType.TIMEOUT) self.assertEqual(info[1], 2.0) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock): pynisher_mock.return_value = None ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger) info = ta.run(None, cutoff=30, memory_limit=3000) self.assertEqual(info[0], StatusType.CRASHED) self.assertEqual(info[1], 2.0) self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout(self, pynisher_mock): pynisher_mock.side_effect = safe_eval_success_mock ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1, resampling_strategy='holdout', logger=self.logger) info = ta.run(None, cutoff=30, memory_limit=3000) self.assertEqual(info[0], StatusType.SUCCESS) self.assertEqual(info[1], 0.5) self.assertIsInstance(info[2], float)
def _do_dummy_prediction(self, datamanager, num_run): # When using partial-cv it makes no sense to do dummy predictions if self._resampling_strategy in [ 'partial-cv', 'partial-cv-iterative-fit' ]: return num_run self._logger.info("Starting to create dummy predictions.") memory_limit = int(self._ml_memory_limit) scenario_mock = unittest.mock.Mock() scenario_mock.wallclock_limit = self._time_for_task # This stats object is a hack - maybe the SMAC stats object should # already be generated here! stats = Stats(scenario_mock) stats.start_timing() ta = ExecuteTaFuncWithQueue( backend=self._backend, autosklearn_seed=self._seed, resampling_strategy=self._resampling_strategy, initial_num_run=num_run, logger=self._logger, stats=stats, metric=self._metric, memory_limit=memory_limit, disable_file_output=self._disable_evaluator_output, **self._resampling_strategy_arguments) status, cost, runtime, additional_info = \ ta.run(1, cutoff=self._time_for_task) if status == StatusType.SUCCESS: self._logger.info("Finished creating dummy predictions.") else: self._logger.error('Error creating dummy predictions: %s ', str(additional_info)) # Fail if dummy prediction fails. raise ValueError("Dummy prediction failed: %s " % str(additional_info)) return ta.num_run