def _do_dummy_prediction(self, datamanager, num_run):
self._logger.info("Starting to create dummy predictions.")
# time_limit = int(self._time_for_task / 6.)
memory_limit = int(self._ml_memory_limit)
ta = ExecuteTaFuncWithQueue(backend=self._backend,
autosklearn_seed=self._seed,
resampling_strategy=self._resampling_strategy,
initial_num_run=num_run,
logger=self._logger,
**self._resampling_strategy_arguments)
status, cost, runtime, additional_info = \
ta.run(1, cutoff=self._time_for_task, memory_limit=memory_limit)
if status == StatusType.SUCCESS:
self._logger.info("Finished creating dummy predictions.")
else:
self._logger.error('Error creating dummy predictions:%s ',
additional_info)
#status, cost, runtime, additional_info = \
# ta.run(2, cutoff=time_limit, memory_limit=memory_limit)
#if status == StatusType.SUCCESS:
# self._logger.info("Finished creating dummy prediction 2/2.")
#else:
# self._logger.error('Error creating dummy prediction 2/2 %s',
# additional_info)
return ta.num_run
def _do_dummy_prediction(self, datamanager, num_run):
# When using partial-cv it makes no sense to do dummy predictions
if self._resampling_strategy in ['partial-cv',
'partial-cv-iterative-fit']:
return num_run
self._logger.info("Starting to create dummy predictions.")
memory_limit = int(self._ml_memory_limit)
scenario_mock = unittest.mock.Mock()
scenario_mock.wallclock_limit = self._time_for_task
# This stats object is a hack - maybe the SMAC stats object should
# already be generated here!
stats = Stats(scenario_mock)
stats.start_timing()
ta = ExecuteTaFuncWithQueue(backend=self._backend,
autosklearn_seed=self._seed,
resampling_strategy=self._resampling_strategy,
initial_num_run=num_run,
logger=self._logger,
stats=stats,
metric=self._metric,
memory_limit=memory_limit,
disable_file_output=self._disable_evaluator_output,
**self._resampling_strategy_arguments)
status, cost, runtime, additional_info = \
ta.run(1, cutoff=self._time_for_task)
if status == StatusType.SUCCESS:
self._logger.info("Finished creating dummy predictions.")
else:
self._logger.error('Error creating dummy predictions: %s ',
str(additional_info))
return ta.num_run
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock):
pynisher_mock.side_effect = pynisher.TimeoutException
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger)
info = ta.run(None, cutoff=30, memory_limit=3000)
self.assertEqual(info[0], StatusType.TIMEOUT)
self.assertEqual(info[1], 2.0)
self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock):
pynisher_mock.return_value = None
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger)
info = ta.run(None, cutoff=30, memory_limit=3000)
self.assertEqual(info[0], StatusType.CRASHED)
self.assertEqual(info[1], 2.0)
self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout(self, pynisher_mock):
pynisher_mock.side_effect = safe_eval_success_mock
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger)
info = ta.run(None, cutoff=30, memory_limit=3000)
self.assertEqual(info[0], StatusType.SUCCESS)
self.assertEqual(info[1], 0.5)
self.assertIsInstance(info[2], float)
def _do_dummy_prediction(self, datamanager, num_run):
# When using partial-cv it makes no sense to do dummy predictions
if self._resampling_strategy in [
'partial-cv', 'partial-cv-iterative-fit'
]:
return num_run
self._logger.info("Starting to create dummy predictions.")
memory_limit = int(self._ml_memory_limit)
scenario_mock = unittest.mock.Mock()
scenario_mock.wallclock_limit = self._time_for_task
# This stats object is a hack - maybe the SMAC stats object should
# already be generated here!
stats = Stats(scenario_mock)
stats.start_timing()
ta = ExecuteTaFuncWithQueue(
backend=self._backend,
autosklearn_seed=self._seed,
resampling_strategy=self._resampling_strategy,
initial_num_run=num_run,
logger=self._logger,
stats=stats,
metric=self._metric,
memory_limit=memory_limit,
disable_file_output=self._disable_evaluator_output,
**self._resampling_strategy_arguments)
status, cost, runtime, additional_info = \
ta.run(1, cutoff=self._time_for_task)
if status == StatusType.SUCCESS:
self._logger.info("Finished creating dummy predictions.")
else:
self._logger.error('Error creating dummy predictions: %s ',
str(additional_info))
# Fail if dummy prediction fails.
raise ValueError("Dummy prediction failed: %s " %
str(additional_info))
return ta.num_run
