def test_zero_or_negative_cutoff(self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
)
self.scenario.wallclock_limit = 5
self.stats.submitted_ta_runs += 1
run_info, run_value = ta.run_wrapper(
RunInfo(config=config,
cutoff=9,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(run_value.status, StatusType.STOP)
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock):
pynisher_mock.side_effect = MemoryError
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1,
port=self.logger_port,
resampling_strategy='holdout',
stats=self.stats,
memory_limit=3072,
metric=log_loss,
cost_for_crash=get_cost_of_crash(log_loss),
abort_on_first_run_crash=False,
pynisher_context='fork',
)
info = ta.run_wrapper(RunInfo(config=config, cutoff=30, instance=None,
instance_specific=None, seed=1, capped=False))
self.assertEqual(info[1].status, StatusType.MEMOUT)
# For logloss, worst possible result is MAXINT
worst_possible_result = MAXINT
self.assertEqual(info[1].cost, worst_possible_result)
self.assertIsInstance(info[1].time, float)
self.assertNotIn('exitcode', info[1].additional_info)
def test_silent_exception_in_target_function(self):
config = unittest.mock.Mock()
config.config_id = 198
delattr(self.backend, 'save_targets_ensemble')
ta = ExecuteTaFuncWithQueue(backend=self.backend,
port=self.logger_port,
autosklearn_seed=1,
resampling_strategy='holdout',
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
iterative=False,
pynisher_context='fork',
)
ta.pynisher_logger = unittest.mock.Mock()
self.stats.submitted_ta_runs += 1
info = ta.run_wrapper(RunInfo(config=config, cutoff=3000, instance=None,
instance_specific=None, seed=1, capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED, msg=str(info[1].additional_info))
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertIn(
info[1].additional_info['error'],
(
"""AttributeError("'BackendMock' object has no attribute """
"""'save_targets_ensemble'",)""",
"""AttributeError("'BackendMock' object has no attribute """
"""'save_targets_ensemble'")""",
"""AttributeError('save_targets_ensemble')"""
)
)
self.assertNotIn('exitcode', info[1].additional_info)
self.assertNotIn('exit_status', info[1].additional_info)
self.assertNotIn('traceback', info[1])
def test_exception_in_target_function(self, eval_holdout_mock):
config = unittest.mock.Mock()
config.config_id = 198
eval_holdout_mock.side_effect = ValueError
ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1,
port=self.logger_port,
resampling_strategy='holdout',
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
pynisher_context='fork',
)
self.stats.submitted_ta_runs += 1
info = ta.run_wrapper(RunInfo(config=config, cutoff=30, instance=None,
instance_specific=None, seed=1, capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertEqual(info[1].additional_info['error'], 'ValueError()')
self.assertIn('traceback', info[1].additional_info)
self.assertNotIn('exitcode', info[1].additional_info)
resampling_strategy='test',
memory_limit=memory_lim,
disable_file_output=True,
logger=logger,
stats=stats,
scoring_functions=scoring_functions,
include=include,
metric=automl_arguments['metric'],
cost_for_crash=get_cost_of_crash(automl_arguments['metric']),
abort_on_first_run_crash=False,
)
run_info, run_value = ta.run_wrapper(
RunInfo(
config=config,
instance=None,
instance_specific=None,
seed=1,
cutoff=per_run_time_limit * 3,
capped=False,
))
if run_value.status == StatusType.SUCCESS:
assert len(
run_value.additional_info) > 1, run_value.additional_info
# print(additional_run_info)
validated_trajectory.append(
list(entry) + [task_id] + [run_value.additional_info])
print('Finished validating configuration %d/%d' %
(i + 1, len(trajectory)))
def test_eval_with_limits_holdout_timeout_with_results_in_queue(
self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
def side_effect(**kwargs):
queue = kwargs['queue']
queue.put({
'status': StatusType.SUCCESS,
'loss': 0.5,
'additional_run_info': {}
})
m1 = unittest.mock.Mock()
m2 = unittest.mock.Mock()
m1.return_value = m2
pynisher_mock.return_value = m1
m2.side_effect = side_effect
m2.exit_status = pynisher.TimeoutException
m2.wall_clock_time = 30
# Test for a succesful run
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.SUCCESS)
self.assertEqual(info[1].cost, 0.5)
self.assertIsInstance(info[1].time, float)
# And a crashed run which is in the queue
def side_effect(**kwargs):
queue = kwargs['queue']
queue.put({
'status': StatusType.CRASHED,
'loss': 2.0,
'additional_run_info': {}
})
m2.side_effect = side_effect
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)