def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock):
pynisher_mock.return_value = None
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
)
# The following should not fail because abort on first config crashed is false
info = ta.start(config=None, instance=None, cutoff=60)
self.assertEqual(info[0], StatusType.CRASHED)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
self.assertEqual(info[3], {
'configuration_origin': 'UNKNOWN',
'error': "Result queue is empty"
})
self.stats.ta_runs += 1
info = ta.start(config=None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.CRASHED)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
self.assertEqual(info[3], {
'configuration_origin': 'UNKNOWN',
'error': "Result queue is empty"
})
def test_eval_with_limits_holdout_timeout_with_results_in_queue(
self, pynisher_mock):
def side_effect(**kwargs):
queue = kwargs['queue']
queue.put({
'status': StatusType.SUCCESS,
'loss': 0.5,
'additional_run_info': {}
})
m1 = unittest.mock.Mock()
m2 = unittest.mock.Mock()
m1.return_value = m2
pynisher_mock.return_value = m1
m2.side_effect = side_effect
m2.exit_status = pynisher.TimeoutException
m2.wall_clock_time = 30
# Test for a succesful run
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
)
info = ta.start(None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.SUCCESS)
self.assertEqual(info[1], 0.5)
self.assertIsInstance(info[2], float)
# And a crashed run which is in the queue
def side_effect(**kwargs):
queue = kwargs['queue']
queue.put({
'status': StatusType.CRASHED,
'loss': 2.0,
'additional_run_info': {}
})
m2.side_effect = side_effect
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
)
info = ta.start(None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.CRASHED)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
def test_cutoff_lower_than_remaining_time(self, pynisher_mock):
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats)
self.stats.ta_runs = 1
ta.start(None, cutoff=30, instance=None)
self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4)
self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'], int)
def test_cutoff_lower_than_remaining_time(self, pynisher_mock):
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
metric=accuracy)
self.stats.ta_runs = 1
ta.start(None, cutoff=30, instance=None)
self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4)
self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'], int)
def test_cutoff_lower_than_remaining_time(self, pynisher_mock):
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
)
self.stats.ta_runs = 1
ta.start(None, cutoff=30, instance=None)
self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4)
self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'],
int)
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock):
pynisher_mock.return_value = None
ta = ExecuteTaFuncWithQueue(backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
self.assertRaisesRegex(FirstRunCrashedException,
"First run crashed, abort. \(To prevent this, "
"toggle the "
"'abort_on_first_run_crash'-option!\)",
ta.start,
config=None,
instance=None,
cutoff=30)
self.stats.ta_runs += 1
info = ta.start(config=None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.CRASHED)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
self.assertEqual(info[3], {'error': "Result queue is empty"})
def test_eval_with_limits_holdout_2(self, eval_houldout_mock):
def side_effect(*args, **kwargs):
queue = kwargs['queue']
queue.put({
'status': StatusType.SUCCESS,
'loss': 0.5,
'additional_run_info': kwargs['instance']
})
eval_houldout_mock.side_effect = side_effect
ta = ExecuteTaFuncWithQueue(backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
self.scenario.wallclock_limit = 180
instance = "{'subsample': 30}"
info = ta.start(None, cutoff=30, instance=instance)
self.assertEqual(info[0], StatusType.SUCCESS)
self.assertEqual(info[-1], {
'message': "{'subsample': 30}",
'configuration_origin': 'UNKNOWN'
})
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock):
pynisher_mock.return_value = None
ta = ExecuteTaFuncWithQueue(backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
self.assertRaisesRegex(FirstRunCrashedException,
"First run crashed, abort. Please check your "
"setup -- we assume that your "
"defaultconfiguration does not crashes. \(To "
"deactivate this exception, use the SMAC "
"scenario option 'abort_on_first_run_crash'\)",
ta.start,
config=None,
instance=None,
cutoff=30)
self.stats.ta_runs += 1
info = ta.start(config=None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.CRASHED)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
self.assertEqual(info[3], {
'configuration_origin': 'UNKNOWN',
'error': "Result queue is empty"
})
def test_zero_or_negative_cutoff(self, pynisher_mock):
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats)
self.scenario.wallclock_limit = 5
info = ta.start(None, instance=None, cutoff=10)
fixture = (StatusType.ABORT, np.nan, 0, {"misc": "exhausted bugdet -- ABORT"})
self.assertEqual(info, fixture)
def test_eval_with_limits_holdout_timeout_with_results_in_queue(self, pynisher_mock):
def side_effect(**kwargs):
queue = kwargs['queue']
queue.put({'status': StatusType.SUCCESS,
'loss': 0.5,
'additional_run_info': {}})
m1 = unittest.mock.Mock()
m2 = unittest.mock.Mock()
m1.return_value = m2
pynisher_mock.return_value = m1
m2.side_effect = side_effect
m2.exit_status = pynisher.TimeoutException
m2.wall_clock_time = 30
# Test for a succesful run
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
info = ta.start(None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.SUCCESS)
self.assertEqual(info[1], 0.5)
self.assertIsInstance(info[2], float)
# And a crashed run which is in the queue
def side_effect(**kwargs):
queue = kwargs['queue']
queue.put({'status': StatusType.CRASHED,
'loss': 2.0,
'additional_run_info': {}})
m2.side_effect = side_effect
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
info = ta.start(None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.CRASHED)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout(self, pynisher_mock):
pynisher_mock.side_effect = safe_eval_success_mock
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
info = ta.start(None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.SUCCESS)
self.assertEqual(info[1], 0.5)
self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout(self, pynisher_mock):
pynisher_mock.side_effect = safe_eval_success_mock
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
info = ta.start(None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.SUCCESS)
self.assertEqual(info[1], 0.5)
self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout(self, eval_houldout_mock):
def side_effect(*args, **kwargs):
queue = kwargs['queue']
queue.put((StatusType.SUCCESS, 0.5, 0.12345, kwargs['subsample']))
eval_houldout_mock.side_effect = side_effect
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072)
self.scenario.wallclock_limit = 180
info = ta.start(None, cutoff=30, instance=None,
instance_specific='subsample=30')
self.assertEqual(info[0], StatusType.SUCCESS)
self.assertEqual(info[-1], 30)
def test_exception_in_target_function(self, eval_holdout_mock):
eval_holdout_mock.side_effect = ValueError
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
self.stats.ta_runs += 1
info = ta.start(None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.CRASHED)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
self.assertEqual(info[3]['error'], 'ValueError()')
self.assertIn('traceback', info[3])
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock):
pynisher_mock.side_effect = MemoryError
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=log_loss)
info = ta.start(None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.MEMOUT)
# For logloss, worst possible result is MAXINT
worst_possible_result = MAXINT
self.assertEqual(info[1], worst_possible_result)
self.assertIsInstance(info[2], float)
def test_exception_in_target_function(self, eval_holdout_mock):
eval_holdout_mock.side_effect = ValueError
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
self.stats.ta_runs += 1
info = ta.start(None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.CRASHED)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
self.assertEqual(info[3]['error'], 'ValueError()')
self.assertIn('traceback', info[3])
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock):
m1 = unittest.mock.Mock()
m2 = unittest.mock.Mock()
m1.return_value = m2
pynisher_mock.return_value = m1
m2.exit_status = pynisher.TimeoutException
m2.wall_clock_time = 30
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
info = ta.start(config=None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.TIMEOUT)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock):
m1 = unittest.mock.Mock()
m2 = unittest.mock.Mock()
m1.return_value = m2
pynisher_mock.return_value = m1
m2.exit_status = pynisher.TimeoutException
m2.wall_clock_time = 30
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
info = ta.start(config=None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.TIMEOUT)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
def test_eval_with_limits_holdout_2(self, eval_houldout_mock):
def side_effect(*args, **kwargs):
queue = kwargs['queue']
queue.put({'status': StatusType.SUCCESS,
'loss': 0.5,
'additional_run_info': kwargs['instance']})
eval_houldout_mock.side_effect = side_effect
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
self.scenario.wallclock_limit = 180
instance = "{'subsample': 30}"
info = ta.start(None, cutoff=30, instance=instance)
self.assertEqual(info[0], StatusType.SUCCESS)
self.assertEqual(info[-1], {'message': "{'subsample': 30}",
'configuration_origin': 'UNKNOWN'})
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock):
pynisher_mock.return_value = None
ta = ExecuteTaFuncWithQueue(backend=BackendMock(), autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=accuracy)
self.assertRaisesRegex(FirstRunCrashedException,
"First run crashed, abort. Please check your "
"setup -- we assume that your "
"defaultconfiguration does not crashes. \(To "
"deactivate this exception, use the SMAC "
"scenario option 'abort_on_first_run_crash'\)",
ta.start, config=None, instance=None, cutoff=30)
self.stats.ta_runs += 1
info = ta.start(config=None, instance=None, cutoff=30)
self.assertEqual(info[0], StatusType.CRASHED)
self.assertEqual(info[1], 1.0)
self.assertIsInstance(info[2], float)
self.assertEqual(info[3], {'configuration_origin': 'UNKNOWN',
'error': "Result queue is empty"})
def run_configuration(backend, config_id, task_id, configuration, run_args,
memory_limit, per_run_time_limit):
evaluation, iterative_fit, early_stopping, N_FOLDS, searchspace = run_args
# TODO make this an argument from the command line!
scenario_mock = unittest.mock.Mock()
scenario_mock.wallclock_limit = per_run_time_limit * 100
scenario_mock.algo_runs_timelimit = per_run_time_limit * 100
scenario_mock.ta_run_limit = np.inf
stats = Stats(scenario_mock)
stats.ta_runs = 2
# Resampling strategies
kwargs = {}
if evaluation == "holdout" and iterative_fit:
resampling_strategy = 'holdout-iterative-fit'
elif evaluation == "holdout" and not iterative_fit:
resampling_strategy = 'holdout'
elif evaluation == "CV" and not iterative_fit:
resampling_strategy = 'cv'
kwargs = {'folds': N_FOLDS}
elif evaluation == "CV" and iterative_fit:
resampling_strategy = 'cv-iterative-fit'
kwargs = {'folds': N_FOLDS}
else:
raise ValueError("Unknown resampling strategy", evaluation)
iterative_wo_early_stopping = [
'extra_trees', 'PassiveAggressiveWOEarlyStopping', 'random_forest',
'SGDWOEarlyStopping', 'GradientBoostingClassifierWOEarlyStopping'
]
iterative_w_early_stopping = [
'extra_trees', 'passive_aggressive', 'random_forest', 'sgd',
'gradient_boosting'
]
if not early_stopping:
add_classifier_wo_early_stopping()
if searchspace == "iterative":
include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping
include_preprocessor = [
"no_preprocessing",
]
elif searchspace == "full":
assert early_stopping is True
include_estimator = None
include_preprocessor = None
# elif searchspace == 'only-iterative-nopreproc':
# include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping
# include_preprocessor = ["no_preprocessing", ]
# elif searchspace == 'only-iterative-cheappreproc':
# include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping
# include_preprocessor = ["no_preprocessing", 'kitchen_sinks', 'polynomial', 'select_percentile_classification', 'select_rates']
# elif searchspace == 'only-iterative':
# include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping
# include_preprocessor = None
# elif searchspace == "gb":
# include_estimator = ['GradientBoostingClassifierWOEarlyStopping'] if early_stopping else ['GradientBoostingClassifierWEarlyStopping']
# include_preprocessor = None
else:
raise ValueError(searchspace)
stats.start_timing()
tae = ExecuteTaFuncWithQueue(
backend=backend,
autosklearn_seed=3,
resampling_strategy=resampling_strategy,
metric=balanced_accuracy,
logger=logging.getLogger(name="%s_%s" % (task_id, config_id)),
initial_num_run=2,
stats=stats,
runhistory=None,
run_obj='quality',
par_factor=1,
all_scoring_functions=False,
output_y_hat_optimization=True,
include={
"classifier": include_estimator,
"feature_preprocessor": include_preprocessor
},
exclude=None,
memory_limit=memory_limit,
disable_file_output=True,
init_params=None,
**kwargs)
# Finally run configuration
status, cost, runtime, additional_run_info = tae.start(
config=configuration,
instance=None,
cutoff=per_run_time_limit,
instance_specific=None,
capped=False,
)
return status, cost, runtime, additional_run_info
'run_obj': 'quality',
}))
stats.start_timing()
# To avoid the output "first run crashed"...
stats.ta_runs += 1
ta = ExecuteTaFuncWithQueue(backend=automl._automl._backend,
autosklearn_seed=seed,
resampling_strategy='test',
memory_limit=memory_limit_factor *
automl_arguments['ml_memory_limit'],
disable_file_output=True,
logger=logger,
stats=stats,
all_scoring_functions=True,
metric=metric)
status, cost, runtime, additional_run_info = ta.start(
config=config, instance=None, cutoff=per_run_time_limit * 3)
if status == StatusType.SUCCESS:
assert len(additional_run_info) > 1, additional_run_info
# print(additional_run_info)
validated_trajectory.append(
list(entry) + [task_id] + [additional_run_info])
validated_trajectory = [
entry[:2] + [entry[2].get_dictionary()] + entry[3:]
for entry in validated_trajectory
]
validated_trajectory_file = os.path.join(tmp_dir, 'smac3-output',
'run_%d' % seed,
'run_obj': 'quality',
})
)
stats.start_timing()
# To avoid the output "first run crashed"...
stats.ta_runs += 1
ta = ExecuteTaFuncWithQueue(backend=automl._automl._backend,
autosklearn_seed=seed,
resampling_strategy='test',
memory_limit=memory_limit_factor * automl_arguments['ml_memory_limit'],
disable_file_output=True,
logger=logger,
stats=stats,
all_scoring_functions=True,
metric=metric)
status, cost, runtime, additional_run_info = ta.start(
config=config, instance=None, cutoff=per_run_time_limit*3)
if status == StatusType.SUCCESS:
assert len(additional_run_info) > 1, additional_run_info
# print(additional_run_info)
validated_trajectory.append(list(entry) + [task_id] +
[additional_run_info])
validated_trajectory = [entry[:2] + [entry[2].get_dictionary()] + entry[3:]
for entry in validated_trajectory]
validated_trajectory_file = os.path.join(tmp_dir,
'smac3-output',
'run_%d' % seed,
'validation_trajectory.json')