def test_eval_with_limits_holdout(self, pynisher_mock):
pynisher_mock.side_effect = safe_eval_success_mock
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[0].config.config_id, 198)
self.assertEqual(info[1].status, StatusType.SUCCESS, info)
self.assertEqual(info[1].cost, 0.5)
self.assertIsInstance(info[1].time, float)
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock):
pynisher_mock.side_effect = MemoryError
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.MEMOUT)
# For accuracy, worst possible result is MAXINT
worst_possible_result = 1
self.assertEqual(info[1].cost, worst_possible_result)
self.assertIsInstance(info[1].time, float)
self.assertNotIn('exitcode', info[1].additional_info)
def test_exception_in_target_function(self, eval_holdout_mock):
config = unittest.mock.Mock()
config.config_id = 198
eval_holdout_mock.side_effect = ValueError
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
self.stats.submitted_ta_runs += 1
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertEqual(info[1].additional_info['error'], 'ValueError()')
self.assertIn('traceback', info[1].additional_info)
self.assertNotIn('exitcode', info[1].additional_info)
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
m1 = unittest.mock.Mock()
m2 = unittest.mock.Mock()
m1.return_value = m2
pynisher_mock.return_value = m1
m2.exit_status = pynisher.TimeoutException
m2.wall_clock_time = 30
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.TIMEOUT)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertNotIn('exitcode', info[1].additional_info)
def test_cutoff_lower_than_remaining_time(self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
self.stats.ta_runs = 1
ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4)
self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'],
int)
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock):
pynisher_mock.return_value = None
config = unittest.mock.Mock()
config.origin = 'MOCK'
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
# The following should not fail because abort on first config crashed is false
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=60,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertEqual(
info[1].additional_info, {
'configuration_origin': 'MOCK',
'error': "Result queue is empty",
'exit_status': '0',
'exitcode': 0,
'subprocess_stdout': '',
'subprocess_stderr': ''
})
self.stats.submitted_ta_runs += 1
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertEqual(
info[1].additional_info, {
'configuration_origin': 'MOCK',
'error': "Result queue is empty",
'exit_status': '0',
'exitcode': 0,
'subprocess_stdout': '',
'subprocess_stderr': ''
})
def test_silent_exception_in_target_function(self):
config = unittest.mock.Mock(spec=int)
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
ta.pynisher_logger = unittest.mock.Mock()
self.stats.submitted_ta_runs += 1
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=3000,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status,
StatusType.CRASHED,
msg=str(info[1].additional_info))
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertIn(info[1].additional_info['error'], (
"""AttributeError("'BackendMock' object has no attribute """
"""'save_targets_ensemble'",)""",
"""AttributeError("'BackendMock' object has no attribute """
"""'save_targets_ensemble'")""",
"""AttributeError('save_targets_ensemble')"""
"""AttributeError("'BackendMock' object has no attribute """
"""'setup_logger'",)""",
"""AttributeError("'BackendMock' object has no attribute """
"""'setup_logger'")""",
))
self.assertNotIn('exitcode', info[1].additional_info)
self.assertNotIn('exit_status', info[1].additional_info)
self.assertNotIn('traceback', info[1])
def test_eval_with_limits_holdout_2(self, eval_houldout_mock):
config = unittest.mock.Mock()
config.config_id = 198
def side_effect(*args, **kwargs):
queue = kwargs['queue']
queue.put({
'status': StatusType.SUCCESS,
'loss': 0.5,
'additional_run_info': kwargs['instance']
})
eval_houldout_mock.side_effect = side_effect
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
self.scenario.wallclock_limit = 180
instance = "{'subsample': 30}"
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=instance,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.SUCCESS, info)
self.assertEqual(len(info[1].additional_info), 2)
self.assertIn('configuration_origin', info[1].additional_info)
self.assertEqual(info[1].additional_info['message'],
"{'subsample': 30}")
def test_zero_or_negative_cutoff(self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
self.scenario.wallclock_limit = 5
self.stats.submitted_ta_runs += 1
run_info, run_value = ta.run_wrapper(
RunInfo(config=config,
cutoff=9,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(run_value.status, StatusType.STOP)
def test_get_cost_of_crash(metric, expected):
assert get_cost_of_crash(metric) == expected
def test_eval_with_limits_holdout_timeout_with_results_in_queue(
self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
def side_effect(**kwargs):
queue = kwargs['queue']
queue.put({
'status': StatusType.SUCCESS,
'loss': 0.5,
'additional_run_info': {}
})
m1 = unittest.mock.Mock()
m2 = unittest.mock.Mock()
m1.return_value = m2
pynisher_mock.return_value = m1
m2.side_effect = side_effect
m2.exit_status = pynisher.TimeoutException
m2.wall_clock_time = 30
# Test for a succesful run
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.SUCCESS)
self.assertEqual(info[1].cost, 0.5)
self.assertIsInstance(info[1].time, float)
self.assertNotIn('exitcode', info[1].additional_info)
# And a crashed run which is in the queue
def side_effect(**kwargs):
queue = kwargs['queue']
queue.put({
'status': StatusType.CRASHED,
'loss': 2.0,
'additional_run_info': {}
})
m2.side_effect = side_effect
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertNotIn('exitcode', info[1].additional_info)
def __init__(
self,
config_space: ConfigSpace.ConfigurationSpace,
dataset_name: str,
backend: Backend,
total_walltime_limit: float,
func_eval_time_limit_secs: float,
memory_limit: Optional[int],
metric: autoPyTorchMetric,
watcher: StopWatch,
n_jobs: int,
dask_client: Optional[dask.distributed.Client],
pipeline_config: Dict[str, Any],
start_num_run: int = 1,
seed: int = 1,
resampling_strategy: Union[
HoldoutValTypes,
CrossValTypes] = HoldoutValTypes.holdout_validation,
resampling_strategy_args: Optional[Dict[str, Any]] = None,
include: Optional[Dict[str, Any]] = None,
exclude: Optional[Dict[str, Any]] = None,
disable_file_output: List = [],
smac_scenario_args: Optional[Dict[str, Any]] = None,
get_smac_object_callback: Optional[Callable] = None,
all_supported_metrics: bool = True,
ensemble_callback: Optional[EnsembleBuilderManager] = None,
logger_port: Optional[int] = None,
search_space_updates: Optional[
HyperparameterSearchSpaceUpdates] = None,
portfolio_selection: Optional[str] = None,
pynisher_context: str = 'spawn',
min_budget: int = 5,
max_budget: int = 50,
):
"""
Interface to SMAC. This method calls the SMAC optimize method, and allows
to pass a callback (ensemble_callback) to make launch task at the end of each
optimize() algorithm. The later is needed due to the nature of blocking long running
tasks in Dask.
Args:
config_space (ConfigSpace.ConfigurationSpac):
The configuration space of the whole process
dataset_name (str):
The name of the dataset, used to identify the current job
backend (Backend):
An interface with disk
total_walltime_limit (float):
The maximum allowed time for this job
func_eval_time_limit_secs (float):
How much each individual task is allowed to last
memory_limit (Optional[int]):
Maximum allowed CPU memory this task can use
metric (autoPyTorchMetric):
An scorer object to evaluate the performance of each jon
watcher (StopWatch):
A stopwatch object to debug time consumption
n_jobs (int):
How many workers are allowed in each task
dask_client (Optional[dask.distributed.Client]):
An user provided scheduler. Else smac will create its own.
start_num_run (int):
The ID index to start runs
seed (int):
To make the run deterministic
resampling_strategy (str):
What strategy to use for performance validation
resampling_strategy_args (Optional[Dict[str, Any]]):
Arguments to the resampling strategy -- like number of folds
include (Optional[Dict[str, Any]] = None):
Optimal Configuration space modifiers
exclude (Optional[Dict[str, Any]] = None):
Optimal Configuration space modifiers
disable_file_output List:
Support to disable file output to disk -- to reduce space
smac_scenario_args (Optional[Dict[str, Any]]):
Additional arguments to the smac scenario
get_smac_object_callback (Optional[Callable]):
Allows to create a user specified SMAC object
pynisher_context (str):
A string indicating the multiprocessing context to use
ensemble_callback (Optional[EnsembleBuilderManager]):
A callback used in this scenario to start ensemble building subtasks
portfolio_selection (Optional[str]):
This argument controls the initial configurations that
AutoPyTorch uses to warm start SMAC for hyperparameter
optimization. By default, no warm-starting happens.
The user can provide a path to a json file containing
configurations, similar to (autoPyTorch/configs/greedy_portfolio.json).
Additionally, the keyword 'greedy' is supported,
which would use the default portfolio from
`AutoPyTorch Tabular <https://arxiv.org/abs/2006.13799>_`
min_budget (int):
Auto-PyTorch uses `Hyperband <https://arxiv.org/abs/1603.06560>_` to
trade-off resources between running many pipelines at min_budget and
running the top performing pipelines on max_budget.
min_budget states the minimum resource allocation a pipeline should have
so that we can compare and quickly discard bad performing models.
For example, if the budget_type is epochs, and min_budget=5, then we will
run every pipeline to a minimum of 5 epochs before performance comparison.
max_budget (int):
Auto-PyTorch uses `Hyperband <https://arxiv.org/abs/1603.06560>_` to
trade-off resources between running many pipelines at min_budget and
running the top performing pipelines on max_budget.
max_budget states the maximum resource allocation a pipeline is going to
be ran. For example, if the budget_type is epochs, and max_budget=50,
then the pipeline training will be terminated after 50 epochs.
"""
super(AutoMLSMBO, self).__init__()
# data related
self.dataset_name = dataset_name
self.datamanager: Optional[BaseDataset] = None
self.metric = metric
self.task: Optional[str] = None
self.backend = backend
self.all_supported_metrics = all_supported_metrics
self.pipeline_config = pipeline_config
# the configuration space
self.config_space = config_space
# the number of parallel workers/jobs
self.n_jobs = n_jobs
self.dask_client = dask_client
# Evaluation
self.resampling_strategy = resampling_strategy
if resampling_strategy_args is None:
resampling_strategy_args = DEFAULT_RESAMPLING_PARAMETERS[
resampling_strategy]
self.resampling_strategy_args = resampling_strategy_args
# and a bunch of useful limits
self.worst_possible_result = get_cost_of_crash(self.metric)
self.total_walltime_limit = int(total_walltime_limit)
self.func_eval_time_limit_secs = int(func_eval_time_limit_secs)
self.memory_limit = memory_limit
self.watcher = watcher
self.seed = seed
self.start_num_run = start_num_run
self.include = include
self.exclude = exclude
self.disable_file_output = disable_file_output
self.smac_scenario_args = smac_scenario_args
self.get_smac_object_callback = get_smac_object_callback
self.pynisher_context = pynisher_context
self.min_budget = min_budget
self.max_budget = max_budget
self.ensemble_callback = ensemble_callback
self.search_space_updates = search_space_updates
if logger_port is None:
self.logger_port = logging.handlers.DEFAULT_TCP_LOGGING_PORT
else:
self.logger_port = logger_port
logger_name = '%s(%d):%s' % (self.__class__.__name__, self.seed,
":" + self.dataset_name)
self.logger = get_named_client_logger(name=logger_name,
port=self.logger_port)
self.logger.info("initialised {}".format(self.__class__.__name__))
self.initial_configurations: Optional[List[Configuration]] = None
if portfolio_selection is not None:
self.initial_configurations = read_return_initial_configurations(
config_space=config_space,
portfolio_selection=portfolio_selection)