def _calculate_metafeatures_with_limits(self, time_limit):
res = None
time_limit = max(time_limit, 1)
try:
context = multiprocessing.get_context(self.pynisher_context)
preload_modules(context)
safe_mf = pynisher.enforce_limits(mem_in_mb=self.memory_limit,
wall_time_in_s=int(time_limit),
grace_period_in_s=30,
context=context,
logger=self.logger)(
self._calculate_metafeatures)
res = safe_mf()
except Exception as e:
self.logger.error('Error getting metafeatures: %s', str(e))
return res
def _calculate_metafeatures_with_limits(self, time_limit):
res = None
time_limit = max(time_limit, 1)
try:
context = multiprocessing.get_context(self.pynisher_context)
preload_modules(context)
safe_mf = pynisher.enforce_limits(
mem_in_mb=self.memory_limit,
wall_time_in_s=int(time_limit),
grace_period_in_s=30,
context=context,
logger=self.logger)(_calculate_metafeatures)
res = safe_mf(data_feat_type=self.datamanager.feat_type,
data_info_task=self.datamanager.info['task'],
x_train=self.datamanager.data['X_train'],
y_train=self.datamanager.data['Y_train'],
basename=self.dataset_name,
watcher=self.watcher,
logger_=self.logger)
except Exception as e:
self.logger.error('Error getting metafeatures: %s', str(e))
return res
def run(
self,
config: Configuration,
instance: Optional[str] = None,
cutoff: Optional[float] = None,
seed: int = 12345,
budget: float = 0.0,
instance_specific: Optional[str] = None,
) -> Tuple[StatusType, float, float, Dict[str, Union[int, float, str, Dict, List, Tuple]]]:
# Additional information of each of the tae executions
# Defined upfront for mypy
additional_run_info: TYPE_ADDITIONAL_INFO = {}
context = multiprocessing.get_context(self.pynisher_context)
preload_modules(context)
queue = context.Queue()
if not (instance_specific is None or instance_specific == '0'):
raise ValueError(instance_specific)
init_params = {'instance': instance}
if self.init_params is not None:
init_params.update(self.init_params)
if self.port is None:
logger: Union[logging.Logger, PickableLoggerAdapter] = logging.getLogger("pynisher")
else:
logger = get_named_client_logger(
name="pynisher",
port=self.port,
)
arguments = dict(
logger=logger,
wall_time_in_s=cutoff,
mem_in_mb=self.memory_limit,
capture_output=True,
context=context,
)
if isinstance(config, int):
num_run = self.initial_num_run
else:
num_run = config.config_id + self.initial_num_run
obj_kwargs = dict(
queue=queue,
config=config,
backend=self.backend,
port=self.port,
metric=self.metric,
seed=self.autosklearn_seed,
num_run=num_run,
scoring_functions=self.scoring_functions,
output_y_hat_optimization=self.output_y_hat_optimization,
include=self.include,
exclude=self.exclude,
disable_file_output=self.disable_file_output,
instance=instance,
init_params=init_params,
budget=budget,
budget_type=self.budget_type,
additional_components=autosklearn.pipeline.components.base._addons,
)
if self.resampling_strategy != 'test':
obj_kwargs['resampling_strategy'] = self.resampling_strategy
obj_kwargs['resampling_strategy_args'] = self.resampling_strategy_args
try:
obj = pynisher.enforce_limits(**arguments)(self.ta)
obj(**obj_kwargs)
except Exception as e:
exception_traceback = traceback.format_exc()
error_message = repr(e)
additional_run_info.update({
'traceback': exception_traceback,
'error': error_message
})
return StatusType.CRASHED, self.worst_possible_result, 0.0, additional_run_info
if obj.exit_status in (pynisher.TimeoutException, pynisher.MemorylimitException):
# Even if the pynisher thinks that a timeout or memout occured,
# it can be that the target algorithm wrote something into the queue
# - then we treat it as a succesful run
try:
info = autosklearn.evaluation.util.read_queue(queue)
result = info[-1]['loss']
status = info[-1]['status']
additional_run_info = info[-1]['additional_run_info']
if obj.stdout:
additional_run_info['subprocess_stdout'] = obj.stdout
if obj.stderr:
additional_run_info['subprocess_stderr'] = obj.stderr
if obj.exit_status is pynisher.TimeoutException:
additional_run_info['info'] = 'Run stopped because of timeout.'
elif obj.exit_status is pynisher.MemorylimitException:
additional_run_info['info'] = 'Run stopped because of memout.'
if status in [StatusType.SUCCESS, StatusType.DONOTADVANCE]:
cost = result
else:
cost = self.worst_possible_result
except Empty:
info = None
if obj.exit_status is pynisher.TimeoutException:
status = StatusType.TIMEOUT
additional_run_info = {'error': 'Timeout'}
elif obj.exit_status is pynisher.MemorylimitException:
status = StatusType.MEMOUT
additional_run_info = {
"error": "Memout (used more than {} MB).".format(self.memory_limit)
}
else:
raise ValueError(obj.exit_status)
cost = self.worst_possible_result
elif obj.exit_status is TAEAbortException:
info = None
status = StatusType.ABORT
cost = self.worst_possible_result
additional_run_info = {'error': 'Your configuration of '
'auto-sklearn does not work!',
'exit_status': _encode_exit_status(obj.exit_status),
'subprocess_stdout': obj.stdout,
'subprocess_stderr': obj.stderr,
}
else:
try:
info = autosklearn.evaluation.util.read_queue(queue)
result = info[-1]['loss']
status = info[-1]['status']
additional_run_info = info[-1]['additional_run_info']
if obj.exit_status == 0:
cost = result
else:
status = StatusType.CRASHED
cost = self.worst_possible_result
additional_run_info['info'] = 'Run treated as crashed ' \
'because the pynisher exit ' \
'status %s is unknown.' % \
str(obj.exit_status)
additional_run_info['exit_status'] = _encode_exit_status(obj.exit_status)
additional_run_info['subprocess_stdout'] = obj.stdout
additional_run_info['subprocess_stderr'] = obj.stderr
except Empty:
info = None
additional_run_info = {
'error': 'Result queue is empty',
'exit_status': _encode_exit_status(obj.exit_status),
'subprocess_stdout': obj.stdout,
'subprocess_stderr': obj.stderr,
'exitcode': obj.exitcode
}
status = StatusType.CRASHED
cost = self.worst_possible_result
if (
(self.budget_type is None or budget == 0)
and status == StatusType.DONOTADVANCE
):
status = StatusType.SUCCESS
if not isinstance(additional_run_info, dict):
additional_run_info = {'message': additional_run_info}
if (
info is not None
and self.resampling_strategy in ('holdout-iterative-fit', 'cv-iterative-fit')
and status != StatusType.CRASHED
):
learning_curve = autosklearn.evaluation.util.extract_learning_curve(info)
learning_curve_runtime = autosklearn.evaluation.util.extract_learning_curve(
info, 'duration'
)
if len(learning_curve) > 1:
additional_run_info['learning_curve'] = learning_curve
additional_run_info['learning_curve_runtime'] = learning_curve_runtime
train_learning_curve = autosklearn.evaluation.util.extract_learning_curve(
info, 'train_loss'
)
if len(train_learning_curve) > 1:
additional_run_info['train_learning_curve'] = train_learning_curve
additional_run_info['learning_curve_runtime'] = learning_curve_runtime
if self._get_validation_loss:
validation_learning_curve = autosklearn.evaluation.util.extract_learning_curve(
info, 'validation_loss',
)
if len(validation_learning_curve) > 1:
additional_run_info['validation_learning_curve'] = \
validation_learning_curve
additional_run_info[
'learning_curve_runtime'] = learning_curve_runtime
if self._get_test_loss:
test_learning_curve = autosklearn.evaluation.util.extract_learning_curve(
info, 'test_loss',
)
if len(test_learning_curve) > 1:
additional_run_info['test_learning_curve'] = test_learning_curve
additional_run_info[
'learning_curve_runtime'] = learning_curve_runtime
if isinstance(config, int):
origin = 'DUMMY'
config_id = config
else:
origin = getattr(config, 'origin', 'UNKNOWN')
config_id = config.config_id
additional_run_info['configuration_origin'] = origin
runtime = float(obj.wall_clock_time)
autosklearn.evaluation.util.empty_queue(queue)
self.logger.info("Finished evaluating configuration %d" % config_id)
return status, cost, runtime, additional_run_info