class AbstractTAFunc(SerialRunner):
"""Baseclass to execute target algorithms which are python functions.
**Note:*** Do not use directly
Parameters
----------
ta : callable
Function (target algorithm) to be optimized.
stats: Stats()
stats object to collect statistics about runtime and so on
multi_objectives: List[str]
names of the objectives, by default it is a single objective parameter "cost"
run_obj: str
run objective of SMAC
memory_limit : int, optional
Memory limit (in MB) that will be applied to the target algorithm.
par_factor: int
penalization factor
cost_for_crash : float
cost that is used in case of crashed runs (including runs
that returned NaN or inf)
use_pynisher: bool
use pynisher to limit resources;
if disabled
* TA func can use as many resources
as it wants (time and memory) --- use with caution
* all runs will be returned as SUCCESS if returned value is not None
Attributes
----------
memory_limit
use_pynisher
"""
def __init__(
self,
ta: Callable,
stats: Stats,
multi_objectives: List[str] = ["cost"],
run_obj: str = "quality",
memory_limit: Optional[int] = None,
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = False,
use_pynisher: bool = True,
):
super().__init__(
ta=ta,
stats=stats,
multi_objectives=multi_objectives,
run_obj=run_obj,
par_factor=par_factor,
cost_for_crash=cost_for_crash,
abort_on_first_run_crash=abort_on_first_run_crash,
)
self.ta = ta
self.stats = stats
self.multi_objectives = multi_objectives
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
signature = inspect.signature(ta).parameters
self._accepts_seed = "seed" in signature.keys()
self._accepts_instance = "instance" in signature.keys()
self._accepts_budget = "budget" in signature.keys()
if not callable(ta):
raise TypeError("Argument `ta` must be a callable, but is %s" % type(ta))
self._ta = cast(Callable, ta)
if memory_limit is not None:
memory_limit = int(math.ceil(memory_limit))
self.memory_limit = memory_limit
self.use_pynisher = use_pynisher
self.logger = PickableLoggerAdapter(
self.__module__ + "." + self.__class__.__name__
)
def run(
self,
config: Configuration,
instance: Optional[str] = None,
cutoff: Optional[float] = None,
seed: int = 12345,
budget: Optional[float] = None,
instance_specific: str = "0",
) -> Tuple[StatusType, float, float, Dict]:
"""Runs target algorithm <self._ta> with configuration <config> for at
most <cutoff> seconds, allowing it to use at most <memory_limit> RAM.
Whether the target algorithm is called with the <instance> and
<seed> depends on the subclass implementing the actual call to
the target algorithm.
Parameters
----------
config : Configuration, dictionary (or similar)
Dictionary param -> value
instance : str, optional
Problem instance
cutoff : float, optional
Wallclock time limit of the target algorithm. If no value is
provided no limit will be enforced. It is casted to integer internally.
seed : int
Random seed
budget : float, optional
A positive, real-valued number representing an arbitrary limit to the target algorithm
Handled by the target algorithm internally
instance_specific: str
Instance specific information (e.g., domain file or solution)
Returns
-------
status: enum of StatusType (int)
{SUCCESS, TIMEOUT, CRASHED, ABORT}
cost: np.ndarray
cost/regret/quality/runtime (float) (None, if not returned by TA)
runtime: float
runtime (None if not returned by TA)
additional_info: dict
all further additional run information
"""
obj_kwargs = {} # type: Dict[str, Union[int, str, float, None]]
if self._accepts_seed:
obj_kwargs["seed"] = seed
if self._accepts_instance:
obj_kwargs["instance"] = instance
if self._accepts_budget:
obj_kwargs["budget"] = budget
cost = self.cost_for_crash # type: Union[float, List[float]]
if self.use_pynisher:
# walltime for pynisher has to be a rounded up integer
if cutoff is not None:
cutoff = int(math.ceil(cutoff))
if cutoff > MAX_CUTOFF:
raise ValueError(
"%d is outside the legal range of [0, 65535] "
"for cutoff (when using pynisher, due to OS limitations)"
% cutoff
)
arguments = {
"logger": self.logger,
"wall_time_in_s": cutoff,
"mem_in_mb": self.memory_limit,
}
# call ta
try:
obj = pynisher.enforce_limits(**arguments)(self._ta)
rval = self._call_ta(obj, config, obj_kwargs)
except Exception as e:
cost = np.asarray(cost).squeeze().tolist()
exception_traceback = traceback.format_exc()
error_message = repr(e)
additional_info = {
"traceback": exception_traceback,
"error": error_message,
}
return StatusType.CRASHED, cost, 0.0, additional_info # type: ignore
if isinstance(rval, tuple):
result = rval[0]
additional_run_info = rval[1]
else:
result = rval
additional_run_info = {}
# get status, cost, time
if obj.exit_status is pynisher.TimeoutException:
status = StatusType.TIMEOUT
elif obj.exit_status is pynisher.MemorylimitException:
status = StatusType.MEMOUT
elif obj.exit_status == 0 and result is not None:
status = StatusType.SUCCESS
cost = result # type: ignore # noqa
else:
status = StatusType.CRASHED
runtime = float(obj.wall_clock_time)
else:
start_time = time.time()
# call ta
try:
rval = self._call_ta(self._ta, config, obj_kwargs)
if isinstance(rval, tuple):
result = rval[0]
additional_run_info = rval[1]
else:
result = rval
additional_run_info = {}
status = StatusType.SUCCESS
cost = result # type: ignore
except Exception as e:
self.logger.exception(e)
status = StatusType.CRASHED
additional_run_info = {}
runtime = time.time() - start_time
# Do some sanity checking (for multi objective)
if len(self.multi_objectives) > 1:
error = f"Returned costs {cost} does not match the number of objectives {len(self.multi_objectives)}."
# If dict convert to array
# Make sure the ordering is correct
if isinstance(cost, dict):
ordered_cost = []
for name in self.multi_objectives:
if name not in cost:
raise RuntimeError(
f"Objective {name} was not found in the returned costs."
)
ordered_cost.append(cost[name])
cost = ordered_cost
if isinstance(cost, list):
if len(cost) != len(self.multi_objectives):
raise RuntimeError(error)
if isinstance(cost, float):
raise RuntimeError(error)
if cost is None or status == StatusType.CRASHED:
status = StatusType.CRASHED
cost = self.cost_for_crash
cost = np.asarray(cost).squeeze().tolist()
return status, cost, runtime, additional_run_info # type: ignore
def _call_ta(
self,
obj: Callable,
config: Configuration,
obj_kwargs: Dict[str, Union[int, str, float, None]],
) -> Union[float, Tuple[float, Dict]]:
raise NotImplementedError()
class AbstractTAFunc(SerialRunner):
"""Baseclass to execute target algorithms which are python functions.
**Note:*** Do not use directly
Attributes
----------
memory_limit
use_pynisher
"""
def __init__(
self,
ta: typing.Callable,
stats: Stats,
run_obj: str = "quality",
memory_limit: typing.Optional[int] = None,
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = False,
use_pynisher: bool = True,
):
super().__init__(
ta=ta,
stats=stats,
run_obj=run_obj,
par_factor=par_factor,
cost_for_crash=cost_for_crash,
abort_on_first_run_crash=abort_on_first_run_crash,
)
"""
Abstract class for having a function as target algorithm
Parameters
----------
ta : callable
Function (target algorithm) to be optimized.
stats: Stats()
stats object to collect statistics about runtime and so on
run_obj: str
run objective of SMAC
memory_limit : int, optional
Memory limit (in MB) that will be applied to the target algorithm.
par_factor: int
penalization factor
cost_for_crash : float
cost that is used in case of crashed runs (including runs
that returned NaN or inf)
use_pynisher: bool
use pynisher to limit resources;
if disabled
* TA func can use as many resources
as it wants (time and memory) --- use with caution
* all runs will be returned as SUCCESS if returned value is not None
"""
self.ta = ta
self.stats = stats
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
signature = inspect.signature(ta).parameters
self._accepts_seed = 'seed' in signature.keys()
self._accepts_instance = 'instance' in signature.keys()
self._accepts_budget = 'budget' in signature.keys()
if not callable(ta):
raise TypeError('Argument `ta` must be a callable, but is %s' %
type(ta))
self._ta = typing.cast(typing.Callable, ta)
if memory_limit is not None:
memory_limit = int(math.ceil(memory_limit))
self.memory_limit = memory_limit
self.use_pynisher = use_pynisher
self.logger = PickableLoggerAdapter(self.__module__ + '.' +
self.__class__.__name__)
def run(
self,
config: Configuration,
instance: typing.Optional[str] = None,
cutoff: typing.Optional[float] = None,
seed: int = 12345,
budget: typing.Optional[float] = None,
instance_specific: str = "0"
) -> typing.Tuple[StatusType, float, float, typing.Dict]:
"""Runs target algorithm <self._ta> with configuration <config> for at
most <cutoff> seconds, allowing it to use at most <memory_limit> RAM.
Whether the target algorithm is called with the <instance> and
<seed> depends on the subclass implementing the actual call to
the target algorithm.
Parameters
----------
config : Configuration, dictionary (or similar)
Dictionary param -> value
instance : str, optional
Problem instance
cutoff : float, optional
Wallclock time limit of the target algorithm. If no value is
provided no limit will be enforced. It is casted to integer internally.
seed : int
Random seed
budget : float, optional
A positive, real-valued number representing an arbitrary limit to the target algorithm
Handled by the target algorithm internally
instance_specific: str
Instance specific information (e.g., domain file or solution)
Returns
-------
status: enum of StatusType (int)
{SUCCESS, TIMEOUT, CRASHED, ABORT}
cost: float
cost/regret/quality/runtime (float) (None, if not returned by TA)
runtime: float
runtime (None if not returned by TA)
additional_info: dict
all further additional run information
"""
obj_kwargs = {
} # type: typing.Dict[str, typing.Union[int, str, float, None]]
if self._accepts_seed:
obj_kwargs['seed'] = seed
if self._accepts_instance:
obj_kwargs['instance'] = instance
if self._accepts_budget:
obj_kwargs['budget'] = budget
if self.use_pynisher:
# walltime for pynisher has to be a rounded up integer
if cutoff is not None:
cutoff = int(math.ceil(cutoff))
if cutoff > MAX_CUTOFF:
raise ValueError(
"%d is outside the legal range of [0, 65535] "
"for cutoff (when using pynisher, due to OS limitations)"
% cutoff)
arguments = {
'logger': self.logger,
'wall_time_in_s': cutoff,
'mem_in_mb': self.memory_limit
}
# call ta
try:
obj = pynisher.enforce_limits(**arguments)(self._ta)
rval = self._call_ta(obj, config, obj_kwargs)
except Exception as e:
exception_traceback = traceback.format_exc()
error_message = repr(e)
additional_info = {
'traceback': exception_traceback,
'error': error_message
}
return StatusType.CRASHED, self.cost_for_crash, 0.0, additional_info
if isinstance(rval, tuple):
result = rval[0]
additional_run_info = rval[1]
else:
result = rval
additional_run_info = {}
# get status, cost, time
if obj.exit_status is pynisher.TimeoutException:
status = StatusType.TIMEOUT
cost = self.cost_for_crash
elif obj.exit_status is pynisher.MemorylimitException:
status = StatusType.MEMOUT
cost = self.cost_for_crash
elif obj.exit_status == 0 and result is not None:
status = StatusType.SUCCESS
cost = result
else:
status = StatusType.CRASHED
cost = self.cost_for_crash
runtime = float(obj.wall_clock_time)
else:
start_time = time.time()
# call ta
try:
rval = self._call_ta(self._ta, config, obj_kwargs)
if isinstance(rval, tuple):
result = rval[0]
additional_run_info = rval[1]
else:
result = rval
additional_run_info = {}
status = StatusType.SUCCESS
cost = result
except Exception as e:
self.logger.exception(e)
cost, result = self.cost_for_crash, self.cost_for_crash
status = StatusType.CRASHED
additional_run_info = {}
runtime = time.time() - start_time
if status == StatusType.SUCCESS and not isinstance(
result, (int, float)):
status = StatusType.CRASHED
cost = self.cost_for_crash
return status, cost, runtime, additional_run_info
def _call_ta(
self,
obj: typing.Callable,
config: Configuration,
obj_kwargs: typing.Dict[str, typing.Union[int, str, float, None]],
) -> typing.Union[float, typing.Tuple[float, typing.Dict]]:
raise NotImplementedError()