def __init__(self,
configspace: ConfigurationSpace,
types: np.ndarray,
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
hidden_dims: typing.List[int] = [50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 10000,
batch_size: int = 8,
var: bool = True,
**kwargs):
super().__init__(configspace, types, bounds, seed, **kwargs)
print("USE DNGO")
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 100
self.log_error = 1000
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.nn = None
self.blr = None
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(
self,
ta: typing.Union[typing.List[str], typing.Callable],
stats: Stats,
run_obj: str = "runtime",
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = True,
):
# The results is a FIFO structure, implemented via a list
# (because the Queue lock is not pickable). Finished runs are
# put in this list and collected via process_finished_runs
self.results = [] # type: typing.List[typing.Tuple[RunInfo, RunValue]]
# Below state the support for a Runner algorithm that
# implements a ta
self.ta = ta
self.stats = stats
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
self.logger = PickableLoggerAdapter(self.__module__ + '.' +
self.__class__.__name__)
self._supports_memory_limit = False
super().__init__()
def __init__(
self,
ta: typing.Union[typing.List[str], typing.Callable],
stats: Stats,
run_obj: str = "runtime",
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = True,
):
"""
Attributes
----------
results
ta
stats
run_obj
par_factor
cost_for_crash
abort_first_run_crash
Parameters
----------
ta : typing.Union[typing.List[str], typing.Callable]
target algorithm
stats: Stats
stats object to collect statistics about runtime/additional info
run_obj: str
run objective of SMAC
par_factor: int
penalization factor
cost_for_crash : float
cost that is used in case of crashed runs (including runs
that returned NaN or inf)
abort_on_first_run_crash: bool
if true and first run crashes, raise FirstRunCrashedException
"""
# The results is a FIFO structure, implemented via a list
# (because the Queue lock is not pickable). Finished runs are
# put in this list and collected via process_finished_runs
self.results = [] # type: typing.List[typing.Tuple[RunInfo, RunValue]]
# Below state the support for a Runner algorithm that
# implements a ta
self.ta = ta
self.stats = stats
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
self.logger = PickableLoggerAdapter(self.__module__ + '.' +
self.__class__.__name__)
self._supports_memory_limit = False
super().__init__()
def __init__(self,
hidden_dims,
input_size,
feat_type=None,
var: bool = True,
max_cat: int = np.inf):
super(NeuralNet, self).__init__()
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
self.feat_type = feat_type
self.input_size = input_size
self.num_neurons = hidden_dims
self.activation = nn.Tanh
self.num_layer = len(hidden_dims)
self.max_cat = max_cat
if var:
self.n_output = 2
else:
self.n_output = 1
if np.sum(self.feat_type) == 0:
self.feat_type = None
if self.feat_type is not None:
self.logger.info("Use cat embedding")
assert len(self.feat_type) == self.input_size
emb = nn.ModuleList()
sz = int(0)
for f in self.feat_type:
if f == 0:
# In SMAC 0 encodes a numerical
emb.append(None)
sz += 1
else:
es = min(self.max_cat, int(f))
emb.append(nn.Embedding(int(f), es))
sz += es
assert int(sz) == sz
sz = int(sz)
num_neurons = [sz] + self.num_neurons
self.embedding = emb
else:
num_neurons = [self.input_size] + self.num_neurons
self.weights = nn.ModuleList()
self.acts = nn.ModuleList()
print(num_neurons)
for i in range(self.num_layer):
self.weights.append(nn.Linear(num_neurons[i], num_neurons[i + 1]))
self.acts.append(self.activation())
self.outlayer = nn.Linear(num_neurons[-1], self.n_output)
def __init__(
self,
ta: Callable,
stats: Stats,
multi_objectives: List[str] = ["cost"],
run_obj: str = "quality",
memory_limit: Optional[int] = None,
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = False,
use_pynisher: bool = True,
):
super().__init__(
ta=ta,
stats=stats,
multi_objectives=multi_objectives,
run_obj=run_obj,
par_factor=par_factor,
cost_for_crash=cost_for_crash,
abort_on_first_run_crash=abort_on_first_run_crash,
)
self.ta = ta
self.stats = stats
self.multi_objectives = multi_objectives
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
signature = inspect.signature(ta).parameters
self._accepts_seed = "seed" in signature.keys()
self._accepts_instance = "instance" in signature.keys()
self._accepts_budget = "budget" in signature.keys()
if not callable(ta):
raise TypeError("Argument `ta` must be a callable, but is %s" % type(ta))
self._ta = cast(Callable, ta)
if memory_limit is not None:
memory_limit = int(math.ceil(memory_limit))
self.memory_limit = memory_limit
self.use_pynisher = use_pynisher
self.logger = PickableLoggerAdapter(
self.__module__ + "." + self.__class__.__name__
)
def __init__(
self,
configspace: ConfigurationSpace,
types: typing.List[int],
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
instance_features: typing.Optional[np.ndarray] = None,
pca_components: typing.Optional[int] = 7,
) -> None:
self.configspace = configspace
self.seed = seed
self.instance_features = instance_features
self.pca_components = pca_components
if instance_features is not None:
self.n_feats = instance_features.shape[1]
else:
self.n_feats = 0
self.n_params = len(self.configspace.get_hyperparameters())
self.pca = PCA(n_components=self.pca_components)
self.scaler = MinMaxScaler()
self._apply_pca = False
# Never use a lower variance than this
self.var_threshold = VERY_SMALL_NUMBER
self.bounds = bounds
self.types = types
# Initial types array which is used to reset the type array at every call to train()
self._initial_types = copy.deepcopy(types)
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(
self,
configspace: ConfigurationSpace,
types: typing.List[int],
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
instance_features: typing.Optional[np.ndarray] = None,
pca_components: typing.Optional[int] = 7,
) -> None:
"""Constructor
Parameters
----------
configspace : ConfigurationSpace
Configuration space to tune for.
types : List[int]
Specifies the number of categorical values of an input dimension where
the i-th entry corresponds to the i-th input dimension. Let's say we
have 2 dimension where the first dimension consists of 3 different
categorical choices and the second dimension is continuous than we
have to pass [3, 0]. Note that we count starting from 0.
bounds : List[Tuple[float, float]]
bounds of input dimensions: (lower, uppper) for continuous dims; (n_cat, np.nan) for categorical dims
seed : int
The seed that is passed to the model library.
instance_features : np.ndarray (I, K)
Contains the K dimensional instance features
of the I different instances
pca_components : float
Number of components to keep when using PCA to reduce
dimensionality of instance features. Requires to
set n_feats (> pca_dims).
"""
self.configspace = configspace
self.seed = seed
self.instance_features = instance_features
self.pca_components = pca_components
if instance_features is not None:
self.n_feats = instance_features.shape[1]
else:
self.n_feats = 0
self.n_params = len(self.configspace.get_hyperparameters())
self.pca = PCA(n_components=self.pca_components)
self.scaler = MinMaxScaler()
self._apply_pca = False
# Never use a lower variance than this
self.var_threshold = VERY_SMALL_NUMBER
self.bounds = bounds
self.types = types
# Initial types array which is used to reset the type array at every call to train()
self._initial_types = copy.deepcopy(types)
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(
self,
aggregate_func: typing.Callable,
overwrite_existing_runs: bool = False,
file_system=LocalFS()) -> None:
"""Constructor
Parameters
----------
aggregate_func: callable
function to aggregate perf across instances
overwrite_existing_runs: bool
allows to overwrites old results if pairs of
algorithm-instance-seed were measured
multiple times
"""
self.file_system = file_system
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
# By having the data in a deterministic order we can do useful tests
# when we serialize the data and can assume it's still in the same
# order as it was added.
self.data = collections.OrderedDict(
) # type: typing.Dict[RunKey, RunValue]
# for fast access, we have also an unordered data structure
# to get all instance seed pairs of a configuration
self._configid_to_inst_seed = {} # type: typing.Dict[int, InstSeedKey]
self.config_ids = {} # type: typing.Dict[Configuration, int]
self.ids_config = {} # type: typing.Dict[int, Configuration]
self._n_id = 0
# Stores cost for each configuration ID
self.cost_per_config = {} # type: typing.Dict[int, float]
# runs_per_config maps the configuration ID to the number of runs for that configuration
# and is necessary for computing the moving average
self.runs_per_config = {} # type: typing.Dict[int, int]
# Store whether a datapoint is "external", which means it was read from
# a JSON file. Can be chosen to not be written to disk
self.external = {} # type: typing.Dict[RunKey, DataOrigin]
self.aggregate_func = aggregate_func
self.overwrite_existing_runs = overwrite_existing_runs
def __init__(self, model: AbstractEPM):
"""Constructor
Parameters
----------
model : AbstractEPM
Models the objective function.
"""
self.model = model
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(self, model: AbstractEPM):
"""Constructor
Parameters
----------
model : AbstractEPM
Models the objective function.
"""
self.model = model
self._required_updates = ('model', ) # type: Tuple[str, ...]
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(self,
configspace: ConfigurationSpace,
types: np.ndarray,
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
hidden_dims: typing.List[int] = [50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 10000,
batch_size: int = 8,
number_of_networks: int = 10,
var: bool = True,
train_with_lognormal_llh=False,
compute_mean_in_logspace=True,
**kwargs):
super().__init__(configspace, types, bounds, seed, **kwargs)
assert not (train_with_lognormal_llh and compute_mean_in_logspace)
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 1000
self.log_error = 5000
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.number_of_networks = number_of_networks
self.train_with_lognormal = train_with_lognormal_llh
self.compute_mean_in_logspace = compute_mean_in_logspace
self.nns = None
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(
self,
overwrite_existing_runs: bool = False,
) -> None:
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
# By having the data in a deterministic order we can do useful tests
# when we serialize the data and can assume it's still in the same
# order as it was added.
self.data = collections.OrderedDict() # type: Dict[RunKey, RunValue]
# for fast access, we have also an unordered data structure
# to get all instance seed pairs of a configuration.
# This does not include capped runs.
self._configid_to_inst_seed_budget = (
{}) # type: Dict[int, Dict[InstSeedKey, List[float]]]
self.config_ids = {} # type: Dict[Configuration, int]
self.ids_config = {} # type: Dict[int, Configuration]
self._n_id = 0
# Stores cost for each configuration ID
self._cost_per_config = {} # type: Dict[int, np.ndarray]
# Stores min cost across all budgets for each configuration ID
self._min_cost_per_config = {} # type: Dict[int, np.ndarray]
# runs_per_config maps the configuration ID to the number of runs for that configuration
# and is necessary for computing the moving average
self.num_runs_per_config = {} # type: Dict[int, int]
# Store whether a datapoint is "external", which means it was read from
# a JSON file. Can be chosen to not be written to disk
self.external = {} # type: Dict[RunKey, DataOrigin]
self.overwrite_existing_runs = overwrite_existing_runs
self.num_obj = -1 # type: int
self.objective_bounds = [] # type: List[Tuple[float, float]]
def __init__(self,
seed: int,
feat_types: typing.List[int] = None,
hidden_dims: typing.List[int] = [50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 10000,
batch_size: int = 8,
var: bool = True,
lognormal_nllh: bool = False,
var_bias_init: float = 1,
max_cat: int = np.inf,
learned_weight_init: bool = False,
optimization_algorithm: str = 'sgd',
**kwargs):
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 1000
self.log_error = iterations
self.seed = seed
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.lognormal_nllh = lognormal_nllh
self.var_bias_init = var_bias_init
self.max_cat = max_cat
self.learned_weight_init = learned_weight_init
self.optimization_algorithm = optimization_algorithm
self.feat_types = feat_types
if self.lognormal_nllh:
assert self.var, "Can't train with lognormal nllh if no var is selected"
self.model = None
class BaseRunner(ABC):
"""Interface class to handle the execution of SMAC' configurations.
This interface defines how to interact with the SMBO loop.
The complexity of running a configuration as well as handling the
results is abstracted to the SMBO via a BaseRunner.
From SMBO perspective, launching a configuration follows a
submit/collect scheme as follows:
1- A run is launched via submit_run()
1.1- Submit_run internally calls run_wrapper(), a method that
contains common processing functions among different runners,
for example, handling capping and stats checking.
1.2- A class that implements BaseRunner defines run() which is
really the algorithm to translate a RunInfo to a RunValue, i.e.
a configuration to an actual result.
2- A completed run is collected via get_finished_runs(), which returns
any finished runs, if any.
3- This interface also offers the method wait() as a mechanism to make
sure we have enough data in the next iteration to make a decision. For
example, the intensifier might not be able to select the next challenger
until more results are available.
Attributes
----------
results
ta
stats
run_obj
par_factor
cost_for_crash
abort_first_run_crash
Parameters
---------
ta : typing.Union[typing.List[str], typing.Callable]
target algorithm
stats: Stats
stats object to collect statistics about runtime/additional info
run_obj: str
run objective of SMAC
par_factor: int
penalization factor
cost_for_crash : float
cost that is used in case of crashed runs (including runs
that returned NaN or inf)
abort_on_first_run_crash: bool
if true and first run crashes, raise FirstRunCrashedException
"""
def __init__(
self,
ta: typing.Union[typing.List[str], typing.Callable],
stats: Stats,
run_obj: str = "runtime",
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = True,
):
# The results is a FIFO structure, implemented via a list
# (because the Queue lock is not pickable). Finished runs are
# put in this list and collected via process_finished_runs
self.results = [] # type: typing.List[typing.Tuple[RunInfo, RunValue]]
# Below state the support for a Runner algorithm that
# implements a ta
self.ta = ta
self.stats = stats
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
self.logger = PickableLoggerAdapter(self.__module__ + '.' +
self.__class__.__name__)
self._supports_memory_limit = False
super().__init__()
@abstractmethod
def submit_run(self, run_info: RunInfo) -> None:
"""This function submits a configuration
embedded in a RunInfo object, and uses one of the workers
to produce a result (such result will eventually be available
on the self.results FIFO).
This interface method will be called by SMBO, with the expectation
that a function will be executed by a worker.
What will be executed is dictated by run_info, and "how" will it be
executed is decided via the child class that implements a run() method.
Because config submission can be a serial/parallel endeavor,
it is expected to be implemented by a child class.
Parameters
----------
run_info: RunInfo
An object containing the configuration and the necessary data to run it
"""
pass
@abstractmethod
def run(
self,
config: Configuration,
instance: str,
cutoff: typing.Optional[float] = None,
seed: int = 12345,
budget: typing.Optional[float] = None,
instance_specific: str = "0",
) -> typing.Tuple[StatusType, float, float, typing.Dict]:
"""Runs target algorithm <self.ta> with configuration <config> on
instance <instance> with instance specifics <specifics> for at most
<cutoff> seconds and random seed <seed>
This method exemplifies how to defined the run() method
Parameters
----------
config : Configuration
dictionary param -> value
instance : string
problem instance
cutoff : float, optional
Wallclock time limit of the target algorithm. If no value is
provided no limit will be enforced.
seed : int
random seed
budget : float, optional
A positive, real-valued number representing an arbitrary limit to the target
algorithm. Handled by the target algorithm internally
instance_specific: str
instance specific information (e.g., domain file or solution)
Returns
-------
status: enum of StatusType (int)
{SUCCESS, TIMEOUT, CRASHED, ABORT}
cost: float
cost/regret/quality (float) (None, if not returned by TA)
runtime: float
runtime (None if not returned by TA)
additional_info: dict
all further additional run information
"""
pass
def run_wrapper(
self,
run_info: RunInfo,
) -> typing.Tuple[RunInfo, RunValue]:
"""Wrapper around run() to exec and check the execution of a given config file
This function encapsulates common handling/processing, so that run() implementation
is simplified.
Parameters
----------
run_info : RunInfo
Object that contains enough information to execute a configuration run in
isolation.
Returns
-------
RunInfo:
an object containing the configuration launched
RunValue:
Contains information about the status/performance of config
"""
start = time.time()
if run_info.cutoff is None and self.run_obj == "runtime":
if self.logger:
self.logger.critical(
"For scenarios optimizing running time "
"(run objective), a cutoff time is required, "
"but not given to this call.")
raise ValueError("For scenarios optimizing running time "
"(run objective), a cutoff time is required, "
"but not given to this call.")
cutoff = None
if run_info.cutoff is not None:
cutoff = int(math.ceil(run_info.cutoff))
try:
status, cost, runtime, additional_info = self.run(
config=run_info.config,
instance=run_info.instance,
cutoff=cutoff,
seed=run_info.seed,
budget=run_info.budget,
instance_specific=run_info.instance_specific)
except Exception as e:
status = StatusType.CRASHED
cost = self.cost_for_crash
runtime = time.time() - start
# Add context information to the error message
exception_traceback = traceback.format_exc()
error_message = repr(e)
additional_info = {
'traceback': exception_traceback,
'error': error_message
}
end = time.time()
if run_info.budget == 0 and status == StatusType.DONOTADVANCE:
raise ValueError(
"Cannot handle DONOTADVANCE state when using intensify or SH/HB on "
"instances.")
# Catch NaN or inf.
if (self.run_obj == 'runtime' and not np.isfinite(runtime)
or self.run_obj == 'quality' and not np.isfinite(cost)):
if self.logger:
self.logger.warning(
"Target Algorithm returned NaN or inf as {}. "
"Algorithm run is treated as CRASHED, cost "
"is set to {} for quality scenarios. "
"(Change value through \"cost_for_crash\""
"-option.)".format(self.run_obj, self.cost_for_crash))
status = StatusType.CRASHED
if self.run_obj == "runtime":
# The following line pleases mypy - we already check for cutoff not being none above,
# prior to calling run. However, mypy assumes that the data type of cutoff
# is still Optional[int]
assert cutoff is not None
if runtime > self.par_factor * cutoff:
self.logger.warning("Returned running time is larger "
"than {0} times the passed cutoff time. "
"Clamping to {0} x cutoff.".format(
self.par_factor))
runtime = cutoff * self.par_factor
status = StatusType.TIMEOUT
if status == StatusType.SUCCESS:
cost = runtime
else:
cost = cutoff * self.par_factor
if status == StatusType.TIMEOUT and run_info.capped:
status = StatusType.CAPPED
else:
if status == StatusType.CRASHED:
cost = self.cost_for_crash
return run_info, RunValue(status=status,
cost=cost,
time=runtime,
additional_info=additional_info,
starttime=start,
endtime=end)
@abstractmethod
def get_finished_runs(
self) -> typing.List[typing.Tuple[RunInfo, RunValue]]:
"""This method returns any finished configuration, and returns a list with
the results of exercising the configurations. This class keeps populating results
to self.results until a call to get_finished runs is done. In this case, the
self.results list is emptied and all RunValues produced by running run() are
returned.
Returns
-------
List[RunInfo, RunValue]: A list of pais RunInfo/RunValues
a submitted configuration
"""
raise NotImplementedError()
@abstractmethod
def wait(self) -> None:
"""SMBO/intensifier might need to wait for runs to finish before making a decision.
This method waits until 1 run completes
"""
pass
@abstractmethod
def pending_runs(self) -> bool:
"""
Whether or not there are configs still running. Generally if the runner is serial,
launching a run instantly returns it's result. On parallel runners, there might
be pending configurations to complete.
"""
pass
@abstractmethod
def num_workers(self) -> int:
"""
Return the active number of workers that will execute tae runs.
"""
pass
class AbstractTAFunc(SerialRunner):
"""Baseclass to execute target algorithms which are python functions.
**Note:*** Do not use directly
Parameters
----------
ta : callable
Function (target algorithm) to be optimized.
stats: Stats()
stats object to collect statistics about runtime and so on
multi_objectives: List[str]
names of the objectives, by default it is a single objective parameter "cost"
run_obj: str
run objective of SMAC
memory_limit : int, optional
Memory limit (in MB) that will be applied to the target algorithm.
par_factor: int
penalization factor
cost_for_crash : float
cost that is used in case of crashed runs (including runs
that returned NaN or inf)
use_pynisher: bool
use pynisher to limit resources;
if disabled
* TA func can use as many resources
as it wants (time and memory) --- use with caution
* all runs will be returned as SUCCESS if returned value is not None
Attributes
----------
memory_limit
use_pynisher
"""
def __init__(
self,
ta: Callable,
stats: Stats,
multi_objectives: List[str] = ["cost"],
run_obj: str = "quality",
memory_limit: Optional[int] = None,
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = False,
use_pynisher: bool = True,
):
super().__init__(
ta=ta,
stats=stats,
multi_objectives=multi_objectives,
run_obj=run_obj,
par_factor=par_factor,
cost_for_crash=cost_for_crash,
abort_on_first_run_crash=abort_on_first_run_crash,
)
self.ta = ta
self.stats = stats
self.multi_objectives = multi_objectives
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
signature = inspect.signature(ta).parameters
self._accepts_seed = "seed" in signature.keys()
self._accepts_instance = "instance" in signature.keys()
self._accepts_budget = "budget" in signature.keys()
if not callable(ta):
raise TypeError("Argument `ta` must be a callable, but is %s" % type(ta))
self._ta = cast(Callable, ta)
if memory_limit is not None:
memory_limit = int(math.ceil(memory_limit))
self.memory_limit = memory_limit
self.use_pynisher = use_pynisher
self.logger = PickableLoggerAdapter(
self.__module__ + "." + self.__class__.__name__
)
def run(
self,
config: Configuration,
instance: Optional[str] = None,
cutoff: Optional[float] = None,
seed: int = 12345,
budget: Optional[float] = None,
instance_specific: str = "0",
) -> Tuple[StatusType, float, float, Dict]:
"""Runs target algorithm <self._ta> with configuration <config> for at
most <cutoff> seconds, allowing it to use at most <memory_limit> RAM.
Whether the target algorithm is called with the <instance> and
<seed> depends on the subclass implementing the actual call to
the target algorithm.
Parameters
----------
config : Configuration, dictionary (or similar)
Dictionary param -> value
instance : str, optional
Problem instance
cutoff : float, optional
Wallclock time limit of the target algorithm. If no value is
provided no limit will be enforced. It is casted to integer internally.
seed : int
Random seed
budget : float, optional
A positive, real-valued number representing an arbitrary limit to the target algorithm
Handled by the target algorithm internally
instance_specific: str
Instance specific information (e.g., domain file or solution)
Returns
-------
status: enum of StatusType (int)
{SUCCESS, TIMEOUT, CRASHED, ABORT}
cost: np.ndarray
cost/regret/quality/runtime (float) (None, if not returned by TA)
runtime: float
runtime (None if not returned by TA)
additional_info: dict
all further additional run information
"""
obj_kwargs = {} # type: Dict[str, Union[int, str, float, None]]
if self._accepts_seed:
obj_kwargs["seed"] = seed
if self._accepts_instance:
obj_kwargs["instance"] = instance
if self._accepts_budget:
obj_kwargs["budget"] = budget
cost = self.cost_for_crash # type: Union[float, List[float]]
if self.use_pynisher:
# walltime for pynisher has to be a rounded up integer
if cutoff is not None:
cutoff = int(math.ceil(cutoff))
if cutoff > MAX_CUTOFF:
raise ValueError(
"%d is outside the legal range of [0, 65535] "
"for cutoff (when using pynisher, due to OS limitations)"
% cutoff
)
arguments = {
"logger": self.logger,
"wall_time_in_s": cutoff,
"mem_in_mb": self.memory_limit,
}
# call ta
try:
obj = pynisher.enforce_limits(**arguments)(self._ta)
rval = self._call_ta(obj, config, obj_kwargs)
except Exception as e:
cost = np.asarray(cost).squeeze().tolist()
exception_traceback = traceback.format_exc()
error_message = repr(e)
additional_info = {
"traceback": exception_traceback,
"error": error_message,
}
return StatusType.CRASHED, cost, 0.0, additional_info # type: ignore
if isinstance(rval, tuple):
result = rval[0]
additional_run_info = rval[1]
else:
result = rval
additional_run_info = {}
# get status, cost, time
if obj.exit_status is pynisher.TimeoutException:
status = StatusType.TIMEOUT
elif obj.exit_status is pynisher.MemorylimitException:
status = StatusType.MEMOUT
elif obj.exit_status == 0 and result is not None:
status = StatusType.SUCCESS
cost = result # type: ignore # noqa
else:
status = StatusType.CRASHED
runtime = float(obj.wall_clock_time)
else:
start_time = time.time()
# call ta
try:
rval = self._call_ta(self._ta, config, obj_kwargs)
if isinstance(rval, tuple):
result = rval[0]
additional_run_info = rval[1]
else:
result = rval
additional_run_info = {}
status = StatusType.SUCCESS
cost = result # type: ignore
except Exception as e:
self.logger.exception(e)
status = StatusType.CRASHED
additional_run_info = {}
runtime = time.time() - start_time
# Do some sanity checking (for multi objective)
if len(self.multi_objectives) > 1:
error = f"Returned costs {cost} does not match the number of objectives {len(self.multi_objectives)}."
# If dict convert to array
# Make sure the ordering is correct
if isinstance(cost, dict):
ordered_cost = []
for name in self.multi_objectives:
if name not in cost:
raise RuntimeError(
f"Objective {name} was not found in the returned costs."
)
ordered_cost.append(cost[name])
cost = ordered_cost
if isinstance(cost, list):
if len(cost) != len(self.multi_objectives):
raise RuntimeError(error)
if isinstance(cost, float):
raise RuntimeError(error)
if cost is None or status == StatusType.CRASHED:
status = StatusType.CRASHED
cost = self.cost_for_crash
cost = np.asarray(cost).squeeze().tolist()
return status, cost, runtime, additional_run_info # type: ignore
def _call_ta(
self,
obj: Callable,
config: Configuration,
obj_kwargs: Dict[str, Union[int, str, float, None]],
) -> Union[float, Tuple[float, Dict]]:
raise NotImplementedError()
def __init__(self, model: AbstractEPM):
self.model = model
self._required_updates = ('model', ) # type: Tuple[str, ...]
self.logger = PickableLoggerAdapter(self.__module__ + "." + self.__class__.__name__)
class EnsembleNN(BaseModel):
def __init__(self,
configspace: ConfigurationSpace,
types: np.ndarray,
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
hidden_dims: typing.List[int] = [50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 10000,
batch_size: int = 8,
number_of_networks: int = 10,
var: bool = True,
train_with_lognormal_llh=False,
compute_mean_in_logspace=True,
**kwargs):
super().__init__(configspace, types, bounds, seed, **kwargs)
assert not (train_with_lognormal_llh and compute_mean_in_logspace)
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 1000
self.log_error = 5000
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.number_of_networks = number_of_networks
self.train_with_lognormal = train_with_lognormal_llh
self.compute_mean_in_logspace = compute_mean_in_logspace
self.nns = None
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def _train(self, X: np.ndarray, y: np.ndarray):
self._my = np.mean(y)
self._sy = np.std(y)
if not self.train_with_lognormal:
y -= self._my
y /= self._sy
self.train_data = (X, y)
self.nns = []
self.logger.debug("Start Training %d networks" %
self.number_of_networks)
for i in range(self.number_of_networks):
nn = SimpleNetworkEmbedding(
hidden_dims=self.hidden_dims,
lr=self.lr,
seed=self.seed + i,
momentum=self.momentum,
weight_decay=self.weight_decay,
iterations=self.iterations,
batch_size=self.batch_size,
var=self.var,
lognormal_nllh=self.train_with_lognormal)
nn.train(X, y)
self.nns.append(nn)
def _predict_individual(
self, X: np.ndarray) -> typing.Tuple[np.ndarray, np.ndarray]:
ms = np.zeros([X.shape[0], self.number_of_networks])
vs = np.zeros([X.shape[0], self.number_of_networks])
for i_nn, nn in enumerate(self.nns):
pred = nn.predict(X)
m = pred[:, 0]
v = pred[:, 1]
if not self.train_with_lognormal:
m = m * self._sy + self._my
v = v * self._sy**2
ms[:, i_nn] = m
vs[:, i_nn] = v
return ms, vs
def _predict(self, X: np.ndarray) -> typing.Tuple[np.ndarray, np.ndarray]:
ms, _ = self._predict_individual(X)
m = ms.mean(axis=1)
v = ms.var(axis=1)
return m, v
def predict_marginalized_over_instances(self, X: np.ndarray):
"""Predict mean and variance marginalized over all instances.
Returns the predictive mean and variance marginalised over all
instances for a set of configurations.
Note
----
This method overwrites the same method of ~smac.epm.base_epm.AbstractEPM;
the following method is random forest specific
and follows the SMAC2 implementation;
it requires no distribution assumption
to marginalize the uncertainty estimates
Parameters
----------
X : np.ndarray
[n_samples, n_features (config)]
Returns
-------
means : np.ndarray of shape = [n_samples, 1]
Predictive mean
vars : np.ndarray of shape = [n_samples, 1]
Predictive variance
"""
if self.instance_features is None or \
len(self.instance_features) == 0:
mean_, var = self.predict(X)
var[var < self.var_threshold] = self.var_threshold
var[np.isnan(var)] = self.var_threshold
return mean_, var
if len(X.shape) != 2:
raise ValueError('Expected 2d array, got %dd array!' %
len(X.shape))
if X.shape[1] != len(self.bounds):
raise ValueError('Rows in X should have %d entries but have %d!' %
(len(self.bounds), X.shape[1]))
mean_ = np.zeros(X.shape[0])
var = np.zeros(X.shape[0])
for i, x in enumerate(X):
# marginalize over instance
# 1. Get predictions for all networks
# Not very efficient
preds_nns1 = np.zeros(
[len(self.instance_features), self.number_of_networks])
#for i_f, feat in enumerate(self.instance_features):
# x_ = np.concatenate([x, feat]).reshape([1, -1])
# print(i_f, x_)
# m, _ = self._predict_individual(x_)
# preds_nns1[i_f, :] = m
input = np.concatenate((np.tile(
x, (len(self.instance_features), 1)), self.instance_features),
axis=1)
preds_nns, _ = self._predict_individual(input)
# 2. Average in each NN for all instances
pred_per_nn = []
for nn_id in range(self.number_of_networks):
if self.compute_mean_in_logspace:
pred_per_nn.append(
np.log(np.mean(np.exp(preds_nns[:, nn_id]))))
else:
pred_per_nn.append(np.mean(preds_nns[:, nn_id]))
# 3. compute statistics across trees
mean_x = np.mean(pred_per_nn)
var_x = np.var(pred_per_nn)
if var_x < self.var_threshold:
var_x = self.var_threshold
var[i] = var_x
mean_[i] = mean_x
if len(mean_.shape) == 1:
mean_ = mean_.reshape((-1, 1))
if len(var.shape) == 1:
var = var.reshape((-1, 1))
return mean_, var
def __init__(
self,
ta: typing.Callable,
stats: Stats,
run_obj: str = "quality",
memory_limit: typing.Optional[int] = None,
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = False,
use_pynisher: bool = True,
):
super().__init__(
ta=ta,
stats=stats,
run_obj=run_obj,
par_factor=par_factor,
cost_for_crash=cost_for_crash,
abort_on_first_run_crash=abort_on_first_run_crash,
)
"""
Abstract class for having a function as target algorithm
Parameters
----------
ta : callable
Function (target algorithm) to be optimized.
stats: Stats()
stats object to collect statistics about runtime and so on
run_obj: str
run objective of SMAC
memory_limit : int, optional
Memory limit (in MB) that will be applied to the target algorithm.
par_factor: int
penalization factor
cost_for_crash : float
cost that is used in case of crashed runs (including runs
that returned NaN or inf)
use_pynisher: bool
use pynisher to limit resources;
if disabled
* TA func can use as many resources
as it wants (time and memory) --- use with caution
* all runs will be returned as SUCCESS if returned value is not None
"""
self.ta = ta
self.stats = stats
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
signature = inspect.signature(ta).parameters
self._accepts_seed = 'seed' in signature.keys()
self._accepts_instance = 'instance' in signature.keys()
self._accepts_budget = 'budget' in signature.keys()
if not callable(ta):
raise TypeError('Argument `ta` must be a callable, but is %s' %
type(ta))
self._ta = typing.cast(typing.Callable, ta)
if memory_limit is not None:
memory_limit = int(math.ceil(memory_limit))
self.memory_limit = memory_limit
self.use_pynisher = use_pynisher
self.logger = PickableLoggerAdapter(self.__module__ + '.' +
self.__class__.__name__)
class OutputWriter(object):
"""Writing scenario to file."""
def __init__(self) -> None:
self.logger = PickableLoggerAdapter(name=self.__module__ + "." +
self.__class__.__name__)
def write_scenario_file(self, scenario: 'Scenario') -> None:
"""Write scenario to a file (format is compatible with input_reader).
Will overwrite if file exists. If you have arguments that need special
parsing when saving, specify so in the _parse_argument-function.
Creates output-dir if necessesary.
Parameters
----------
scenario: Scenario
Scenario to be written to file
Returns
-------
status: False or None
False indicates that writing process failed
"""
if scenario.output_dir_for_this_run is None or scenario.output_dir_for_this_run == "":
scenario.logger.info("No output directory for scenario logging "
"specified -- scenario will not be logged.")
return
# Create output-dir if necessary
if not os.path.isdir(scenario.output_dir_for_this_run):
scenario.logger.debug("Output directory does not exist! Will be "
"created.")
try:
os.makedirs(scenario.output_dir_for_this_run)
except OSError:
scenario.logger.debug("Could not make output directory.",
exc_info=True)
raise OSError("Could not make output directory: "
"{}.".format(scenario.output_dir_for_this_run))
# options_dest2name maps scenario._arguments from dest -> name
options_dest2name = {(scenario._arguments[v]['dest']
if scenario._arguments[v]['dest'] else v):
v.lstrip('-').replace('-', '_')
for v in scenario._arguments}
# Write all options into "output_dir/scenario.txt"
path = os.path.join(scenario.output_dir_for_this_run, "scenario.txt")
scenario.logger.debug("Writing scenario-file to {}.".format(path))
with open(path, 'w') as fh:
for key in options_dest2name:
key = key.lstrip('-').replace('-', '_')
new_value = self._parse_argument(scenario, key,
getattr(scenario, key))
if new_value is not None:
fh.write("{} = {}\n".format(options_dest2name[key],
new_value))
def _parse_argument(self, scenario: 'Scenario', key: str,
value: typing.Any) -> typing.Any:
"""Some values of the scenario-file need to be changed upon writing,
such as the 'ta' (target algorithm), due to it's callback. Also,
the configspace, features, train_inst- and test-inst-lists are saved
to output_dir, if they exist.
Parameters:
-----------
scenario: Scenario
Scenario-file to be written
key: string
Name of the attribute in scenario-file
value: Any
Corresponding attribute
Returns:
--------
new value: string
The altered value, to be written to file
Sideeffects:
------------
- copies files pcs_fn, train_inst_fn, test_inst_fn and feature_fn to
output if possible, creates the files from attributes otherwise
"""
if key in ['pcs_fn', 'train_inst_fn', 'test_inst_fn', 'feature_fn']:
# Copy if file exists, else write to new file
if value is not None and os.path.isfile(value):
try:
assert scenario.output_dir_for_this_run is not None # please mypy
new_path = shutil.copy(value,
scenario.output_dir_for_this_run)
except shutil.SameFileError:
new_path = value # File is already in output_dir
# For .pcs-file, also save with the same basename as json and use json-path!
if key == 'pcs_fn' and scenario.cs is not None and value.endswith(
'.pcs'): # type: ignore[attr-defined] # noqa F821
file_name = os.path.splitext(os.path.basename(value))[0]
assert scenario.output_dir_for_this_run is not None # please mypy
new_path = os.path.join(scenario.output_dir_for_this_run,
file_name + '.json')
self.save_configspace(
scenario.cs, new_path,
'json') # type: ignore[attr-defined] # noqa F821
scenario.logger.debug(
"Setting the pcs_fn-attr of written scenario from %s to %s",
value, new_path)
elif key == 'pcs_fn' and scenario.cs is not None: # type: ignore[attr-defined] # noqa F821
try:
assert scenario.output_dir_for_this_run is not None # please mypy
pcs_path = os.path.join(scenario.output_dir_for_this_run,
'configspace.pcs')
self.save_configspace(
scenario.cs, pcs_path,
'pcs_new') # type: ignore[attr-defined] # noqa F821
except TypeError:
self.logger.error(
"Could not write pcs file to disk."
" ConfigSpace not compatible with (new) pcs format.")
assert scenario.output_dir_for_this_run is not None # please mypy
new_path = os.path.join(scenario.output_dir_for_this_run,
'configspace.json')
self.save_configspace(
scenario.cs, new_path,
'json') # type: ignore[attr-defined] # noqa F821
elif key == 'train_inst_fn' and scenario.train_insts != [None]:
assert scenario.output_dir_for_this_run is not None # please mypy
new_path = os.path.join(scenario.output_dir_for_this_run,
'train_insts.txt')
self.write_inst_file(scenario.train_insts, new_path)
elif key == 'test_inst_fn' and scenario.test_insts != [None]:
assert scenario.output_dir_for_this_run is not None # please mypy
new_path = os.path.join(scenario.output_dir_for_this_run,
'test_insts.txt')
self.write_inst_file(scenario.test_insts, new_path)
elif key == 'feature_fn' and scenario.feature_dict != {}:
assert scenario.output_dir_for_this_run is not None # please mypy
new_path = os.path.join(scenario.output_dir_for_this_run,
'features.txt')
self.write_inst_features_file(scenario.n_features,
scenario.feature_dict, new_path)
else:
return None
# New value -> new path
return new_path
elif key == 'ta' and value is not None:
# Reversing the callback on 'ta' (shlex.split)
return " ".join(value)
elif key in ['train_insts', 'test_insts', 'cs', 'feature_dict']:
# No need to log, recreated from files
return None
else:
return value
def write_inst_file(self, insts: typing.List[str], fn: str) -> None:
"""Writes instance-list to file.
Parameters
----------
insts: list<string>
Instance list to be written
fn: string
Output path
"""
with open(fn, 'w') as fh:
fh.write("\n".join(insts))
def write_inst_features_file(
self,
n_features: int,
feat_dict: typing.Dict[str, typing.Iterable[float]],
fn: str,
) -> None:
"""Writes features to file.
Parameters
----------
n_features: int
Number of features
feat_dict: dict
Features to be written
fn: string
File name of instance feature file
"""
header = "Instance, " + ", ".join(
["feature" + str(i) for i in range(n_features)]) + "\n"
body = [
", ".join([inst] + [str(f) for f in feat_dict[inst]]) + "\n"
for inst in feat_dict
]
with open(fn, 'w') as fh:
fh.write(header + "".join(body))
def save_configspace(self, cs: ConfigurationSpace, fn: str,
output_format: str) -> None:
"""Writing ConfigSpace to file.
Parameters
----------
cs : ConfigurationSpace
Config-space to be written
fn : str
Output-file-path
output_format : str
Output format of the configuration space file. Currently,
``json`` and ``pcs_new`` are supported.
"""
writers = {'pcs_new': pcs_new.write, 'json': json.write}
writer = writers.get(output_format)
if writer:
with open(fn, 'w') as fh:
fh.write(writer(cs))
else:
raise ValueError(
"Configuration space output format %s not supported. "
"Please choose one of %s" %
(output_format, set(writers.keys())))
def __init__(self, file_system=LocalFS()):
self.file_system = file_system
self.logger = PickableLoggerAdapter(name=self.__module__ + "." +
self.__class__.__name__)
class DNGO(BaseModel):
def __init__(self,
configspace: ConfigurationSpace,
types: np.ndarray,
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
hidden_dims: typing.List[int] = [50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 10000,
batch_size: int = 8,
var: bool = True,
**kwargs):
super().__init__(configspace, types, bounds, seed, **kwargs)
print("USE DNGO")
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 100
self.log_error = 1000
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.nn = None
self.blr = None
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def _train(self, X: np.ndarray, y: np.ndarray):
self.nn = SimpleNetworkEmbedding(
hidden_dims=self.hidden_dims,
lr=self.lr,
seed=self.seed,
momentum=self.momentum,
weight_decay=self.weight_decay,
iterations=self.iterations,
batch_size=self.batch_size,
var=self.var,
)
self.blr = BayesianLinearRegressionLayer()
self._my = np.mean(y)
self._sy = np.std(y)
y -= self._my
y /= self._sy
#print(X, y)
#import matplotlib.pyplot as plt
self.nn.train(X, y)
#plt.scatter(X, y)
#x_dense = np.linspace(-0.1, 1.1, 100)
#pred = self._predict_nn(x_dense.reshape([-1, 1]))
#m = pred[:, 0].flatten()
#v = pred[:, 1].flatten()
#plt.plot(x_dense, m, label="nn")
#plt.fill_between(x_dense, m - v, m + v, alpha=0.5)
self.blr.optimize_alpha_beta(self.nn.model, X, y)
#m, v = self.blr.predict(self.model, x_dense.reshape([-1, 1]))
#m = m.data.numpy().flatten()
#v = v.data.numpy().flatten()
#plt.scatter(X, y)
#plt.plot(x_dense, m, label="blr")
#plt.fill_between(x_dense, m-v, m+v, alpha=0.5)
#plt.legend()
#plt.ylim([-10, 10])
#plt.show()
def _predict(self, X: np.ndarray) -> typing.Tuple[np.ndarray, np.ndarray]:
means, vars = self.blr.predict(self.nn.model, X)
means = means.data.numpy().flatten()
vars = vars.data.numpy().flatten()
means = np.array(means * self._sy + self._my).reshape([-1, 1])
vars = np.array(vars * self._sy**2).reshape([-1, 1])
if not np.isfinite(means).any():
self.logger.critical(
"All DNGO predictions are NaN. Fall back to random predictions"
)
return np.random.randn(means.shape[0],
means.shape[1]), np.zeros_like(vars)
else:
return means, vars
class NeuralNet(nn.Module):
def __init__(self,
hidden_dims,
input_size,
feat_type=None,
var: bool = True,
max_cat: int = np.inf):
super(NeuralNet, self).__init__()
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
self.feat_type = feat_type
self.input_size = input_size
self.num_neurons = hidden_dims
self.activation = nn.Tanh
self.num_layer = len(hidden_dims)
self.max_cat = max_cat
if var:
self.n_output = 2
else:
self.n_output = 1
if np.sum(self.feat_type) == 0:
self.feat_type = None
if self.feat_type is not None:
self.logger.info("Use cat embedding")
assert len(self.feat_type) == self.input_size
emb = nn.ModuleList()
sz = int(0)
for f in self.feat_type:
if f == 0:
# In SMAC 0 encodes a numerical
emb.append(None)
sz += 1
else:
es = min(self.max_cat, int(f))
emb.append(nn.Embedding(int(f), es))
sz += es
assert int(sz) == sz
sz = int(sz)
num_neurons = [sz] + self.num_neurons
self.embedding = emb
else:
num_neurons = [self.input_size] + self.num_neurons
self.weights = nn.ModuleList()
self.acts = nn.ModuleList()
print(num_neurons)
for i in range(self.num_layer):
self.weights.append(nn.Linear(num_neurons[i], num_neurons[i + 1]))
self.acts.append(self.activation())
self.outlayer = nn.Linear(num_neurons[-1], self.n_output)
def initialize_weights(self, var_bias_init: float = 1):
# Use Xavier normal intialization, slightly modified from "Understanding the difficulty of ..."
for i in range(len(self.weights)):
torch.nn.init.xavier_normal_(self.weights[i].weight)
self.weights[i].bias.data.fill_(0)
torch.nn.init.xavier_normal_(self.outlayer.weight)
# TODO Second bias should be initialised to np.log(np.exp(x) - 1), s.t. softplus = x
self.outlayer.bias.data[0].fill_(0)
if var_bias_init == 0:
self.logger.critical(
"Can't properly initialize bias unit, initialize wih zero")
self.outlayer.bias.data[0].fill_(0)
else:
self.outlayer.bias.data[1].fill_(np.log(np.exp(var_bias_init) - 1))
def learn_initial_weights(self, X):
"""Learn initial weights such that the mean over the data is on average zero per neuron"""
output = torch.tensor(X, dtype=torch.float32)
for i in range(len(self.weights)):
torch.nn.init.xavier_normal_(self.weights[i].weight,
torch.nn.init.calculate_gain('tanh'))
self.weights[i].bias.data.fill_(0)
output2 = self.weights[i].forward(output)
mean = output2.mean(axis=0)
self.weights[i].bias.data = -mean
output = self.weights[i].forward(output)
output = self.acts[i](output)
# print(output.mean(axis=0), output.mean(axis=0).shape)
torch.nn.init.xavier_normal_(self.outlayer.weight,
torch.nn.init.calculate_gain('tanh'))
self.outlayer.bias.data.fill_(0)
# self.outlayer.bias.data[1].fill_(np.log(np.exp(1) - 1))
# Noise can be tuned here...
self.outlayer.bias.data[1] = -5
def forward(self, x):
out = []
if self.feat_type is not None:
for idx, (emb,
typ) in enumerate(zip(self.embedding, self.feat_type)):
if typ == 0:
# a numerical
out.append(x[:, idx].view(-1, 1))
else:
# a categorical
out.append(
emb(x[:, idx].long().view(-1, 1)).view(
[-1, min(self.max_cat, typ)]))
out = torch.cat(out, 1)
else:
out = x
for i in range(self.num_layer):
out = self.weights[i](out)
out = self.acts[i](out)
out = self.outlayer(out)
if self.n_output == 2:
# Passing second output through softplus function (see Lakshminarayanan (2017))
out[:, 1] = torch.log(1 + torch.exp(out[:, 1])) + 10e-6
return out
class EnsembleNN(AbstractEPM):
def __init__(self,
configspace: ConfigurationSpace,
types: typing.List[int],
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
hidden_dims: typing.List[int] = [50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 5000,
batch_size: int = 16,
number_of_networks: int = 5,
var: bool = True,
train_with_lognormal_llh=False,
compute_mean_in_logspace=False,
max_cat: int = np.inf,
ignore_cens: bool = False,
learned_weight_init: bool = False,
optimization_algorithm: str = 'sgd',
**kwargs):
super().__init__(configspace, types, bounds, seed, **kwargs)
#self.types[self.types == 0] = -1
self.types = [int(f) for f in self.types]
assert not (train_with_lognormal_llh and compute_mean_in_logspace)
if type(self.seed) != int:
self.seed = self.seed[0]
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 1000
self.log_error = 5000
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.number_of_networks = number_of_networks
self.train_with_lognormal = train_with_lognormal_llh
self.compute_mean_in_logspace = compute_mean_in_logspace
self.max_cat = max_cat
self.ignore_cens = ignore_cens
self.learned_weight_init = learned_weight_init
self.optimization_algorithm = optimization_algorithm
self._my = None
self._sy = None
# Quick check, should not take too long
a = np.random.normal(42, 23, 1000)
m1, v1 = (np.mean(a), np.var(a))
a = self._preprocess_y(a)
m2, v2 = self._postprocess_mv(np.mean(a), np.var(a))
assert np.abs(m1 - m2) < 1e-3, (m1, m2)
assert np.abs(v1 - v2) < 1e-3, (v1, v2)
self._my = None
self._sy = None
self.nns = None
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def _preprocess_y(self, y: np.ndarray, redo=False):
if self._my is None or redo:
self._my = np.mean(y)
self._sy = np.std(y)
if self._sy == 0:
# all y's are the same
self._sy = 1
if not self.train_with_lognormal:
y -= self._my
y /= self._sy
return y
def _postprocess_mv(self, m: np.ndarray, v: np.ndarray):
# zero mean scaling
m = m * self._sy + self._my
v = v * self._sy**2
return m, v
def _preprocess_x(self, x: np.ndarray, redo: bool = False):
# Replace nans with 0, should be fine for both cats and conts
# TODO: Maybe refine this and replace cont with mean
x = np.nan_to_num(x)
return x
def _train(self, X: np.ndarray, Y: np.ndarray, C: np.ndarray = None):
self.logger.critical("Not using C as this is not a Tobit model")
Y = self._preprocess_y(Y, redo=True)
X = self._preprocess_x(X, redo=True)
self.train_data = (X, Y)
self.nns = []
self.logger.debug("Start Training %d networks" %
self.number_of_networks)
for i in range(self.number_of_networks):
nn = SimpleNetworkEmbedding(
hidden_dims=self.hidden_dims,
feat_types=self.types,
lr=self.lr,
seed=self.seed + i,
momentum=self.momentum,
weight_decay=self.weight_decay,
iterations=self.iterations,
batch_size=self.batch_size,
var=self.var,
lognormal_nllh=self.train_with_lognormal,
var_bias_init=np.std(Y),
max_cat=self.max_cat,
learned_weight_init=self.learned_weight_init,
optimization_algorithm=self.optimization_algorithm,
)
nn.reset()
nn.train(X, Y)
self.nns.append(nn)
def _predict_individual(
self, X: np.ndarray) -> typing.Tuple[np.ndarray, np.ndarray]:
X = self._preprocess_x(X, redo=True)
ms = np.zeros([X.shape[0], self.number_of_networks])
vs = np.zeros([X.shape[0], self.number_of_networks])
for i_nn, nn in enumerate(self.nns):
pred = nn.predict(X)
m = pred[:, 0]
v = pred[:, 1]
if not self.train_with_lognormal:
m, v = self._postprocess_mv(m, v)
ms[:, i_nn] = m
vs[:, i_nn] = v
return ms, vs
def _predict(self, X: np.ndarray) -> typing.Tuple[np.ndarray, np.ndarray]:
ms, _ = self._predict_individual(X)
m = ms.mean(axis=1)
v = ms.var(axis=1)
return m.reshape((-1, 1)), v.reshape((-1, 1))
def predict_marginalized_over_instances(self, X: np.ndarray):
"""Predict mean and variance marginalized over all instances.
Returns the predictive mean and variance marginalised over all
instances for a set of configurations.
Note
----
This method overwrites the same method of ~smac.epm.base_epm.AbstractEPM;
the following method is random forest specific
and follows the SMAC2 implementation;
it requires no distribution assumption
to marginalize the uncertainty estimates
Parameters
----------
X : np.ndarray
[n_samples, n_features (config)]
Returns
-------
means : np.ndarray of shape = [n_samples, 1]
Predictive mean
vars : np.ndarray of shape = [n_samples, 1]
Predictive variance
"""
if self.instance_features is None or \
len(self.instance_features) == 0:
mean_, var = self.predict(X)
var[var < self.var_threshold] = self.var_threshold
var[np.isnan(var)] = self.var_threshold
return mean_, var
if len(X.shape) != 2:
raise ValueError('Expected 2d array, got %dd array!' %
len(X.shape))
if X.shape[1] != len(self.bounds):
raise ValueError('Rows in X should have %d entries but have %d!' %
(len(self.bounds), X.shape[1]))
mean_ = np.zeros((X.shape[0], 1))
var = np.zeros(X.shape[0])
for i, x in enumerate(X):
# marginalize over instance
# 1. Get predictions for all networks
# Not very efficient
# preds_nns1 = np.zeros([len(self.instance_features), self.number_of_networks])
#for i_f, feat in enumerate(self.instance_features):
# x_ = np.concatenate([x, feat]).reshape([1, -1])
# print(i_f, x_)
# m, _ = self._predict_individual(x_)
# preds_nns1[i_f, :] = m
input = np.concatenate((np.tile(
x, (len(self.instance_features), 1)), self.instance_features),
axis=1)
preds_nns, _ = self._predict_individual(input)
# 2. Average in each NN for all instances
pred_per_nn = []
for nn_id in range(self.number_of_networks):
if self.compute_mean_in_logspace:
pred_per_nn.append(
np.log(np.mean(np.exp(preds_nns[:, nn_id]))))
else:
pred_per_nn.append(np.mean(preds_nns[:, nn_id]))
# 3. compute statistics across trees
mean_x = np.mean(pred_per_nn)
var_x = np.var(pred_per_nn)
if var_x < self.var_threshold:
var_x = self.var_threshold
var[i] = var_x
mean_[i] = mean_x
if len(mean_.shape) == 1:
mean_ = mean_.reshape((-1, 1))
if len(var.shape) == 1:
var = var.reshape((-1, 1))
return mean_, var
def __init__(self,
configspace: ConfigurationSpace,
types: typing.List[int],
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
hidden_dims: typing.List[int] = [50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 5000,
batch_size: int = 16,
number_of_networks: int = 5,
var: bool = True,
train_with_lognormal_llh=False,
compute_mean_in_logspace=False,
max_cat: int = np.inf,
ignore_cens: bool = False,
learned_weight_init: bool = False,
optimization_algorithm: str = 'sgd',
**kwargs):
super().__init__(configspace, types, bounds, seed, **kwargs)
#self.types[self.types == 0] = -1
self.types = [int(f) for f in self.types]
assert not (train_with_lognormal_llh and compute_mean_in_logspace)
if type(self.seed) != int:
self.seed = self.seed[0]
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 1000
self.log_error = 5000
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.number_of_networks = number_of_networks
self.train_with_lognormal = train_with_lognormal_llh
self.compute_mean_in_logspace = compute_mean_in_logspace
self.max_cat = max_cat
self.ignore_cens = ignore_cens
self.learned_weight_init = learned_weight_init
self.optimization_algorithm = optimization_algorithm
self._my = None
self._sy = None
# Quick check, should not take too long
a = np.random.normal(42, 23, 1000)
m1, v1 = (np.mean(a), np.var(a))
a = self._preprocess_y(a)
m2, v2 = self._postprocess_mv(np.mean(a), np.var(a))
assert np.abs(m1 - m2) < 1e-3, (m1, m2)
assert np.abs(v1 - v2) < 1e-3, (v1, v2)
self._my = None
self._sy = None
self.nns = None
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
class RunHistory(object):
"""Container for target algorithm run information.
Most importantly, the runhistory contains an efficient mapping from each evaluated configuration to the
empirical cost observed on either the full instance set or a subset. The cost is the average over all
observed costs for one configuration:
* If using budgets for a single instance, only the cost on the highest observed budget is returned.
* If using instances as the budget, the average cost over all evaluated instances is returned.
* Theoretically, the runhistory object can handle instances and budgets at the same time. This is
neither used nor tested.
* Capped runs are not included in this cost.
Note
----
Guaranteed to be picklable.
Attributes
----------
data : collections.OrderedDict()
TODO
config_ids : dict
Maps config -> id
ids_config : dict
Maps id -> config
num_runs_per_config : dict
Maps config_id -> number of runs
Parameters
----------
overwrite_existing_runs : bool (default=True)
If set to ``True`` and a run of a configuration on an instance-budget-seed-pair already exists,
it is overwritten.
"""
def __init__(self, overwrite_existing_runs: bool = False) -> None:
"""Constructor
Parameters
----------
overwrite_existing_runs: bool
allows to overwrites old results if pairs of
algorithm-instance-seed were measured
multiple times
"""
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
# By having the data in a deterministic order we can do useful tests
# when we serialize the data and can assume it's still in the same
# order as it was added.
self.data = collections.OrderedDict(
) # type: typing.Dict[RunKey, RunValue]
# for fast access, we have also an unordered data structure
# to get all instance seed pairs of a configuration.
# This does not include capped runs.
self._configid_to_inst_seed_budget = {
} # type: typing.Dict[int, typing.Dict[InstSeedKey, typing.List[float]]]
self.config_ids = {} # type: typing.Dict[Configuration, int]
self.ids_config = {} # type: typing.Dict[int, Configuration]
self._n_id = 0
# Stores cost for each configuration ID
self._cost_per_config = {} # type: typing.Dict[int, float]
# Stores min cost across all budgets for each configuration ID
self._min_cost_per_config = {} # type: typing.Dict[int, float]
# runs_per_config maps the configuration ID to the number of runs for that configuration
# and is necessary for computing the moving average
self.num_runs_per_config = {} # type: typing.Dict[int, int]
# Store whether a datapoint is "external", which means it was read from
# a JSON file. Can be chosen to not be written to disk
self.external = {} # type: typing.Dict[RunKey, DataOrigin]
self.overwrite_existing_runs = overwrite_existing_runs
def add(
self,
config: Configuration,
cost: float,
time: float,
status: StatusType,
instance_id: typing.Optional[str] = None,
seed: typing.Optional[int] = None,
budget: float = 0.0,
starttime: float = 0.0,
endtime: float = 0.0,
additional_info: typing.Optional[typing.Dict] = None,
origin: DataOrigin = DataOrigin.INTERNAL,
force_update: bool = False,
) -> None:
"""Adds a data of a new target algorithm (TA) run;
it will update data if the same key values are used
(config, instance_id, seed)
Parameters
----------
config : dict (or other type -- depending on config space module)
Parameter configuration
cost: float
Cost of TA run (will be minimized)
time: float
Runtime of TA run
status: str
Status in {SUCCESS, TIMEOUT, CRASHED, ABORT, MEMOUT}
instance_id: str
String representing an instance (default: None)
seed: int
Random seed used by TA (default: None)
budget: float
budget (cutoff) used in intensifier to limit TA (default: 0)
starttime: float
starting timestamp of TA evaluation
endtime: float
ending timestamp of TA evaluation
additional_info: dict
Additional run infos (could include further returned
information from TA or fields such as start time and host_id)
origin: DataOrigin
Defines how data will be used.
force_update: bool (default: False)
Forces the addition of a config to the history
"""
if config is None:
raise TypeError(
'Configuration to add to the runhistory must not be None')
elif not isinstance(config, Configuration):
raise TypeError(
'Configuration to add to the runhistory is not of type Configuration, but %s'
% type(config))
# Get the config id
config_id_tmp = self.config_ids.get(config)
if config_id_tmp is None:
self._n_id += 1
self.config_ids[config] = self._n_id
config_id = typing.cast(int, self.config_ids.get(config))
self.ids_config[self._n_id] = config
else:
config_id = typing.cast(int, config_id_tmp)
# Construct keys and values for the data dictionary
k = RunKey(config_id, instance_id, seed, budget)
v = RunValue(cost, time, status, starttime, endtime, additional_info)
# Each runkey is supposed to be used only once. Repeated tries to add
# the same runkey will be ignored silently if not capped.
if self.overwrite_existing_runs or force_update or self.data.get(
k) is None:
self._add(k, v, status, origin)
elif status != StatusType.CAPPED and self.data[
k].status == StatusType.CAPPED:
# overwrite capped runs with uncapped runs
self._add(k, v, status, origin)
elif status == StatusType.CAPPED and self.data[
k].status == StatusType.CAPPED and cost > self.data[k].cost:
# overwrite if censored with a larger cutoff
self._add(k, v, status, origin)
def _add(self, k: RunKey, v: RunValue, status: StatusType,
origin: DataOrigin) -> None:
"""Actual function to add new entry to data structures
TODO
"""
self.data[k] = v
self.external[k] = origin
# Capped data is added above
# Do not register the cost until the run has completed
if origin in (DataOrigin.INTERNAL, DataOrigin.EXTERNAL_SAME_INSTANCES) \
and status not in [StatusType.CAPPED, StatusType.RUNNING]:
# also add to fast data structure
is_k = InstSeedKey(k.instance_id, k.seed)
self._configid_to_inst_seed_budget[
k.config_id] = self._configid_to_inst_seed_budget.get(
k.config_id, {})
if is_k not in self._configid_to_inst_seed_budget[
k.config_id].keys():
# add new inst-seed-key with budget to main dict
self._configid_to_inst_seed_budget[k.config_id][is_k] = [
k.budget
]
elif k.budget not in is_k:
# append new budget to existing inst-seed-key dict
self._configid_to_inst_seed_budget[k.config_id][is_k].append(
k.budget)
# if budget is used, then update cost instead of incremental updates
if not self.overwrite_existing_runs and k.budget == 0:
# assumes an average across runs as cost function aggregation, this is used for algorithm configuration
# (incremental updates are used to save time as getting the cost for > 100 instances is high)
self.incremental_update_cost(self.ids_config[k.config_id],
v.cost)
else:
# this is when budget > 0 (only successive halving and hyperband so far)
self.update_cost(config=self.ids_config[k.config_id])
if k.budget > 0:
if self.num_runs_per_config[
k.
config_id] != 1: # This is updated in update_cost
raise ValueError('This should not happen!')
def update_cost(self, config: Configuration) -> None:
"""Store the performance of a configuration across the instances in
self.cost_per_config and also updates self.runs_per_config;
Note
----
This method ignores capped runs.
Parameters
----------
config: Configuration
configuration to update cost based on all runs in runhistory
"""
config_id = self.config_ids[config]
# removing duplicates while keeping the order
inst_seed_budgets = list(
dict.fromkeys(
self.get_runs_for_config(config,
only_max_observed_budget=True)))
self._cost_per_config[config_id] = self.average_cost(
config, inst_seed_budgets)
self.num_runs_per_config[config_id] = len(inst_seed_budgets)
all_inst_seed_budgets = list(
dict.fromkeys(
self.get_runs_for_config(config,
only_max_observed_budget=False)))
self._min_cost_per_config[config_id] = self.min_cost(
config, all_inst_seed_budgets)
def incremental_update_cost(self, config: Configuration,
cost: float) -> None:
"""Incrementally updates the performance of a configuration by using a
moving average;
Parameters
----------
config: Configuration
configuration to update cost based on all runs in runhistory
cost: float
cost of new run of config
"""
config_id = self.config_ids[config]
n_runs = self.num_runs_per_config.get(config_id, 0)
old_cost = self._cost_per_config.get(config_id, 0.)
self._cost_per_config[config_id] = (
(old_cost * n_runs) + cost) / (n_runs + 1)
self.num_runs_per_config[config_id] = n_runs + 1
def get_cost(self, config: Configuration) -> float:
"""Returns empirical cost for a configuration.
See the class docstring for how the costs are computed. The costs are not re-computed, but are read from cache.
Parameters
----------
config: Configuration
Returns
-------
cost: float
Computed cost for configuration
"""
config_id = self.config_ids.get(config)
return self._cost_per_config.get(
config_id, np.nan) # type: ignore[arg-type] # noqa F821
def get_runs_for_config(
self, config: Configuration,
only_max_observed_budget: bool) -> typing.List[InstSeedBudgetKey]:
"""Return all runs (instance seed pairs) for a configuration.
Note
----
This method ignores capped runs.
Parameters
----------
config : Configuration from ConfigSpace
Parameter configuration
only_max_observed_budget : bool
Select only the maximally observed budget run for this configuration
Returns
-------
instance_seed_budget_pairs : list<tuples of instance, seed, budget>
"""
config_id = self.config_ids.get(config)
runs = self._configid_to_inst_seed_budget.get(
config_id, {}).copy() # type: ignore[arg-type] # noqa F821
# select only the max budget run if specified
if only_max_observed_budget:
for k, v in runs.items():
runs[k] = [max(v)]
# convert to inst-seed-budget key
rval = [
InstSeedBudgetKey(k.instance, k.seed, budget)
for k, v in runs.items() for budget in v
]
return rval
def get_all_configs(self) -> typing.List[Configuration]:
"""Return all configurations in this RunHistory object
Returns
-------
parameter configurations: list
"""
return list(self.config_ids.keys())
def get_all_configs_per_budget(
self,
budget_subset: typing.Optional[typing.List] = None,
) -> typing.List[Configuration]:
"""
Return all configs in this RunHistory object that have been run on one of these budgets
Parameter
---------
budget_subset: list
Returns
-------
parameter configurations: list
"""
if budget_subset is None:
return self.get_all_configs()
configs = []
for c, i, s, b in self.data.keys():
if b in budget_subset:
configs.append(self.ids_config[c])
return configs
def get_min_cost(self, config: Configuration) -> float:
"""Returns the lowest empirical cost for a configuration, across all runs (budgets)
See the class docstring for how the costs are computed. The costs are not re-computed, but are read from cache.
Parameters
----------
config: Configuration
Returns
-------
min_cost: float
Computed cost for configuration
"""
config_id = self.config_ids.get(config)
return self._min_cost_per_config.get(
config_id, np.nan) # type: ignore[arg-type] # noqa F821
def empty(self) -> bool:
"""Check whether or not the RunHistory is empty.
Returns
-------
emptiness: bool
True if runs have been added to the RunHistory,
False otherwise
"""
return len(self.data) == 0
def save_json(self,
fn: str = "runhistory.json",
save_external: bool = False) -> None:
"""
saves runhistory on disk
Parameters
----------
fn : str
file name
save_external : bool
Whether to save external data in the runhistory file.
"""
data = [([
int(k.config_id),
str(k.instance_id) if k.instance_id is not None else None,
int(k.seed),
float(k.budget) if k[3] is not None else 0
], list(v)) for k, v in self.data.items()
if save_external or self.external[k] == DataOrigin.INTERNAL]
config_ids_to_serialize = set([entry[0][0] for entry in data])
configs = {
id_: conf.get_dictionary()
for id_, conf in self.ids_config.items()
if id_ in config_ids_to_serialize
}
config_origins = {
id_: conf.origin
for id_, conf in self.ids_config.items()
if (id_ in config_ids_to_serialize and conf.origin is not None)
}
with open(fn, "w") as fp:
json.dump(
{
"data": data,
"config_origins": config_origins,
"configs": configs
},
fp,
cls=EnumEncoder,
indent=2)
def load_json(self, fn: str, cs: ConfigurationSpace) -> None:
"""Load and runhistory in json representation from disk.
Overwrites current runhistory!
Parameters
----------
fn : str
file name to load from
cs : ConfigSpace
instance of configuration space
"""
try:
with open(fn) as fp:
all_data = json.load(fp, object_hook=StatusType.enum_hook)
except Exception as e:
self.logger.warning(
'Encountered exception %s while reading runhistory from %s. '
'Not adding any runs!',
e,
fn,
)
return
config_origins = all_data.get("config_origins", {})
self.ids_config = {
int(id_): Configuration(cs,
values=values,
origin=config_origins.get(id_, None))
for id_, values in all_data["configs"].items()
}
self.config_ids = {
config: id_
for id_, config in self.ids_config.items()
}
self._n_id = len(self.config_ids)
# important to use add method to use all data structure correctly
for k, v in all_data["data"]:
self.add(config=self.ids_config[int(k[0])],
cost=float(v[0]),
time=float(v[1]),
status=StatusType(v[2]),
instance_id=k[1],
seed=int(k[2]),
budget=float(k[3]) if len(k) == 4 else 0,
starttime=v[3],
endtime=v[4],
additional_info=v[5])
def update_from_json(
self,
fn: str,
cs: ConfigurationSpace,
origin: DataOrigin = DataOrigin.EXTERNAL_SAME_INSTANCES,
) -> None:
"""Update the current runhistory by adding new runs from a json file.
Parameters
----------
fn : str
File name to load from.
cs : ConfigSpace
Instance of configuration space.
origin : DataOrigin
What to store as data origin.
"""
new_runhistory = RunHistory()
new_runhistory.load_json(fn, cs)
self.update(runhistory=new_runhistory, origin=origin)
def update(
self,
runhistory: 'RunHistory',
origin: DataOrigin = DataOrigin.EXTERNAL_SAME_INSTANCES,
) -> None:
"""Update the current runhistory by adding new runs from a RunHistory.
Parameters
----------
runhistory: RunHistory
Runhistory with additional data to be added to self
origin: DataOrigin
If set to ``INTERNAL`` or ``EXTERNAL_FULL`` the data will be
added to the internal data structure self._configid_to_inst_seed_budget
and be available :meth:`through get_runs_for_config`.
"""
# Configurations might be already known, but by a different ID. This
# does not matter here because the add() method handles this
# correctly by assigning an ID to unknown configurations and re-using
# the ID
for key, value in runhistory.data.items():
config_id, instance_id, seed, budget = key
cost, time, status, start, end, additional_info = value
config = runhistory.ids_config[config_id]
self.add(config=config,
cost=cost,
time=time,
status=status,
instance_id=instance_id,
starttime=start,
endtime=end,
seed=seed,
budget=budget,
additional_info=additional_info,
origin=origin)
def _cost(
self,
config: Configuration,
instance_seed_budget_keys: typing.Optional[
typing.Iterable[InstSeedBudgetKey]] = None,
) -> typing.List[float]:
"""Return array of all costs for the given config for further calculations.
Parameters
----------
config : Configuration
Configuration to calculate objective for
instance_seed_budget_keys : list, optional (default=None)
List of tuples of instance-seeds-budget keys. If None, the run_history is
queried for all runs of the given configuration.
Returns
-------
Costs: list
Array of all costs
"""
try:
id_ = self.config_ids[config]
except KeyError: # challenger was not running so far
return []
if instance_seed_budget_keys is None:
instance_seed_budget_keys = self.get_runs_for_config(
config, only_max_observed_budget=True)
costs = []
for i, r, b in instance_seed_budget_keys:
k = RunKey(id_, i, r, b)
costs.append(self.data[k].cost)
return costs
def average_cost(
self,
config: Configuration,
instance_seed_budget_keys: typing.Optional[
typing.Iterable[InstSeedBudgetKey]] = None,
) -> float:
"""Return the average cost of a configuration.
This is the mean of costs of all instance-seed pairs.
Parameters
----------
config : Configuration
Configuration to calculate objective for
instance_seed_budget_keys : list, optional (default=None)
List of tuples of instance-seeds-budget keys. If None, the run_history is
queried for all runs of the given configuration.
Returns
----------
Cost: float
Average cost
"""
costs = self._cost(config, instance_seed_budget_keys)
if costs:
return float(np.mean(costs))
return np.nan
def sum_cost(
self,
config: Configuration,
instance_seed_budget_keys: typing.Optional[
typing.Iterable[InstSeedBudgetKey]] = None,
) -> float:
"""Return the sum of costs of a configuration.
This is the sum of costs of all instance-seed pairs.
Parameters
----------
config : Configuration
Configuration to calculate objective for
instance_seed_budget_keys : list, optional (default=None)
List of tuples of instance-seeds-budget keys. If None, the run_history is
queried for all runs of the given configuration.
Returns
----------
sum_cost: float
Sum of costs of config
"""
return float(np.sum(self._cost(config, instance_seed_budget_keys)))
def min_cost(
self,
config: Configuration,
instance_seed_budget_keys: typing.Optional[
typing.Iterable[InstSeedBudgetKey]] = None,
) -> float:
"""Return the minimum cost of a configuration
This is the minimum cost of all instance-seed pairs.
Parameters
----------
config : Configuration
Configuration to calculate objective for
instance_seed_budget_keys : list, optional (default=None)
List of tuples of instance-seeds-budget keys. If None, the run_history is
queried for all runs of the given configuration.
Returns
----------
min_cost: float
minimum cost of config
"""
costs = self._cost(config, instance_seed_budget_keys)
if costs:
return float(np.min(costs))
return np.nan
def compute_all_costs(self,
instances: typing.Optional[typing.List[str]] = None
) -> None:
"""Computes the cost of all configurations from scratch and overwrites
self.cost_perf_config and self.runs_per_config accordingly;
Note
----
This method is only used for ``merge_foreign_data`` and should be removed.
Parameters
----------
instances: typing.List[str]
list of instances; if given, cost is only computed wrt to this instance set
"""
self._cost_per_config = {}
self.num_runs_per_config = {}
for config, config_id in self.config_ids.items():
# removing duplicates while keeping the order
inst_seed_budgets = list(
dict.fromkeys(
self.get_runs_for_config(config,
only_max_observed_budget=True)))
if instances is not None:
inst_seed_budgets = list(
filter(
lambda x: x.instance in typing.cast(
typing.List, instances), inst_seed_budgets))
if inst_seed_budgets: # can be empty if never saw any runs on <instances>
self._cost_per_config[config_id] = self.average_cost(
config, inst_seed_budgets)
self._min_cost_per_config[config_id] = self.min_cost(
config, inst_seed_budgets)
self.num_runs_per_config[config_id] = len(inst_seed_budgets)
def get_instance_costs_for_config(
self,
config: Configuration) -> typing.Dict[str, typing.List[float]]:
""" Returns the average cost per instance (across seeds) for a configuration
If the runhistory contains budgets, only the highest budget for a configuration is returned.
Note
----
This is used by the pSMAC facade to determine the incumbent after the evaluation.
Parameters
----------
config : Configuration from ConfigSpace
Parameter configuration
Returns
-------
cost_per_inst: dict<instance name<str>, cost<float>>
"""
runs_ = self.get_runs_for_config(config, only_max_observed_budget=True)
cost_per_inst = {} # type: typing.Dict[str, typing.List[float]]
for inst, seed, budget in runs_:
cost_per_inst[inst] = cost_per_inst.get(inst, [])
rkey = RunKey(self.config_ids[config], inst, seed, budget)
vkey = self.data[rkey]
cost_per_inst[inst].append(vkey.cost)
cost_per_inst = dict([(inst, np.mean(costs))
for inst, costs in cost_per_inst.items()])
return cost_per_inst
class RunHistory(object):
"""Container for target algorithm run information.
**Note:** Guaranteed to be picklable.
Attributes
----------
data : collections.OrderedDict()
TODO
config_ids : dict
Maps config -> id
ids_config : dict
Maps id -> config
cost_per_config : dict
Maps config_id -> cost
runs_per_config : dict
Maps config_id -> number of runs
aggregate_func
overwrite_existing_runs
"""
def __init__(self,
aggregate_func: typing.Callable,
overwrite_existing_runs: bool = False) -> None:
"""Constructor
Parameters
----------
aggregate_func: callable
function to aggregate perf across instances
overwrite_existing_runs: bool
allows to overwrites old results if pairs of
algorithm-instance-seed were measured
multiple times
"""
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
# By having the data in a deterministic order we can do useful tests
# when we serialize the data and can assume it's still in the same
# order as it was added.
self.data = collections.OrderedDict(
) # type: typing.Dict[RunKey, RunValue]
# for fast access, we have also an unordered data structure
# to get all instance seed pairs of a configuration
self._configid_to_inst_seed = {} # type: typing.Dict[int, InstSeedKey]
self.config_ids = {} # type: typing.Dict[Configuration, int]
self.ids_config = {} # type: typing.Dict[int, Configuration]
self._n_id = 0
# Stores cost for each configuration ID
self.cost_per_config = {} # type: typing.Dict[int, float]
# runs_per_config maps the configuration ID to the number of runs for that configuration
# and is necessary for computing the moving average
self.runs_per_config = {} # type: typing.Dict[int, int]
# Store whether a datapoint is "external", which means it was read from
# a JSON file. Can be chosen to not be written to disk
self.external = {} # type: typing.Dict[RunKey, DataOrigin]
self.aggregate_func = aggregate_func
self.overwrite_existing_runs = overwrite_existing_runs
def add(self,
config: Configuration,
cost: float,
time: float,
status: StatusType,
instance_id: str = None,
seed: int = None,
additional_info: dict = None,
origin: DataOrigin = DataOrigin.INTERNAL):
"""Adds a data of a new target algorithm (TA) run;
it will update data if the same key values are used
(config, instance_id, seed)
Parameters
----------
config : dict (or other type -- depending on config space module)
Parameter configuration
cost: float
Cost of TA run (will be minimized)
time: float
Runtime of TA run
status: str
Status in {SUCCESS, TIMEOUT, CRASHED, ABORT, MEMOUT}
instance_id: str
String representing an instance (default: None)
seed: int
Random seed used by TA (default: None)
additional_info: dict
Additional run infos (could include further returned
information from TA or fields such as start time and host_id)
origin: DataOrigin
Defines how data will be used.
"""
config_id = self.config_ids.get(config)
if config_id is None:
self._n_id += 1
self.config_ids[config] = self._n_id
config_id = self.config_ids.get(config)
self.ids_config[self._n_id] = config
k = RunKey(config_id, instance_id, seed)
v = RunValue(cost, time, status, additional_info)
# Each runkey is supposed to be used only once. Repeated tries to add
# the same runkey will be ignored silently if not capped.
if self.overwrite_existing_runs or self.data.get(k) is None:
self._add(k, v, status, origin)
elif status != StatusType.CAPPED and self.data[
k].status == StatusType.CAPPED:
# overwrite capped runs with uncapped runs
self._add(k, v, status, origin)
elif status == StatusType.CAPPED and self.data[
k].status == StatusType.CAPPED and cost > self.data[k].cost:
# overwrite if censored with a larger cutoff
self._add(k, v, status, origin)
def _add(self, k: RunKey, v: RunValue, status: StatusType,
origin: DataOrigin):
"""Actual function to add new entry to data structures
TODO
"""
self.data[k] = v
self.external[k] = origin
if origin in (DataOrigin.INTERNAL, DataOrigin.EXTERNAL_SAME_INSTANCES) \
and status != StatusType.CAPPED:
# also add to fast data structure
is_k = InstSeedKey(k.instance_id, k.seed)
self._configid_to_inst_seed[
k.config_id] = self._configid_to_inst_seed.get(
k.config_id, [])
if is_k not in self._configid_to_inst_seed[k.config_id]:
self._configid_to_inst_seed[k.config_id].append(is_k)
if not self.overwrite_existing_runs:
# assumes an average across runs as cost function aggregation
self.incremental_update_cost(self.ids_config[k.config_id],
v.cost)
else:
self.update_cost(config=self.ids_config[k.config_id])
def update_cost(self, config: Configuration):
"""Store the performance of a configuration across the instances in
self.cost_perf_config and also updates self.runs_per_config;
uses self.aggregate_func
Parameters
----------
config: Configuration
configuration to update cost based on all runs in runhistory
"""
inst_seeds = set(self.get_runs_for_config(config))
perf = self.aggregate_func(config, self, inst_seeds)
config_id = self.config_ids[config]
self.cost_per_config[config_id] = perf
self.runs_per_config[config_id] = len(inst_seeds)
def compute_all_costs(self, instances: typing.List[str] = None):
"""Computes the cost of all configurations from scratch and overwrites
self.cost_perf_config and self.runs_per_config accordingly;
Parameters
----------
instances: typing.List[str]
list of instances; if given, cost is only computed wrt to this instance set
"""
self.cost_per_config = {}
self.runs_per_config = {}
for config, config_id in self.config_ids.items():
inst_seeds = set(self.get_runs_for_config(config))
if instances is not None:
inst_seeds = list(
filter(lambda x: x.instance in instances, inst_seeds))
if inst_seeds: # can be empty if never saw any runs on <instances>
perf = self.aggregate_func(config, self, inst_seeds)
self.cost_per_config[config_id] = perf
self.runs_per_config[config_id] = len(inst_seeds)
def incremental_update_cost(self, config: Configuration, cost: float):
"""Incrementally updates the performance of a configuration by using a
moving average;
Parameters
----------
config: Configuration
configuration to update cost based on all runs in runhistory
cost: float
cost of new run of config
"""
config_id = self.config_ids[config]
n_runs = self.runs_per_config.get(config_id, 0)
old_cost = self.cost_per_config.get(config_id, 0.)
self.cost_per_config[config_id] = (
(old_cost * n_runs) + cost) / (n_runs + 1)
self.runs_per_config[config_id] = n_runs + 1
def get_cost(self, config: Configuration):
"""Returns empirical cost for a configuration; uses self.cost_per_config
Parameters
----------
config: Configuration
Returns
-------
cost: float
Computed cost for configuration
"""
config_id = self.config_ids[config]
return self.cost_per_config.get(config_id, np.nan)
def get_runs_for_config(self, config: Configuration):
"""Return all runs (instance seed pairs) for a configuration.
Parameters
----------
config : Configuration from ConfigSpace
Parameter configuration
Returns
-------
instance_seed_pairs : list<tuples of instance, seed>
"""
config_id = self.config_ids.get(config)
return self._configid_to_inst_seed.get(config_id, [])
def get_instance_costs_for_config(self, config: Configuration):
"""
Returns the average cost per instance (across seeds)
for a configuration
Parameters
----------
config : Configuration from ConfigSpace
Parameter configuration
Returns
-------
cost_per_inst: dict<instance name<str>, cost<float>>
"""
config_id = self.config_ids.get(config)
runs_ = self._configid_to_inst_seed.get(config_id, [])
cost_per_inst = {}
for inst, seed in runs_:
cost_per_inst[inst] = cost_per_inst.get(inst, [])
rkey = RunKey(config_id, inst, seed)
vkey = self.data[rkey]
cost_per_inst[inst].append(vkey.cost)
cost_per_inst = dict([(inst, np.mean(costs))
for inst, costs in cost_per_inst.items()])
return cost_per_inst
def get_all_configs(self):
"""Return all configurations in this RunHistory object
Returns
-------
parameter configurations: list
"""
return list(self.config_ids.keys())
def empty(self):
"""Check whether or not the RunHistory is empty.
Returns
-------
emptiness: bool
True if runs have been added to the RunHistory,
False otherwise
"""
return len(self.data) == 0
def save_json(self,
fn: str = "runhistory.json",
save_external: bool = False):
"""
saves runhistory on disk
Parameters
----------
fn : str
file name
save_external : bool
Whether to save external data in the runhistory file.
"""
fn = fn.replace(":", "-")
data = [([
int(k.config_id),
str(k.instance_id) if k.instance_id is not None else None,
int(k.seed)
], list(v)) for k, v in self.data.items()
if save_external or self.external[k] == DataOrigin.INTERNAL]
config_ids_to_serialize = set([entry[0][0] for entry in data])
configs = {
id_: conf.get_dictionary()
for id_, conf in self.ids_config.items()
if id_ in config_ids_to_serialize
}
config_origins = {
id_: conf.origin
for id_, conf in self.ids_config.items()
if (id_ in config_ids_to_serialize and conf.origin is not None)
}
with open(fn, "w") as fp:
json.dump(
{
"data": data,
"config_origins": config_origins,
"configs": configs
},
fp,
cls=EnumEncoder,
indent=2)
def load_json(self, fn: str, cs: ConfigurationSpace):
"""Load and runhistory in json representation from disk.
Overwrites current runhistory!
Parameters
----------
fn : str
file name to load from
cs : ConfigSpace
instance of configuration space
"""
try:
with open(fn) as fp:
all_data = json.load(fp, object_hook=StatusType.enum_hook)
except Exception as e:
self.logger.warning(
'Encountered exception %s while reading runhistory from %s. '
'Not adding any runs!',
e,
fn,
)
return
config_origins = all_data.get("config_origins", {})
self.ids_config = {
int(id_): Configuration(cs,
values=values,
origin=config_origins.get(id_, None))
for id_, values in all_data["configs"].items()
}
self.config_ids = {
config: id_
for id_, config in self.ids_config.items()
}
self._n_id = len(self.config_ids)
# important to use add method to use all data structure correctly
for k, v in all_data["data"]:
self.add(config=self.ids_config[int(k[0])],
cost=float(v[0]),
time=float(v[1]),
status=StatusType(v[2]),
instance_id=k[1],
seed=int(k[2]),
additional_info=v[3])
def update_from_json(
self,
fn: str,
cs: ConfigurationSpace,
origin: DataOrigin = DataOrigin.EXTERNAL_SAME_INSTANCES):
"""Update the current runhistory by adding new runs from a json file.
Parameters
----------
fn : str
File name to load from.
cs : ConfigSpace
Instance of configuration space.
origin : DataOrigin
What to store as data origin.
"""
new_runhistory = RunHistory(self.aggregate_func)
new_runhistory.load_json(fn, cs)
self.update(runhistory=new_runhistory, origin=origin)
def update(self,
runhistory: 'RunHistory',
origin: DataOrigin = DataOrigin.EXTERNAL_SAME_INSTANCES):
"""Update the current runhistory by adding new runs from a RunHistory.
Parameters
----------
runhistory: RunHistory
Runhistory with additional data to be added to self
origin: DataOrigin
If set to ``INTERNAL`` or ``EXTERNAL_FULL`` the data will be
added to the internal data structure self._configid_to_inst_seed
and be available :meth:`through get_runs_for_config`.
"""
# Configurations might be already known, but by a different ID. This
# does not matter here because the add() method handles this
# correctly by assigning an ID to unknown configurations and re-using
# the ID
for key, value in runhistory.data.items():
config_id, instance_id, seed = key
cost, time, status, additional_info = value
config = runhistory.ids_config[config_id]
self.add(config=config,
cost=cost,
time=time,
status=status,
instance_id=instance_id,
seed=seed,
additional_info=additional_info,
origin=origin)
def __init__(self) -> None:
self.logger = PickableLoggerAdapter(name=self.__module__ + "." +
self.__class__.__name__)
class AbstractTAFunc(SerialRunner):
"""Baseclass to execute target algorithms which are python functions.
**Note:*** Do not use directly
Attributes
----------
memory_limit
use_pynisher
"""
def __init__(
self,
ta: typing.Callable,
stats: Stats,
run_obj: str = "quality",
memory_limit: typing.Optional[int] = None,
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = False,
use_pynisher: bool = True,
):
super().__init__(
ta=ta,
stats=stats,
run_obj=run_obj,
par_factor=par_factor,
cost_for_crash=cost_for_crash,
abort_on_first_run_crash=abort_on_first_run_crash,
)
"""
Abstract class for having a function as target algorithm
Parameters
----------
ta : callable
Function (target algorithm) to be optimized.
stats: Stats()
stats object to collect statistics about runtime and so on
run_obj: str
run objective of SMAC
memory_limit : int, optional
Memory limit (in MB) that will be applied to the target algorithm.
par_factor: int
penalization factor
cost_for_crash : float
cost that is used in case of crashed runs (including runs
that returned NaN or inf)
use_pynisher: bool
use pynisher to limit resources;
if disabled
* TA func can use as many resources
as it wants (time and memory) --- use with caution
* all runs will be returned as SUCCESS if returned value is not None
"""
self.ta = ta
self.stats = stats
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
signature = inspect.signature(ta).parameters
self._accepts_seed = 'seed' in signature.keys()
self._accepts_instance = 'instance' in signature.keys()
self._accepts_budget = 'budget' in signature.keys()
if not callable(ta):
raise TypeError('Argument `ta` must be a callable, but is %s' %
type(ta))
self._ta = typing.cast(typing.Callable, ta)
if memory_limit is not None:
memory_limit = int(math.ceil(memory_limit))
self.memory_limit = memory_limit
self.use_pynisher = use_pynisher
self.logger = PickableLoggerAdapter(self.__module__ + '.' +
self.__class__.__name__)
def run(
self,
config: Configuration,
instance: typing.Optional[str] = None,
cutoff: typing.Optional[float] = None,
seed: int = 12345,
budget: typing.Optional[float] = None,
instance_specific: str = "0"
) -> typing.Tuple[StatusType, float, float, typing.Dict]:
"""Runs target algorithm <self._ta> with configuration <config> for at
most <cutoff> seconds, allowing it to use at most <memory_limit> RAM.
Whether the target algorithm is called with the <instance> and
<seed> depends on the subclass implementing the actual call to
the target algorithm.
Parameters
----------
config : Configuration, dictionary (or similar)
Dictionary param -> value
instance : str, optional
Problem instance
cutoff : float, optional
Wallclock time limit of the target algorithm. If no value is
provided no limit will be enforced. It is casted to integer internally.
seed : int
Random seed
budget : float, optional
A positive, real-valued number representing an arbitrary limit to the target algorithm
Handled by the target algorithm internally
instance_specific: str
Instance specific information (e.g., domain file or solution)
Returns
-------
status: enum of StatusType (int)
{SUCCESS, TIMEOUT, CRASHED, ABORT}
cost: float
cost/regret/quality/runtime (float) (None, if not returned by TA)
runtime: float
runtime (None if not returned by TA)
additional_info: dict
all further additional run information
"""
obj_kwargs = {
} # type: typing.Dict[str, typing.Union[int, str, float, None]]
if self._accepts_seed:
obj_kwargs['seed'] = seed
if self._accepts_instance:
obj_kwargs['instance'] = instance
if self._accepts_budget:
obj_kwargs['budget'] = budget
if self.use_pynisher:
# walltime for pynisher has to be a rounded up integer
if cutoff is not None:
cutoff = int(math.ceil(cutoff))
if cutoff > MAX_CUTOFF:
raise ValueError(
"%d is outside the legal range of [0, 65535] "
"for cutoff (when using pynisher, due to OS limitations)"
% cutoff)
arguments = {
'logger': self.logger,
'wall_time_in_s': cutoff,
'mem_in_mb': self.memory_limit
}
# call ta
try:
obj = pynisher.enforce_limits(**arguments)(self._ta)
rval = self._call_ta(obj, config, obj_kwargs)
except Exception as e:
exception_traceback = traceback.format_exc()
error_message = repr(e)
additional_info = {
'traceback': exception_traceback,
'error': error_message
}
return StatusType.CRASHED, self.cost_for_crash, 0.0, additional_info
if isinstance(rval, tuple):
result = rval[0]
additional_run_info = rval[1]
else:
result = rval
additional_run_info = {}
# get status, cost, time
if obj.exit_status is pynisher.TimeoutException:
status = StatusType.TIMEOUT
cost = self.cost_for_crash
elif obj.exit_status is pynisher.MemorylimitException:
status = StatusType.MEMOUT
cost = self.cost_for_crash
elif obj.exit_status == 0 and result is not None:
status = StatusType.SUCCESS
cost = result
else:
status = StatusType.CRASHED
cost = self.cost_for_crash
runtime = float(obj.wall_clock_time)
else:
start_time = time.time()
# call ta
try:
rval = self._call_ta(self._ta, config, obj_kwargs)
if isinstance(rval, tuple):
result = rval[0]
additional_run_info = rval[1]
else:
result = rval
additional_run_info = {}
status = StatusType.SUCCESS
cost = result
except Exception as e:
self.logger.exception(e)
cost, result = self.cost_for_crash, self.cost_for_crash
status = StatusType.CRASHED
additional_run_info = {}
runtime = time.time() - start_time
if status == StatusType.SUCCESS and not isinstance(
result, (int, float)):
status = StatusType.CRASHED
cost = self.cost_for_crash
return status, cost, runtime, additional_run_info
def _call_ta(
self,
obj: typing.Callable,
config: Configuration,
obj_kwargs: typing.Dict[str, typing.Union[int, str, float, None]],
) -> typing.Union[float, typing.Tuple[float, typing.Dict]]:
raise NotImplementedError()