def __init__(self,
configspace: ConfigurationSpace,
types: np.ndarray,
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
hidden_dims: typing.List[int] = [50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 10000,
batch_size: int = 8,
var: bool = True,
**kwargs):
super().__init__(configspace, types, bounds, seed, **kwargs)
print("USE DNGO")
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 100
self.log_error = 1000
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.nn = None
self.blr = None
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(
self,
ta: typing.Union[typing.List[str], typing.Callable],
stats: Stats,
run_obj: str = "runtime",
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = True,
):
# The results is a FIFO structure, implemented via a list
# (because the Queue lock is not pickable). Finished runs are
# put in this list and collected via process_finished_runs
self.results = [] # type: typing.List[typing.Tuple[RunInfo, RunValue]]
# Below state the support for a Runner algorithm that
# implements a ta
self.ta = ta
self.stats = stats
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
self.logger = PickableLoggerAdapter(self.__module__ + '.' +
self.__class__.__name__)
self._supports_memory_limit = False
super().__init__()
def __init__(
self,
configspace: ConfigurationSpace,
types: typing.List[int],
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
instance_features: typing.Optional[np.ndarray] = None,
pca_components: typing.Optional[int] = 7,
) -> None:
self.configspace = configspace
self.seed = seed
self.instance_features = instance_features
self.pca_components = pca_components
if instance_features is not None:
self.n_feats = instance_features.shape[1]
else:
self.n_feats = 0
self.n_params = len(self.configspace.get_hyperparameters())
self.pca = PCA(n_components=self.pca_components)
self.scaler = MinMaxScaler()
self._apply_pca = False
# Never use a lower variance than this
self.var_threshold = VERY_SMALL_NUMBER
self.bounds = bounds
self.types = types
# Initial types array which is used to reset the type array at every call to train()
self._initial_types = copy.deepcopy(types)
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(
self,
configspace: ConfigurationSpace,
types: typing.List[int],
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
instance_features: typing.Optional[np.ndarray] = None,
pca_components: typing.Optional[int] = 7,
) -> None:
"""Constructor
Parameters
----------
configspace : ConfigurationSpace
Configuration space to tune for.
types : List[int]
Specifies the number of categorical values of an input dimension where
the i-th entry corresponds to the i-th input dimension. Let's say we
have 2 dimension where the first dimension consists of 3 different
categorical choices and the second dimension is continuous than we
have to pass [3, 0]. Note that we count starting from 0.
bounds : List[Tuple[float, float]]
bounds of input dimensions: (lower, uppper) for continuous dims; (n_cat, np.nan) for categorical dims
seed : int
The seed that is passed to the model library.
instance_features : np.ndarray (I, K)
Contains the K dimensional instance features
of the I different instances
pca_components : float
Number of components to keep when using PCA to reduce
dimensionality of instance features. Requires to
set n_feats (> pca_dims).
"""
self.configspace = configspace
self.seed = seed
self.instance_features = instance_features
self.pca_components = pca_components
if instance_features is not None:
self.n_feats = instance_features.shape[1]
else:
self.n_feats = 0
self.n_params = len(self.configspace.get_hyperparameters())
self.pca = PCA(n_components=self.pca_components)
self.scaler = MinMaxScaler()
self._apply_pca = False
# Never use a lower variance than this
self.var_threshold = VERY_SMALL_NUMBER
self.bounds = bounds
self.types = types
# Initial types array which is used to reset the type array at every call to train()
self._initial_types = copy.deepcopy(types)
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(
self,
ta: typing.Union[typing.List[str], typing.Callable],
stats: Stats,
run_obj: str = "runtime",
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = True,
):
"""
Attributes
----------
results
ta
stats
run_obj
par_factor
cost_for_crash
abort_first_run_crash
Parameters
----------
ta : typing.Union[typing.List[str], typing.Callable]
target algorithm
stats: Stats
stats object to collect statistics about runtime/additional info
run_obj: str
run objective of SMAC
par_factor: int
penalization factor
cost_for_crash : float
cost that is used in case of crashed runs (including runs
that returned NaN or inf)
abort_on_first_run_crash: bool
if true and first run crashes, raise FirstRunCrashedException
"""
# The results is a FIFO structure, implemented via a list
# (because the Queue lock is not pickable). Finished runs are
# put in this list and collected via process_finished_runs
self.results = [] # type: typing.List[typing.Tuple[RunInfo, RunValue]]
# Below state the support for a Runner algorithm that
# implements a ta
self.ta = ta
self.stats = stats
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
self.logger = PickableLoggerAdapter(self.__module__ + '.' +
self.__class__.__name__)
self._supports_memory_limit = False
super().__init__()
def __init__(self,
hidden_dims,
input_size,
feat_type=None,
var: bool = True,
max_cat: int = np.inf):
super(NeuralNet, self).__init__()
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
self.feat_type = feat_type
self.input_size = input_size
self.num_neurons = hidden_dims
self.activation = nn.Tanh
self.num_layer = len(hidden_dims)
self.max_cat = max_cat
if var:
self.n_output = 2
else:
self.n_output = 1
if np.sum(self.feat_type) == 0:
self.feat_type = None
if self.feat_type is not None:
self.logger.info("Use cat embedding")
assert len(self.feat_type) == self.input_size
emb = nn.ModuleList()
sz = int(0)
for f in self.feat_type:
if f == 0:
# In SMAC 0 encodes a numerical
emb.append(None)
sz += 1
else:
es = min(self.max_cat, int(f))
emb.append(nn.Embedding(int(f), es))
sz += es
assert int(sz) == sz
sz = int(sz)
num_neurons = [sz] + self.num_neurons
self.embedding = emb
else:
num_neurons = [self.input_size] + self.num_neurons
self.weights = nn.ModuleList()
self.acts = nn.ModuleList()
print(num_neurons)
for i in range(self.num_layer):
self.weights.append(nn.Linear(num_neurons[i], num_neurons[i + 1]))
self.acts.append(self.activation())
self.outlayer = nn.Linear(num_neurons[-1], self.n_output)
def __init__(self, model: AbstractEPM):
"""Constructor
Parameters
----------
model : AbstractEPM
Models the objective function.
"""
self.model = model
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(self, model: AbstractEPM):
"""Constructor
Parameters
----------
model : AbstractEPM
Models the objective function.
"""
self.model = model
self._required_updates = ('model', ) # type: Tuple[str, ...]
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(
self,
ta: Callable,
stats: Stats,
multi_objectives: List[str] = ["cost"],
run_obj: str = "quality",
memory_limit: Optional[int] = None,
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = False,
use_pynisher: bool = True,
):
super().__init__(
ta=ta,
stats=stats,
multi_objectives=multi_objectives,
run_obj=run_obj,
par_factor=par_factor,
cost_for_crash=cost_for_crash,
abort_on_first_run_crash=abort_on_first_run_crash,
)
self.ta = ta
self.stats = stats
self.multi_objectives = multi_objectives
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
signature = inspect.signature(ta).parameters
self._accepts_seed = "seed" in signature.keys()
self._accepts_instance = "instance" in signature.keys()
self._accepts_budget = "budget" in signature.keys()
if not callable(ta):
raise TypeError("Argument `ta` must be a callable, but is %s" % type(ta))
self._ta = cast(Callable, ta)
if memory_limit is not None:
memory_limit = int(math.ceil(memory_limit))
self.memory_limit = memory_limit
self.use_pynisher = use_pynisher
self.logger = PickableLoggerAdapter(
self.__module__ + "." + self.__class__.__name__
)
def __init__(
self,
aggregate_func: typing.Callable,
overwrite_existing_runs: bool = False,
file_system=LocalFS()) -> None:
"""Constructor
Parameters
----------
aggregate_func: callable
function to aggregate perf across instances
overwrite_existing_runs: bool
allows to overwrites old results if pairs of
algorithm-instance-seed were measured
multiple times
"""
self.file_system = file_system
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
# By having the data in a deterministic order we can do useful tests
# when we serialize the data and can assume it's still in the same
# order as it was added.
self.data = collections.OrderedDict(
) # type: typing.Dict[RunKey, RunValue]
# for fast access, we have also an unordered data structure
# to get all instance seed pairs of a configuration
self._configid_to_inst_seed = {} # type: typing.Dict[int, InstSeedKey]
self.config_ids = {} # type: typing.Dict[Configuration, int]
self.ids_config = {} # type: typing.Dict[int, Configuration]
self._n_id = 0
# Stores cost for each configuration ID
self.cost_per_config = {} # type: typing.Dict[int, float]
# runs_per_config maps the configuration ID to the number of runs for that configuration
# and is necessary for computing the moving average
self.runs_per_config = {} # type: typing.Dict[int, int]
# Store whether a datapoint is "external", which means it was read from
# a JSON file. Can be chosen to not be written to disk
self.external = {} # type: typing.Dict[RunKey, DataOrigin]
self.aggregate_func = aggregate_func
self.overwrite_existing_runs = overwrite_existing_runs
def __init__(self,
seed: int,
feat_types: typing.List[int] = None,
hidden_dims: typing.List[int] = [50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 10000,
batch_size: int = 8,
var: bool = True,
lognormal_nllh: bool = False,
var_bias_init: float = 1,
max_cat: int = np.inf,
learned_weight_init: bool = False,
optimization_algorithm: str = 'sgd',
**kwargs):
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 1000
self.log_error = iterations
self.seed = seed
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.lognormal_nllh = lognormal_nllh
self.var_bias_init = var_bias_init
self.max_cat = max_cat
self.learned_weight_init = learned_weight_init
self.optimization_algorithm = optimization_algorithm
self.feat_types = feat_types
if self.lognormal_nllh:
assert self.var, "Can't train with lognormal nllh if no var is selected"
self.model = None
def __init__(self,
configspace: ConfigurationSpace,
types: np.ndarray,
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
hidden_dims: typing.List[int] = [50, 50, 50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 20000,
batch_size: int = 16,
number_of_networks: int = 10,
var: bool = True,
train_with_lognormal_llh=False,
compute_mean_in_logspace=True,
**kwargs):
super().__init__(configspace, types, bounds, seed, **kwargs)
assert not (train_with_lognormal_llh and compute_mean_in_logspace)
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 1000
self.log_error = 5000
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.number_of_networks = number_of_networks
self.train_with_lognormal = train_with_lognormal_llh
self.compute_mean_in_logspace = compute_mean_in_logspace
self.nns = None
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
self.p0 = None
def __init__(
self,
overwrite_existing_runs: bool = False,
) -> None:
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
# By having the data in a deterministic order we can do useful tests
# when we serialize the data and can assume it's still in the same
# order as it was added.
self.data = collections.OrderedDict() # type: Dict[RunKey, RunValue]
# for fast access, we have also an unordered data structure
# to get all instance seed pairs of a configuration.
# This does not include capped runs.
self._configid_to_inst_seed_budget = (
{}) # type: Dict[int, Dict[InstSeedKey, List[float]]]
self.config_ids = {} # type: Dict[Configuration, int]
self.ids_config = {} # type: Dict[int, Configuration]
self._n_id = 0
# Stores cost for each configuration ID
self._cost_per_config = {} # type: Dict[int, np.ndarray]
# Stores min cost across all budgets for each configuration ID
self._min_cost_per_config = {} # type: Dict[int, np.ndarray]
# runs_per_config maps the configuration ID to the number of runs for that configuration
# and is necessary for computing the moving average
self.num_runs_per_config = {} # type: Dict[int, int]
# Store whether a datapoint is "external", which means it was read from
# a JSON file. Can be chosen to not be written to disk
self.external = {} # type: Dict[RunKey, DataOrigin]
self.overwrite_existing_runs = overwrite_existing_runs
self.num_obj = -1 # type: int
self.objective_bounds = [] # type: List[Tuple[float, float]]
def __init__(self) -> None:
self.logger = PickableLoggerAdapter(name=self.__module__ + "." +
self.__class__.__name__)
def __init__(self, file_system=LocalFS()):
self.file_system = file_system
self.logger = PickableLoggerAdapter(name=self.__module__ + "." +
self.__class__.__name__)
def __init__(self, model: AbstractEPM):
self.model = model
self._required_updates = ('model', ) # type: Tuple[str, ...]
self.logger = PickableLoggerAdapter(self.__module__ + "." + self.__class__.__name__)
def __init__(self,
configspace: ConfigurationSpace,
types: typing.List[int],
bounds: typing.List[typing.Tuple[float, float]],
seed: int,
hidden_dims: typing.List[int] = [50, 50, 50],
lr: float = 1e-3,
momentum: float = 0.999,
weight_decay: float = 1e-4,
iterations: int = 5000,
batch_size: int = 16,
number_of_networks: int = 5,
var: bool = True,
train_with_lognormal_llh=False,
compute_mean_in_logspace=False,
max_cat: int = np.inf,
ignore_cens: bool = False,
learned_weight_init: bool = False,
optimization_algorithm: str = 'sgd',
**kwargs):
super().__init__(configspace, types, bounds, seed, **kwargs)
#self.types[self.types == 0] = -1
self.types = [int(f) for f in self.types]
assert not (train_with_lognormal_llh and compute_mean_in_logspace)
if type(self.seed) != int:
self.seed = self.seed[0]
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.log_loss = 1000
self.log_error = 5000
self.var = var
self.hidden_dims = hidden_dims
self.lr = lr
self.momentum = momentum
self.iterations = iterations
self.weight_decay = weight_decay
self.batch_size = batch_size
self.number_of_networks = number_of_networks
self.train_with_lognormal = train_with_lognormal_llh
self.compute_mean_in_logspace = compute_mean_in_logspace
self.max_cat = max_cat
self.ignore_cens = ignore_cens
self.learned_weight_init = learned_weight_init
self.optimization_algorithm = optimization_algorithm
self._my = None
self._sy = None
# Quick check, should not take too long
a = np.random.normal(42, 23, 1000)
m1, v1 = (np.mean(a), np.var(a))
a = self._preprocess_y(a)
m2, v2 = self._postprocess_mv(np.mean(a), np.var(a))
assert np.abs(m1 - m2) < 1e-3, (m1, m2)
assert np.abs(v1 - v2) < 1e-3, (v1, v2)
self._my = None
self._sy = None
self.nns = None
self.logger = PickableLoggerAdapter(self.__module__ + "." +
self.__class__.__name__)
def __init__(
self,
ta: typing.Callable,
stats: Stats,
run_obj: str = "quality",
memory_limit: typing.Optional[int] = None,
par_factor: int = 1,
cost_for_crash: float = float(MAXINT),
abort_on_first_run_crash: bool = False,
use_pynisher: bool = True,
):
super().__init__(
ta=ta,
stats=stats,
run_obj=run_obj,
par_factor=par_factor,
cost_for_crash=cost_for_crash,
abort_on_first_run_crash=abort_on_first_run_crash,
)
"""
Abstract class for having a function as target algorithm
Parameters
----------
ta : callable
Function (target algorithm) to be optimized.
stats: Stats()
stats object to collect statistics about runtime and so on
run_obj: str
run objective of SMAC
memory_limit : int, optional
Memory limit (in MB) that will be applied to the target algorithm.
par_factor: int
penalization factor
cost_for_crash : float
cost that is used in case of crashed runs (including runs
that returned NaN or inf)
use_pynisher: bool
use pynisher to limit resources;
if disabled
* TA func can use as many resources
as it wants (time and memory) --- use with caution
* all runs will be returned as SUCCESS if returned value is not None
"""
self.ta = ta
self.stats = stats
self.run_obj = run_obj
self.par_factor = par_factor
self.cost_for_crash = cost_for_crash
self.abort_on_first_run_crash = abort_on_first_run_crash
signature = inspect.signature(ta).parameters
self._accepts_seed = 'seed' in signature.keys()
self._accepts_instance = 'instance' in signature.keys()
self._accepts_budget = 'budget' in signature.keys()
if not callable(ta):
raise TypeError('Argument `ta` must be a callable, but is %s' %
type(ta))
self._ta = typing.cast(typing.Callable, ta)
if memory_limit is not None:
memory_limit = int(math.ceil(memory_limit))
self.memory_limit = memory_limit
self.use_pynisher = use_pynisher
self.logger = PickableLoggerAdapter(self.__module__ + '.' +
self.__class__.__name__)