def test_rng_helper_2():
from hpobench.util.rng_helper import get_rng
rng = get_rng(None, None)
assert isinstance(rng, np.random.RandomState)
old_rng = np.random.RandomState(123)
rng = get_rng(None, old_rng)
assert rng == old_rng
def objective_function_test(self,
configuration: Union[Dict, CS.Configuration],
fidelity: Union[Dict, None] = None,
rng: Union[np.random.RandomState, int,
None] = None,
**kwargs) -> Dict:
"""
Trains a XGBoost model with a given configuration on both the train
and validation data set and evaluates the model on the test data set.
Parameters
----------
configuration : Dict, CS.Configuration
Configuration for the XGBoost Model
fidelity: Dict, None
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
rng : np.random.RandomState, int, None,
Random seed for benchmark. By default the class level random seed.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : test loss
cost : time to train and evaluate the model
info : Dict
fidelity : used fidelities in this evaluation
"""
default_subsample = self.get_fidelity_space().get_hyperparameter(
'subsample').default_value
if fidelity['subsample'] != default_subsample:
raise NotImplementedError(
f'Test error can not be computed for subsample <= {default_subsample:d}'
)
self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng)
start = time.time()
# Impute potential nan values with the feature-
data = np.concatenate((self.X_train, self.X_valid))
targets = np.concatenate((self.y_train, self.y_valid))
model = self._get_pipeline(n_estimators=fidelity['n_estimators'],
**configuration)
model.fit(X=data, y=targets)
test_loss = 1 - self.accuracy_scorer(model, self.X_test, self.y_test)
cost = time.time() - start
return {
'function_value': test_loss,
'cost': cost,
'info': {
'fidelity': fidelity
}
}
def __init__(self,
dataset_name: str,
rng: Union[int, np.random.RandomState, None] = None):
"""
Parameters
----------
dataset_name : str
Must be one of [
"annthyroid", "arrhythmia", "breastw", "cardio", "ionosphere",
"mammography", "musk", "optdigits", "pendigits", "pima",
"satellite", "satimage-2", "thyroid", "vowels", "wbc"]
rng : int, np.random.RandomState, None
defines the random state
"""
super(OutlierDetectionDataManager, self).__init__()
if dataset_name not in ODDS_URL:
raise NotImplementedError()
self.dataset_name = dataset_name
self.rng = get_rng(rng=rng)
self._url_source = ODDS_URL[dataset_name]
self._save_to = hpobench.config_file.data_dir / dataset_name
self.create_save_directory(self._save_to)
filename = self.__load_data(filename=self.dataset_name)
self.dataset = OutlierDataset(filename,
rng=self.rng,
logger=self.logger)
def objective_function(self, configuration: Union[Dict, CS.Configuration],
fidelity: Union[Dict, None] = None,
rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict:
"""
MetaLearna trains an RL agent for 1 hour on the given training set and then tries to solve the sequences.
MetaLearna has only a maximum solving time per sequence of 60 seconds.
Parameters
----------
configuration : Dict, CS.Configuration
fidelity: Dict, None
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
rng : np.random.RandomState, int, None,
Random seed for benchmark. By default the class level random seed.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : sum of min distances
cost : time to train and evaluate the model
info : Dict
num_solved : int - number of solved sequences
sum_of_first_distances : metric describing quality of result
fidelity : the used fidelities in this evaluation
"""
self.rng = rng_helper.get_rng(rng, self.rng)
tmp_dir = Path(tempfile.mkdtemp(dir=self.config.cache_dir))
config, network_config, agent_config, env_config = self._setup(configuration)
start_time = time()
try:
train_info = self._train(budget=fidelity["cutoff_agent_per_sequence"],
tmp_dir=tmp_dir,
network_config=network_config,
agent_config=agent_config,
env_config=env_config)
validation_info = self._validate(evaluation_timeout=60,
restore_path=tmp_dir,
stop_learning=True,
restart_timeout=config["restart_timeout"],
network_config=network_config,
agent_config=agent_config,
env_config=env_config)
cost = time() - start_time
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
return {'function_value': validation_info["sum_of_min_distances"],
'cost': cost,
'info': {'train_info': train_info,
'validation_info': validation_info,
'fidelity': fidelity},
}
def objective_function(self,
configuration: Union[CS.Configuration, Dict],
fidelity: Union[CS.Configuration, Dict,
None] = None,
rng: Union[np.random.RandomState, int, None] = None,
**kwargs) -> Dict:
"""
Start the learna experiment. Dont train a RL agent. Just optimize on the sequences.
Parameters
----------
configuration : Dict, CS.Configuration
fidelity: Dict, None
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
rng : np.random.RandomState, int, None,
Random seed for benchmark. By default the class level random seed.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : sum of min distances
cost : time to train and evaluate the model
info : Dict
num_solved : int - number of soved sequences
sum_of_first_distances : metric describing quality of result
fidelity : the used fidelities in this evaluation
"""
self.rng = rng_helper.get_rng(rng, self_rng=self.rng)
config, network_config, agent_config, env_config = self._setup(
configuration)
start_time = time()
validation_info = self._validate(
evaluation_timeout=fidelity["cutoff_agent_per_sequence"],
restore_path=None,
stop_learning=False,
restart_timeout=config["restart_timeout"],
network_config=network_config,
agent_config=agent_config,
env_config=env_config)
cost = time() - start_time
return {
'function_value': validation_info["sum_of_min_distances"],
'cost': cost,
'info': {
'num_solved': validation_info["num_solved"],
'sum_of_first_distances':
validation_info["sum_of_first_distances"],
'fidelity': fidelity,
}
}
def _train_objective(self,
config: Dict,
fidelity: Dict,
shuffle: bool,
rng: Union[np.random.RandomState, int, None] = None,
evaluation: Union[str, None] = "valid"):
if rng is not None:
rng = get_rng(rng, self.rng)
# initializing model
model = self.init_model(config, fidelity, rng)
# preparing data
if eval == "valid":
train_X = self.train_X
train_y = self.train_y
train_idx = self.train_idx
else:
train_X = np.vstack((self.train_X, self.valid_X))
train_y = pd.concat((self.train_y, self.valid_y))
train_idx = np.arange(len(train_X))
# shuffling data
if shuffle:
train_idx = self.shuffle_data_idx(train_idx, rng)
train_X = train_X.iloc[train_idx]
train_y = train_y.iloc[train_idx]
# subsample here:
# application of the other fidelity to the dataset that the model interfaces
if self.lower_bound_train_size is None:
self.lower_bound_train_size = (
10 * self.n_classes) / self.train_X.shape[0]
self.lower_bound_train_size = np.max(
(1 / 512, self.lower_bound_train_size))
subsample = np.max(
(fidelity['subsample'], self.lower_bound_train_size))
train_idx = self.rng.choice(np.arange(len(train_X)),
size=int(subsample * len(train_X)))
# fitting the model with subsampled data
start = time.time()
model.fit(train_X[train_idx], train_y.iloc[train_idx])
model_fit_time = time.time() - start
# computing statistics on training data
scores = dict()
score_cost = dict()
for k, v in self.scorers.items():
scores[k] = 0.0
score_cost[k] = 0.0
if evaluation == "test":
_start = time.time()
scores[k] = v(model, train_X, train_y)
score_cost[k] = time.time() - _start
train_loss = 1 - scores["acc"]
return model, model_fit_time, train_loss, scores, score_cost
def __init__(self,
openml_task_id: int,
rng: Union[int, np.random.RandomState, None] = None):
super(OpenMLCrossvalidationDataManager, self).__init__()
self._save_to = hpobench.config_file.data_dir / 'OpenML'
self.task_id = openml_task_id
self.rng = get_rng(rng=rng)
self.name = None
self.variable_types = None
self.create_save_directory(self._save_to)
openml.config.apikey = '610344db6388d9ba34f6db45a3cf71de'
openml.config.set_cache_directory(str(self._save_to))
def __init__(self,
rng: Union[int, np.random.RandomState, None] = None,
defaults: Union[Dict, None] = None,
max_episodes: Union[int, None] = 3000):
"""
Base benchmark for "cartpole" benchmark. In this benchmark a PPO agent tries to solve the cartpole task.
Parameters
----------
rng : int,None,np.RandomState
RandomState for the experiment
defaults : dict, None
default configuration used for the PPO agent
max_episodes : int, None
limit of the length of a episode for the cartpole runner. Defaults to 3000
"""
logger.warning('This Benchmark is not deterministic.')
super(CartpoleBase, self).__init__()
self.rng = rng_helper.get_rng(rng=rng)
tf.random.set_random_seed(0)
np.random.seed(0)
self.env = OpenAIGym('CartPole-v0', visualize=False)
self.avg_n_episodes = 20
self.max_episodes = max_episodes
self.defaults = {
"n_units_1": 64,
"n_units_2": 64,
"batch_size": 64,
"learning_rate": 1e-3,
"discount": 0.99,
"likelihood_ratio_clipping": 0.2,
"activation_1": "tanh",
"activation_2": "tanh",
"optimizer_type": "adam",
"optimization_steps": 10,
"baseline_mode": "states",
"baseline_n_units_1": 64,
"baseline_n_units_2": 64,
"baseline_learning_rate": 1e-3,
"baseline_optimization_steps": 10,
"baseline_optimizer_type": "adam"
}
if defaults is not None:
self.defaults.update(defaults)
def __init__(self,
dataset_name: str,
rng: Union[np.random.RandomState, int, None] = None):
"""
Parameters
----------
dataset_name : str
rng : np.random.RandomState, int, None
"""
self.rng = rng_helper.get_rng(rng)
# Load dataset manager
self.dataset_name = dataset_name
self.datamanager = OutlierDetectionDataManager(dataset_name, self.rng)
super(ODTraditional, self).__init__(rng=self.rng)
def objective_function(self,
configuration: Union[CS.Configuration, Dict],
fidelity: Union[CS.Configuration, Dict,
None] = None,
rng: Union[np.random.RandomState, int, None] = None,
**kwargs) -> Dict:
"""
Query the NAS-benchmark using a given configuration and a epoch (=budget).
Parameters
----------
configuration : Dict, CS.Configuration
fidelity: Dict, None
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
rng : np.random.RandomState, int, None
Random seed to use in the benchmark.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : validation error
cost : runtime
info : Dict
fidelity : used fidelities in this evaluation
"""
self.benchmark.reset_tracker()
self.rng = rng_helper.get_rng(rng, self_rng=self.rng)
# Returns (valid_accuracy: 0, runtime: 0) if it is invalid, e.g. config not valid or
# budget not in 4 12 36 108
data = self._query_benchmark(config=configuration,
budget=fidelity['budget'])
return {
'function_value': 1 - data['validation_accuracy'],
'cost': data['training_time'],
'info': {
'fidelity': fidelity,
'data': data
}
}
def objective_function_test(self,
configuration: Union[Dict, CS.Configuration],
fidelity: Union[Dict, None] = None,
rng: Union[np.random.RandomState, int,
None] = None,
**kwargs) -> Dict:
"""
Parameters
----------
configuration : Dict, CS.Configuration
fidelity: Dict, None
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
rng : np.random.RandomState, int, None
Random seed to use in the benchmark. To prevent overfitting on a single seed, it is
possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this
function. If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : validation loss
cost : time to train and evaluate the model
info : Dict
valid_rmse_per_run
runtime_per_run
fidelity : used fidelities in this evaluation
"""
self.rng = rng_helper.get_rng(rng, self_rng=self.rng)
default_fidelity = self.get_fidelity_space().get_default_configuration(
).get_dictionary()
assert fidelity == default_fidelity, 'Test function works only on the highest budget.'
result = self.benchmark.objective_function_test(configuration)
return {
'function_value': float(result[0]),
'cost': float(result[1]),
'info': {
'fidelity': fidelity
},
}
def __init__(self, rng: Union[int, np.random.RandomState, None] = None):
"""
Interface for benchmarks.
A benchmark consists of two building blocks, the target function and
the configuration space. Furthermore it can contain additional
benchmark-specific information such as the location and the function
value of the global optima.
New benchmarks should be derived from this base class or one of its
child classes.
Parameters
----------
rng: int, np.random.RandomState, None
The default random state for the benchmark. If type is int, a
np.random.RandomState with seed `rng` is created. If type is None,
create a new random state.
"""
self.rng = rng_helper.get_rng(rng=rng)
self.configuration_space = self.get_configuration_space()
self.fidelity_space = self.get_fidelity_space()
def __init__(self,
dataset_name: str,
rng: Union[np.random.RandomState, int, None] = None):
"""
Parameters
----------
dataset_name : str
Must be one of [
"annthyroid", "arrhythmia", "breastw", "cardio", "ionosphere",
"mammography", "musk", "optdigits", "pendigits", "pima",
"satellite", "satimage-2", "thyroid", "vowels", "wbc"]
rng : np.random.RandomState, int, None
"""
self.rng = rng_helper.get_rng(rng)
pl.seed_everything(self.rng.randint(0, 10000))
# Load datamanager
# It's important to call it before super
# as AbstractBenchmark samples configuration space in which
# the datamanager is needed
self.dataset_name = dataset_name
self.datamanager = OutlierDetectionDataManager(dataset_name, self.rng)
super(ODAutoencoder, self).__init__(rng=self.rng)
def objective_function(self, configuration: Union[CS.Configuration, Dict],
fidelity: Union[CS.Configuration, Dict, None] = None,
run_index: Union[int, Tuple, None] = (0, 1, 2),
rng: Union[np.random.RandomState, int, None] = None,
**kwargs) -> Dict:
"""
Query the NAS-benchmark using a given configuration and a epoch (=budget).
Parameters
----------
configuration : Dict, CS.Configuration
fidelity: Dict, None
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
run_index : int, Tuple, None
The nas benchmark has for each configuration-budget-pair results from 3 different runs.
- If multiple `run_id`s are given as Tuple, the benchmark returns the mean over the given runs.
- By default (no parameter is specified) all runs are used. A specific run can be chosen by setting the
`run_id` to a value from [0, 3]. While the performance is averaged across the `run_index`, the costs are
the sum of the runtime per `run_index`.
- When this value is explicitly set to `None`, the function will use a random seed.
rng : np.random.RandomState, int, None
Random seed to use in the benchmark.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : validation error
cost : runtime
info : Dict
fidelity : used fidelities in this evaluation
"""
self.rng = rng_helper.get_rng(rng, self_rng=self.rng)
if isinstance(run_index, int):
assert 0 <= run_index <= 2, f'run_index must be in [0, 2], not {run_index}'
run_index = (run_index, )
elif isinstance(run_index, (Tuple, List)):
assert 0 < len(run_index) <= 3, 'run_index must not be empty'
assert min(run_index) >= 0 and max(run_index) <= 2, \
f'all run_index values must be in [0, 2], but were {run_index}'
if len(set(run_index)) != len(run_index):
logger.debug('There are some values more than once in the run_index. We remove the redundant entries.')
run_index = tuple(set(run_index))
elif run_index is None:
logger.debug('The run index is explicitly set to None! A random seed will be selected.')
run_index = tuple(self.rng.choice((0, 1, 2), size=1))
else:
raise ValueError(f'run index must be one of Tuple or Int, but was {type(run_index)}')
self.benchmark.reset_tracker()
# Returns (valid_accuracy: 0, runtime: 0) if it is invalid, e.g. config not valid or
# budget not in 4 12 36 108
train_accuracies = []
valid_accuracies = []
test_accuracies = []
training_times = []
additional = {}
for run_id in run_index:
data = self._query_benchmark(config=configuration, budget=fidelity['budget'], run_index=run_id)
train_accuracies.append(data['train_accuracy'])
valid_accuracies.append(data['validation_accuracy'])
test_accuracies.append(data['test_accuracy'])
training_times.append(data['training_time'])
# Since those information are the same for all run ids, just store one of them.
additional = {'trainable_parameters': data['trainable_parameters'],
'module_operations': data['module_operations']}
return {'function_value': float(1 - np.mean(valid_accuracies)),
'cost': float(np.sum(training_times)),
'info': {'fidelity': fidelity,
'train_accuracies': train_accuracies,
'valid_accuracies': valid_accuracies,
'test_accuracies': test_accuracies,
'training_times': training_times,
'data': additional
}
}
def objective_function(self,
configuration: Union[CS.Configuration, Dict],
fidelity: Union[Dict, None] = None,
rng: Union[np.random.RandomState, int, None] = None,
data_seed: Union[List, Tuple, int,
None] = (777, 888, 999),
**kwargs) -> Dict:
"""
Objective function for the NASBench201 benchmark.
This functions sends a query to NASBench201 and evaluates the configuration.
As already explained in the class definition, different data sets are trained on different splits. For example
cifar10 is trained on the train and validation split and tested on the test split. Therefore, different entries
are returned from the NASBench201 result.
Overview of the used splits for training and testing and which are returned in the objective_function and
which in the objective_function_test.
|-------------------|-----------------------|---------------------------|
| | Returned by | Returned by |
| | objective_function | objective_function_test |
| Data set | train_* | eval_* |
|-------------------|-----------------------|---------------------------|
| 'cifar10-valid' | train | valid |
| 'cifar10' | train + valid | test |
| 'cifar100' | train | valid + test |
| 'ImageNet16-120' | train | valid + test |
|-------------------|-----------------------|---------------------------|
Legend:
* = [losses, acc1es, times]
Parameters
----------
configuration
fidelity: Dict, None
epoch: int - Values: [1, 200]
Number of epochs an architecture was trained.
Note: the number of epoch is 1 indexed! (Results after the first epoch: epoch = 1)
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
rng : np.random.RandomState, int, None
Random seed to use in the benchmark.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
data_seed : List, Tuple, None, int
The nasbench_201 benchmark include for each run 3 different seeds: 777, 888, 999.
The user can specify which seed to use. If more than one seed is given, the results are averaged
across the seeds but then the time needed for training is the sum of the costs per seed.
Note:
For some architectures (configurations) no run was available. We've set missing values to an
available value from another seed. Therefore, it is possible that run results are exactly the same for
different seeds.
kwargs
Returns
-------
Dict -
function_value : training precision
cost : time to train the network
info : Dict
train_precision : float
train_losses : float
train_cost : float
Time needed to train the network for 'epoch' many epochs. If more than one seed is given,
this field is the sum of the training time per network
eval_precision : float
eval_losses : float
eval_cost : float
Time needed to train the network for 'epoch many epochs plus the time to evaluate the network on the
evaluation split. If more than one seed is given, this field is the sum of the eval cost per network
fidelity : Dict
used fidelities in this evaluation
"""
# Check if the data set seeds are valid
assert isinstance(data_seed, List) or isinstance(data_seed, Tuple) or isinstance(data_seed, int), \
f'data seed has unknown data type {type(data_seed)}, but should be tuple or int (777,888,999)'
if isinstance(data_seed, List):
data_seed = tuple(data_seed)
if isinstance(data_seed, int):
data_seed = (data_seed, )
assert len(set(data_seed) - {777, 888, 999}) == 0,\
f'data seed can only contain the elements 777, 888, 999, but was {data_seed}'
self.rng = rng_helper.get_rng(rng)
structure = self.config_to_structure(configuration)
structure_str = structure.tostr()
epoch = fidelity['epoch'] - 1
train_accuracies = [
self.data[(seed, 'train_acc1es')][structure_str][epoch]
for seed in data_seed
]
train_losses = [
self.data[(seed, 'train_losses')][structure_str][epoch]
for seed in data_seed
]
train_times = [
np.sum(self.data[(seed, 'train_times')][structure_str][:epoch + 1])
for seed in data_seed
]
valid_accuracies = [
self.data[(seed, 'valid_acc1es')][structure_str][epoch]
for seed in data_seed
]
valid_losses = [
self.data[(seed, 'valid_losses')][structure_str][epoch]
for seed in data_seed
]
valid_times = [
np.sum(self.data[(seed, 'valid_times')][structure_str][:epoch + 1])
for seed in data_seed
]
# There is a single value for the eval data per seed. (only epoch 200)
test_accuracies = [
self.data[(seed, 'test_acc1es')][structure_str]
for seed in data_seed
]
test_losses = [
self.data[(seed, 'test_losses')][structure_str]
for seed in data_seed
]
test_times = [
np.sum(self.data[(seed, 'test_times')][structure_str])
for seed in data_seed
]
return {
'function_value': float(100 - np.mean(valid_accuracies)),
'cost': float(np.sum(valid_times) + np.sum(train_times)),
'info': {
'train_precision': float(100 - np.mean(train_accuracies)),
'train_losses': float(np.mean(train_losses)),
'train_cost': float(np.sum(train_times)),
'valid_precision': float(100 - np.mean(valid_accuracies)),
'valid_losses': float(np.mean(valid_losses)),
'valid_cost': float(np.sum(valid_times) + np.sum(train_times)),
'test_precision': float(100 - np.mean(test_accuracies)),
'test_losses': float(np.mean(test_losses)),
'test_cost':
float(np.sum(train_times)) + float(np.sum(test_times)),
'fidelity': fidelity
}
}
def run_experiment(out_path, on_travis):
settings = {
'min_budget': 1,
'max_budget':
9, # number of repetitions; this is the fidelity for this bench
'num_iterations': 10, # Set this to a low number for demonstration
'eta': 3,
'output_dir': Path(out_path)
}
if on_travis:
settings.update(get_travis_settings('bohb'))
b = Benchmark(rng=1)
b.get_configuration_space(seed=1)
settings.get('output_dir').mkdir(exist_ok=True)
cs = b.get_configuration_space()
seed = get_rng(rng=0)
run_id = 'BOHB_on_cartpole'
result_logger = hpres.json_result_logger(directory=str(
settings.get('output_dir')),
overwrite=True)
ns = hpns.NameServer(run_id=run_id,
host='localhost',
working_directory=str(settings.get('output_dir')))
ns_host, ns_port = ns.start()
worker = CustomWorker(seed=seed,
nameserver=ns_host,
nameserver_port=ns_port,
run_id=run_id,
max_budget=settings.get('max_budget'))
worker.run(background=True)
master = BOHB(configspace=cs,
run_id=run_id,
host=ns_host,
nameserver=ns_host,
nameserver_port=ns_port,
eta=settings.get('eta'),
min_budget=settings.get('min_budget'),
max_budget=settings.get('max_budget'),
result_logger=result_logger)
result = master.run(n_iterations=settings.get('num_iterations'))
master.shutdown(shutdown_workers=True)
ns.shutdown()
with open(settings.get('output_dir') / 'results.pkl', 'wb') as f:
pickle.dump(result, f)
id2config = result.get_id2config_mapping()
incumbent = result.get_incumbent_id()
inc_value = result.get_runs_by_id(incumbent)[-1]['loss']
inc_cfg = id2config[incumbent]['config']
logger.info(f'Inc Config:\n{inc_cfg}\n'
f'with Performance: {inc_value:.2f}')
if not on_travis:
benchmark = Benchmark(container_source='library://phmueller/automl')
incumbent_result = benchmark.objective_function_test(
configuration=inc_cfg, fidelity={"budget": settings['max_budget']})
print(incumbent_result)
def shuffle_data(self, rng=None):
""" Reshuffle the training data. If 'rng' is None, the training idx are shuffled according to the
class-random-state"""
random_state = rng_helper.get_rng(rng, self.rng)
random_state.shuffle(self.train_idx)
def objective_function(self,
configuration: Union[CS.Configuration, Dict],
fidelity: Union[CS.Configuration, Dict,
None] = None,
shuffle: bool = False,
rng: Union[np.random.RandomState, int, None] = None,
**kwargs) -> Dict:
"""
Trains a XGBoost model given a hyperparameter configuration and
evaluates the model on the validation set.
Parameters
----------
configuration : Dict, CS.Configuration
Configuration for the XGBoost model
fidelity: Dict, None
Fidelity parameters for the XGBoost model, check get_fidelity_space(). Uses default (max) value if None.
shuffle : bool
If ``True``, shuffle the training idx. If no parameter ``rng`` is given, use the class random state.
Defaults to ``False``.
rng : np.random.RandomState, int, None,
Random seed for benchmark. By default the class level random seed.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : validation loss
cost : time to train and evaluate the model
info : Dict
train_loss : trainings loss
fidelity : used fidelities in this evaluation
"""
self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng)
if shuffle:
self.shuffle_data(self.rng)
start = time.time()
if self.lower_bound_train_size > fidelity['dataset_fraction']:
train_data_fraction = self.lower_bound_train_size
logger.warning(
f'The given data set fraction is lower than the lower bound (10 * number of classes.) '
f'Increase the fidelity from {fidelity["dataset_fraction"]:.8f} to '
f'{self.lower_bound_train_size:.8f}')
else:
train_data_fraction = fidelity['dataset_fraction']
train_idx = self.train_idx[:int(
len(self.train_idx) * train_data_fraction)]
model = self._get_pipeline(n_estimators=fidelity["n_estimators"],
**configuration)
model.fit(X=self.x_train[train_idx], y=self.y_train[train_idx])
train_loss = 1 - self.accuracy_scorer(model, self.x_train[train_idx],
self.y_train[train_idx])
val_loss = 1 - self.accuracy_scorer(model, self.x_valid, self.y_valid)
cost = time.time() - start
return {
'function_value': float(val_loss),
'cost': cost,
'info': {
'train_loss': float(train_loss),
'fidelity': fidelity
}
}
def objective_function(self,
configuration: Union[CS.Configuration, Dict],
fidelity: Union[Dict, CS.Configuration,
None] = None,
rng: Union[np.random.RandomState, int, None] = None,
data_seed: Union[List, Tuple, int,
None] = (777, 888, 999),
**kwargs) -> Dict:
"""
Objective function for the NASBench201 benchmark.
This functions sends a query to NASBench201 and evaluates the configuration.
As already explained in the class definition, different data sets are trained on different splits.
The table above gives a detailed summary over the available splits, epochs, and which identifier are used per
dataset.
Parameters
----------
configuration
fidelity: Dict, None
epoch: int - Values: [1, 200]
Number of epochs an architecture was trained.
Note: the number of epoch is 1 indexed! (Results after the first epoch: epoch = 1)
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
rng : np.random.RandomState, int, None
Random seed to use in the benchmark.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
data_seed : List, Tuple, None, int
The nasbench_201 benchmark include for each run 3 different seeds: 777, 888, 999.
The user can specify which seed to use. If more than one seed is given, the results are averaged
across the seeds but then the training time is the sum of the costs per seed.
When this value is explicitly set to `None`, the function will chose randomly one out of [777, 888, 999].
Note:
For some architectures (configurations) no run was available. We've set missing values to an
available value from another seed. Therefore, it is possible that run results are exactly the same for
different seeds.
kwargs
Returns
-------
Dict -
function_value : training precision
cost : time to train the network
info : Dict
train_precision : float
train_losses : float
train_cost : float
Time needed to train the network for 'epoch' many epochs. If more than one seed is given,
this field is the sum of the training time per network
eval_precision : float
eval_losses : float
eval_cost : float
Time needed to train the network for 'epoch many epochs plus the time to evaluate the network on the
evaluation split. If more than one seed is given, this field is the sum of the eval cost per network
fidelity : Dict
used fidelities in this evaluation
"""
self.rng = rng_helper.get_rng(rng)
if isinstance(data_seed, (List, Tuple)):
assert len(data_seed) != 0, 'data_seed must not be empty'
if len(set(data_seed)) != len(data_seed):
logger.debug(
'There are some values more than once in the run_index. We remove the redundant entries.'
)
data_seed = tuple(set(data_seed))
elif isinstance(data_seed, int):
data_seed = (data_seed, )
elif data_seed is None:
logger.debug(
'The data seed is explicitly set to None! A random seed will be selected.'
)
data_seed = tuple(self.rng.choice((777, 888, 999), size=1))
# Check if the data set seeds are valid
else:
raise ValueError(
f'data seed has unknown data type {type(data_seed)}, '
f'but should be tuple or int (777,888,999)')
assert len(set(data_seed) - {777, 888, 999}) == 0,\
f'data seed can only contain the elements 777, 888, 999, but was {data_seed}'
structure = self.config_to_structure(configuration)
structure_str = structure.tostr()
epoch = fidelity['epoch'] - 1
data_seed = [str(seed) for seed in data_seed]
valid_key, test_key = self.dataset_mapping(self.dataset)
train_accuracies = [
self.data[seed][structure_str]['train_acc1es'][f'{epoch}']
for seed in data_seed
]
train_losses = [
self.data[seed][structure_str]['train_losses'][f'{epoch}']
for seed in data_seed
]
train_times = [
np.sum((self.data[seed][structure_str]['train_times'][f'{e}'])
for e in range(1, epoch + 1)) for seed in data_seed
]
valid_accuracies = [
self.data[seed][structure_str]['eval_acc1es']
[f'{valid_key}@{epoch}'] for seed in data_seed
]
valid_losses = [
self.data[seed][structure_str]['eval_losses']
[f'{valid_key}@{epoch}'] for seed in data_seed
]
valid_times = [
np.sum((self.data[seed][structure_str]['eval_times']
[f'{valid_key}@{e}']) for e in range(1, epoch + 1))
for seed in data_seed
]
# There is a single value for the eval data per seed. (only epoch 200)
test_accuracies = [
self.data[seed][structure_str]['eval_acc1es'][f'{valid_key}@{199}']
for seed in data_seed
]
test_losses = [
self.data[seed][structure_str]['eval_losses'][f'{valid_key}@{199}']
for seed in data_seed
]
test_times = [
np.sum((self.data[seed][structure_str]['eval_times']
[f'{test_key}@{199}']) for e in range(1, epoch + 1))
for seed in data_seed
]
return {
'function_value': float(100 - np.mean(valid_accuracies)),
'cost': float(np.sum(valid_times) + np.sum(train_times)),
'info': {
'train_precision': float(100 - np.mean(train_accuracies)),
'train_losses': float(np.mean(train_losses)),
'train_cost': float(np.sum(train_times)),
'valid_precision': float(100 - np.mean(valid_accuracies)),
'valid_losses': float(np.mean(valid_losses)),
'valid_cost': float(np.sum(valid_times) + np.sum(train_times)),
'test_precision': float(100 - np.mean(test_accuracies)),
'test_losses': float(np.mean(test_losses)),
'test_cost':
float(np.sum(train_times)) + float(np.sum(test_times)),
'fidelity': fidelity
}
}
def objective_function(self,
configuration: Union[CS.Configuration, Dict],
fidelity: Union[Dict, CS.Configuration,
None] = None,
rng: Union[np.random.RandomState, int, None] = None,
**kwargs) -> Dict:
"""
Trains a bayesian neural network with 3 layers on the defined data set and evaluates the trained model on
the validation split.
Parameters
----------
configuration : Dict, CS.Configuration
Configuration for the pyBNN model
fidelity: Dict, None
budget : int [500 - 10000]
number of epochs to train the model
Fidelity parameters for the pyBNN model, check get_fidelity_space(). Uses default (max) value if None.
rng : np.random.RandomState, int, None,
Random seed for benchmark. By default the class level random seed.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : validation loss
cost : time to train and evaluate the model
info : Dict
fidelity : used fidelities in this evaluation
"""
start = time.time()
self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng)
np.random.seed(self.rng.randint(1, 10000))
# See comment in objective function test
burn_in_steps = max(1,
int(configuration['burn_in'] * fidelity['budget']))
net = partial(_get_net,
n_units_1=configuration['n_units_1'],
n_units_2=configuration['n_units_2'])
model = BayesianNeuralNetwork(sampling_method="sghmc",
get_net=net,
l_rate=configuration['l_rate'],
mdecay=configuration['mdecay'],
burn_in=burn_in_steps,
n_iters=fidelity['budget'],
precondition=True,
normalize_input=True,
normalize_output=True,
rng=self.rng)
model.train(self.train,
self.train_targets,
valid=self.valid,
valid_targets=self.valid_targets,
valid_after_n_steps=100)
mean_pred, var_pred = model.predict(self.valid)
# Negative log-likelihood
valid_loss = self._neg_log_likelihood(self.valid_targets, mean_pred,
var_pred)
cost = time.time() - start
return {
'function_value': float(valid_loss),
'cost': cost,
'info': {
'fidelity': fidelity
}
}
def objective_function_test(self,
configuration: Union[Dict, CS.Configuration],
fidelity: Union[Dict, None] = None,
shuffle: bool = False,
rng: Union[np.random.RandomState, int,
None] = None,
**kwargs) -> Dict:
"""
Trains a SVM model with a given configuration on both the X_train
and validation data set and evaluates the model on the X_test data set.
Parameters
----------
configuration : Dict, CS.Configuration
Configuration for the SVM Model
fidelity: Dict, None
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
rng : np.random.RandomState, int, None,
Random seed for benchmark. By default the class level random seed.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : X_test loss
cost : time to X_train and evaluate the model
info : Dict
train_valid_loss: Loss on the train+valid data set
fidelity : used fidelities in this evaluation
"""
assert np.isclose(fidelity['dataset_fraction'], 1), \
f'Data set fraction must be 1 but was {fidelity["dataset_fraction"]}'
self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng)
start_time = time.time()
# Concatenate training and validation dataset
if isinstance(self.X_train, sparse.csr.csr_matrix) or isinstance(
self.X_valid, sparse.csr.csr_matrix):
data = sparse.vstack((self.X_train, self.X_valid))
else:
data = np.concatenate((self.X_train, self.X_valid))
targets = np.concatenate((self.y_train, self.y_valid))
# Transform hyperparameters to linear scale
hp_c = np.exp(float(configuration['C']))
hp_gamma = np.exp(float(configuration['gamma']))
model = self.get_pipeline(hp_c, hp_gamma)
model.fit(data, targets)
# Compute validation error
train_valid_loss = 1 - self.accuracy_scorer(model, data, targets)
# Compute test error
test_loss = 1 - self.accuracy_scorer(model, self.X_test, self.y_test)
cost = time.time() - start_time
return {
'function_value': test_loss,
"cost": cost,
'info': {
'train_valid_loss': train_valid_loss,
'fidelity': fidelity
}
}
def objective_function(self,
configuration: Union[Dict, CS.Configuration],
fidelity: Union[Dict, None] = None,
shuffle: bool = False,
rng: Union[np.random.RandomState, int, None] = None,
**kwargs) -> Dict:
"""
Trains a SVM model given a hyperparameter configuration and
evaluates the model on the validation set.
Parameters
----------
configuration : Dict, CS.Configuration
Configuration for the SVM model
fidelity: Dict, None
Fidelity parameters for the SVM model, check get_fidelity_space(). Uses default (max) value if None.
shuffle : bool
If ``True``, shuffle the training idx. If no parameter ``rng`` is given, use the class random state.
Defaults to ``False``.
rng : np.random.RandomState, int, None,
Random seed for benchmark. By default the class level random seed.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : validation loss
cost : time to train and evaluate the model
info : Dict
train_loss : training loss
fidelity : used fidelities in this evaluation
"""
start_time = time.time()
self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng)
if shuffle:
self.shuffle_data(self.rng)
# Split of dataset subset
if self.lower_bound_train_size > fidelity['dataset_fraction']:
train_size = self.lower_bound_train_size
logger.warning(
f'The given data set fraction is lower than the lower bound (10 * number of classes.) '
f'Increase the fidelity from {fidelity["dataset_fraction"]:.2f} to '
f'{self.lower_bound_train_size:.2f}')
else:
train_size = fidelity['dataset_fraction']
train_size = int(train_size * len(self.train_idx))
train_idx = self.train_idx[:train_size]
# Transform hyperparameters to linear scale
hp_c = np.exp(float(configuration['C']))
hp_gamma = np.exp(float(configuration['gamma']))
# Train support vector machine
model = self.get_pipeline(hp_c, hp_gamma)
model.fit(self.X_train[train_idx], self.y_train[train_idx])
# Compute validation error
train_loss = 1 - self.accuracy_scorer(model, self.X_train[train_idx],
self.y_train[train_idx])
val_loss = 1 - self.accuracy_scorer(model, self.X_valid, self.y_valid)
cost = time.time() - start_time
return {
'function_value': val_loss,
"cost": cost,
'info': {
'train_loss': train_loss,
'fidelity': fidelity
}
}
def objective_function(self,
configuration: Union[CS.Configuration, Dict],
fidelity: Union[CS.Configuration, Dict,
None] = None,
run_index: Union[int, Tuple, None] = (0, 1, 2, 3),
rng: Union[np.random.RandomState, int, None] = None,
**kwargs) -> Dict:
"""
Query the NAS-benchmark using a given configuration and a epoch (=budget).
Parameters
----------
configuration : Dict, CS.Configuration
fidelity: Dict, None
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
run_index : int, Tuple, None
The nas benchmark has for each configuration-budget-pair results from 4 different runs.
If multiple `run_id`s are given, the benchmark returns the mean over the given runs.
By default (no parameter is specified) all runs are used. A specific run can be chosen by setting the
`run_id` to a value from [0, 3].
When this value is explicitly set to `None`, the function will use a random seed.
rng : np.random.RandomState, int, None
Random seed to use in the benchmark. To prevent overfitting on a single seed, it is
possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this
function. If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : validation loss
cost : time to train and evaluate the model
info : Dict with valid_rmse_per_run, runtime_per_run
info : Dict
valid_rmse_per_run
runtime_per_run
fidelity : used fidelities in this evaluation
"""
self.rng = rng_helper.get_rng(rng)
if isinstance(run_index, int):
assert 0 <= run_index <= 3, f'run_index must be in [0, 3], not {run_index}'
run_index = (run_index, )
elif isinstance(run_index, (Tuple, List)):
assert len(run_index) != 0, 'run_index must not be empty'
if len(set(run_index)) != len(run_index):
logger.debug(
'There are some values more than once in the run_index. We remove the redundant entries.'
)
run_index = tuple(set(run_index))
assert min(run_index) >= 0 and max(run_index) <= 3, \
f'all run_index values must be in [0, 3], but were {run_index}'
elif run_index is None:
logger.debug(
'The run index is explicitly set to None! A random seed will be selected.'
)
run_index = tuple(self.rng.choice((0, 1, 2, 3), size=1))
else:
raise ValueError(
f'run index must be one of Tuple or Int, but was {type(run_index)}'
)
self._reset_tracker()
valid_rmse_list, runtime_list = [], []
for run_id in run_index:
valid_rmse, runtime = self.benchmark.objective_function_deterministic(
config=configuration, budget=fidelity["budget"], index=run_id)
valid_rmse_list.append(float(valid_rmse))
runtime_list.append(float(runtime))
valid_rmse = sum(valid_rmse_list) / len(valid_rmse_list)
runtime = sum(runtime_list)
return {
'function_value': float(valid_rmse),
'cost': float(runtime),
'info': {
'valid_rmse_per_run': valid_rmse_list,
'runtime_per_run': runtime_list,
'fidelity': fidelity
},
}
def objective_function_test(self,
configuration: Union[Dict, CS.Configuration],
fidelity: Union[Dict, CS.Configuration,
None] = None,
rng: Union[np.random.RandomState, int,
None] = None,
**kwargs) -> Dict:
"""
Trains a bayesian neural network with 3 layers on the train and valid data split and evaluates it on the test
split.
Parameters
----------
configuration : Dict, CS.Configuration
Configuration for the pyBNN model
fidelity: Dict, None
budget : int [500 - 10000]
number of epochs to train the model
Fidelity parameters for the pyBNN model, check get_fidelity_space(). Uses default (max) value if None.
Note: The fidelity should be here the max budget (= 10000). By leaving this field empty, the maximum budget
will be used by default.
rng : np.random.RandomState, int, None,
Random seed for benchmark. By default the class level random seed.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : validation loss
cost : time to train and evaluate the model
info : Dict
fidelity : used fidelities in this evaluation
"""
start = time.time()
self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng)
np.random.seed(self.rng.randint(1, 10000))
# `burn_in_steps` must be at least 1, otherwise, theano will raise an RuntimeError. (Actually, the definition of
# the config space allows as lower limit a zero. In this case, set the number of steps to 1.)
burn_in_steps = max(1,
int(configuration['burn_in'] * fidelity['budget']))
net = partial(_get_net,
n_units_1=configuration['n_units_1'],
n_units_2=configuration['n_units_2'])
model = BayesianNeuralNetwork(sampling_method="sghmc",
get_net=net,
l_rate=configuration['l_rate'],
mdecay=configuration['mdecay'],
burn_in=burn_in_steps,
n_iters=fidelity['budget'],
precondition=True,
normalize_input=True,
normalize_output=True,
rng=self.rng)
train = np.concatenate((self.train, self.valid))
train_targets = np.concatenate(
(self.train_targets, self.valid_targets))
model.train(train, train_targets)
mean_pred, var_pred = model.predict(self.test)
test_loss = self._neg_log_likelihood(self.test_targets, mean_pred,
var_pred)
cost = time.time() - start
return {
'function_value': float(test_loss),
'cost': cost,
'info': {
'fidelity': fidelity
}
}
def objective_function(self,
configuration: Union[Dict, CS.Configuration],
fidelity: Union[Dict, CS.Configuration,
None] = None,
rng: Union[np.random.RandomState, int, None] = None,
**kwargs) -> Dict:
"""
Trains a Tensorforce RL agent on the cartpole experiment. This benchmark was used in the experiments for the
BOHB-paper (see references). A more detailed explanations can be found there.
The budget describes how often the agent is trained on the experiment.
It returns the average number of the length of episodes.
Parameters
----------
configuration : Dict, CS.Configuration
fidelity: Dict, None
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
rng : np.random.RandomState, int, None
Random seed to use in the benchmark. To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : average episode length
cost : time to run all agents
info : Dict
max_episodes : the maximum length of an episode
budget : number of agents used
all_runs : the episode length of all runs of all agents
fidelity : the used fidelities in this evaluation
"""
self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng)
tf.random.set_random_seed(self.rng.randint(1, 100000))
np.random.seed(self.rng.randint(1, 100000))
# fill in missing entries with default values for 'incomplete/reduced' configspaces
new_config = self.defaults
new_config.update(configuration)
configuration = new_config
start_time = time.time()
network_spec = [{
'type': 'dense',
'size': configuration["n_units_1"],
'activation': configuration['activation_1']
}, {
'type': 'dense',
'size': configuration["n_units_2"],
'activation': configuration['activation_2']
}]
converged_episodes = []
for _ in range(fidelity["budget"]):
agent = PPOAgent(
states=self.env.states,
actions=self.env.actions,
network=network_spec,
update_mode={
'unit': 'episodes',
'batch_size': configuration["batch_size"]
},
step_optimizer={
'type': configuration["optimizer_type"],
'learning_rate': configuration["learning_rate"]
},
optimization_steps=configuration["optimization_steps"],
discount=configuration["discount"],
baseline_mode=configuration["baseline_mode"],
baseline={
"type":
"mlp",
"sizes": [
configuration["baseline_n_units_1"],
configuration["baseline_n_units_2"]
]
},
baseline_optimizer={
"type": "multi_step",
"optimizer": {
"type": configuration["baseline_optimizer_type"],
"learning_rate":
configuration["baseline_learning_rate"]
},
"num_steps": configuration["baseline_optimization_steps"]
},
likelihood_ratio_clipping=configuration[
"likelihood_ratio_clipping"],
entropy_regularization=configuration["entropy_regularization"],
)
def episode_finished(record):
# Check if we have converged
return np.mean(
record.episode_rewards[-self.avg_n_episodes:]) != 200
runner = Runner(agent=agent, environment=self.env)
runner.run(episodes=self.max_episodes,
max_episode_timesteps=200,
episode_finished=episode_finished)
converged_episodes.append(len(runner.episode_rewards))
cost = time.time() - start_time
return {
'function_value': np.mean(converged_episodes),
'cost': cost,
'info': {
'max_episodes': self.max_episodes,
'all_runs': converged_episodes,
'fidelity': fidelity
}
}
def objective_function(self,
configuration: Union[Dict, CS.Configuration],
fidelity: Union[Dict, None] = None,
shuffle: bool = False,
rng: Union[np.random.RandomState, int, None] = None,
**kwargs) -> Dict:
"""
Trains a XGBoost model given a hyperparameter configuration and
evaluates the model on the validation set.
Parameters
----------
configuration : Dict, CS.Configuration
Configuration for the XGBoost model
fidelity: Dict, None
Fidelity parameters for the XGBoost model, check get_fidelity_space(). Uses default (max) value if None.
shuffle : bool
If ``True``, shuffle the training idx. If no parameter ``rng`` is given, use the class random state.
Defaults to ``False``.
rng : np.random.RandomState, int, None,
Random seed for benchmark. By default the class level random seed.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : validation loss
cost : time to train and evaluate the model
info : Dict
train_loss : trainings loss
fidelity : used fidelities in this evaluation
"""
self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng)
if shuffle:
self.shuffle_data(self.rng)
start = time.time()
train_idx = self.train_idx[:int(
len(self.train_idx) * fidelity["subsample"])]
model = self._get_pipeline(n_estimators=fidelity["n_estimators"],
**configuration)
model.fit(X=self.X_train[train_idx], y=self.y_train[train_idx])
train_loss = 1 - self.accuracy_scorer(model, self.X_train[train_idx],
self.y_train[train_idx])
val_loss = 1 - self.accuracy_scorer(model, self.X_valid, self.y_valid)
cost = time.time() - start
return {
'function_value': val_loss,
'cost': cost,
'info': {
'train_loss': train_loss,
'fidelity': fidelity
}
}
def objective_function(self,
configuration: Union[CS.Configuration, Dict],
fidelity: Union[CS.Configuration, Dict,
None] = None,
run_index: Union[int, Tuple, List,
None] = (0, 1, 2),
rng: Union[np.random.RandomState, int, None] = None,
**kwargs) -> Dict:
"""
Query the NAS1shot1-benchmark using a given configuration and an epoch (=budget).
Only data for the budgets 4, 12, 36, 108 are available.
Parameters
----------
configuration : Dict, CS.Configuration
fidelity: Dict, None
Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None.
run_index : int, Tuple, None
The nas benchmark has for each configuration-budget-pair results from 3 different runs.
- If multiple `run_id`s are given as Tuple/List, the benchmark returns the mean over the given runs.
- By default (no parameter is specified) all runs are used. A specific run can be chosen by setting the
`run_id` to a value from [0, 3]. While the performance is averaged across the `run_index`, the costs are
the sum of the runtime per `run_index`.
- When this value is explicitly set to `None`, the function will use a random seed.
rng : np.random.RandomState, int, None
Random seed to use in the benchmark.
To prevent overfitting on a single seed, it is possible to pass a
parameter ``rng`` as 'int' or 'np.random.RandomState' to this function.
If this parameter is not given, the default random state is used.
kwargs
Returns
-------
Dict -
function_value : validation error
cost : runtime
info : Dict
train_accuracies
test_accuracies
valid_accuracies
training_times
fidelity : used fidelities in this evaluation
data : additional data such as trainable parameters and used operations
"""
self.rng = rng_helper.get_rng(rng, self_rng=self.rng)
run_index = self._check_run_index(run_index)
configuration = self._parse_configuration(configuration)
train_accuracies = []
valid_accuracies = []
test_accuracies = []
training_times = []
additional = {}
failure = False
for run_id in run_index:
data = self._query_benchmark(config=configuration,
fidelity=fidelity,
run_index=run_id)
train_accuracies.append(data['train_accuracy'])
valid_accuracies.append(data['validation_accuracy'])
test_accuracies.append(data['test_accuracy'])
training_times.append(data['training_time'])
# Since those information are the same for all run ids, just store one of them.
additional = {
'trainable_parameters': data['trainable_parameters'],
'module_operations': data['module_operations']
}
failure = failure or ('info' in data and data['info'] == 'failure')
return {
'function_value': float(1 - np.mean(valid_accuracies)),
'cost': float(np.sum(training_times)),
'info': {
'fidelity': fidelity,
'train_accuracies': train_accuracies,
'valid_accuracies': valid_accuracies,
'test_accuracies': test_accuracies,
'training_times': training_times,
'data': additional,
'failure': 'False' if not failure else 'True'
}
}
