def test_rng_helper_2(): from hpobench.util.rng_helper import get_rng rng = get_rng(None, None) assert isinstance(rng, np.random.RandomState) old_rng = np.random.RandomState(123) rng = get_rng(None, old_rng) assert rng == old_rng
def objective_function_test(self, configuration: Union[Dict, CS.Configuration], fidelity: Union[Dict, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Trains a XGBoost model with a given configuration on both the train and validation data set and evaluates the model on the test data set. Parameters ---------- configuration : Dict, CS.Configuration Configuration for the XGBoost Model fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None, Random seed for benchmark. By default the class level random seed. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : test loss cost : time to train and evaluate the model info : Dict fidelity : used fidelities in this evaluation """ default_subsample = self.get_fidelity_space().get_hyperparameter( 'subsample').default_value if fidelity['subsample'] != default_subsample: raise NotImplementedError( f'Test error can not be computed for subsample <= {default_subsample:d}' ) self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng) start = time.time() # Impute potential nan values with the feature- data = np.concatenate((self.X_train, self.X_valid)) targets = np.concatenate((self.y_train, self.y_valid)) model = self._get_pipeline(n_estimators=fidelity['n_estimators'], **configuration) model.fit(X=data, y=targets) test_loss = 1 - self.accuracy_scorer(model, self.X_test, self.y_test) cost = time.time() - start return { 'function_value': test_loss, 'cost': cost, 'info': { 'fidelity': fidelity } }
def __init__(self, dataset_name: str, rng: Union[int, np.random.RandomState, None] = None): """ Parameters ---------- dataset_name : str Must be one of [ "annthyroid", "arrhythmia", "breastw", "cardio", "ionosphere", "mammography", "musk", "optdigits", "pendigits", "pima", "satellite", "satimage-2", "thyroid", "vowels", "wbc"] rng : int, np.random.RandomState, None defines the random state """ super(OutlierDetectionDataManager, self).__init__() if dataset_name not in ODDS_URL: raise NotImplementedError() self.dataset_name = dataset_name self.rng = get_rng(rng=rng) self._url_source = ODDS_URL[dataset_name] self._save_to = hpobench.config_file.data_dir / dataset_name self.create_save_directory(self._save_to) filename = self.__load_data(filename=self.dataset_name) self.dataset = OutlierDataset(filename, rng=self.rng, logger=self.logger)
def objective_function(self, configuration: Union[Dict, CS.Configuration], fidelity: Union[Dict, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ MetaLearna trains an RL agent for 1 hour on the given training set and then tries to solve the sequences. MetaLearna has only a maximum solving time per sequence of 60 seconds. Parameters ---------- configuration : Dict, CS.Configuration fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None, Random seed for benchmark. By default the class level random seed. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : sum of min distances cost : time to train and evaluate the model info : Dict num_solved : int - number of solved sequences sum_of_first_distances : metric describing quality of result fidelity : the used fidelities in this evaluation """ self.rng = rng_helper.get_rng(rng, self.rng) tmp_dir = Path(tempfile.mkdtemp(dir=self.config.cache_dir)) config, network_config, agent_config, env_config = self._setup(configuration) start_time = time() try: train_info = self._train(budget=fidelity["cutoff_agent_per_sequence"], tmp_dir=tmp_dir, network_config=network_config, agent_config=agent_config, env_config=env_config) validation_info = self._validate(evaluation_timeout=60, restore_path=tmp_dir, stop_learning=True, restart_timeout=config["restart_timeout"], network_config=network_config, agent_config=agent_config, env_config=env_config) cost = time() - start_time finally: shutil.rmtree(tmp_dir, ignore_errors=True) return {'function_value': validation_info["sum_of_min_distances"], 'cost': cost, 'info': {'train_info': train_info, 'validation_info': validation_info, 'fidelity': fidelity}, }
def objective_function(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[CS.Configuration, Dict, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Start the learna experiment. Dont train a RL agent. Just optimize on the sequences. Parameters ---------- configuration : Dict, CS.Configuration fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None, Random seed for benchmark. By default the class level random seed. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : sum of min distances cost : time to train and evaluate the model info : Dict num_solved : int - number of soved sequences sum_of_first_distances : metric describing quality of result fidelity : the used fidelities in this evaluation """ self.rng = rng_helper.get_rng(rng, self_rng=self.rng) config, network_config, agent_config, env_config = self._setup( configuration) start_time = time() validation_info = self._validate( evaluation_timeout=fidelity["cutoff_agent_per_sequence"], restore_path=None, stop_learning=False, restart_timeout=config["restart_timeout"], network_config=network_config, agent_config=agent_config, env_config=env_config) cost = time() - start_time return { 'function_value': validation_info["sum_of_min_distances"], 'cost': cost, 'info': { 'num_solved': validation_info["num_solved"], 'sum_of_first_distances': validation_info["sum_of_first_distances"], 'fidelity': fidelity, } }
def _train_objective(self, config: Dict, fidelity: Dict, shuffle: bool, rng: Union[np.random.RandomState, int, None] = None, evaluation: Union[str, None] = "valid"): if rng is not None: rng = get_rng(rng, self.rng) # initializing model model = self.init_model(config, fidelity, rng) # preparing data if eval == "valid": train_X = self.train_X train_y = self.train_y train_idx = self.train_idx else: train_X = np.vstack((self.train_X, self.valid_X)) train_y = pd.concat((self.train_y, self.valid_y)) train_idx = np.arange(len(train_X)) # shuffling data if shuffle: train_idx = self.shuffle_data_idx(train_idx, rng) train_X = train_X.iloc[train_idx] train_y = train_y.iloc[train_idx] # subsample here: # application of the other fidelity to the dataset that the model interfaces if self.lower_bound_train_size is None: self.lower_bound_train_size = ( 10 * self.n_classes) / self.train_X.shape[0] self.lower_bound_train_size = np.max( (1 / 512, self.lower_bound_train_size)) subsample = np.max( (fidelity['subsample'], self.lower_bound_train_size)) train_idx = self.rng.choice(np.arange(len(train_X)), size=int(subsample * len(train_X))) # fitting the model with subsampled data start = time.time() model.fit(train_X[train_idx], train_y.iloc[train_idx]) model_fit_time = time.time() - start # computing statistics on training data scores = dict() score_cost = dict() for k, v in self.scorers.items(): scores[k] = 0.0 score_cost[k] = 0.0 if evaluation == "test": _start = time.time() scores[k] = v(model, train_X, train_y) score_cost[k] = time.time() - _start train_loss = 1 - scores["acc"] return model, model_fit_time, train_loss, scores, score_cost
def __init__(self, openml_task_id: int, rng: Union[int, np.random.RandomState, None] = None): super(OpenMLCrossvalidationDataManager, self).__init__() self._save_to = hpobench.config_file.data_dir / 'OpenML' self.task_id = openml_task_id self.rng = get_rng(rng=rng) self.name = None self.variable_types = None self.create_save_directory(self._save_to) openml.config.apikey = '610344db6388d9ba34f6db45a3cf71de' openml.config.set_cache_directory(str(self._save_to))
def __init__(self, rng: Union[int, np.random.RandomState, None] = None, defaults: Union[Dict, None] = None, max_episodes: Union[int, None] = 3000): """ Base benchmark for "cartpole" benchmark. In this benchmark a PPO agent tries to solve the cartpole task. Parameters ---------- rng : int,None,np.RandomState RandomState for the experiment defaults : dict, None default configuration used for the PPO agent max_episodes : int, None limit of the length of a episode for the cartpole runner. Defaults to 3000 """ logger.warning('This Benchmark is not deterministic.') super(CartpoleBase, self).__init__() self.rng = rng_helper.get_rng(rng=rng) tf.random.set_random_seed(0) np.random.seed(0) self.env = OpenAIGym('CartPole-v0', visualize=False) self.avg_n_episodes = 20 self.max_episodes = max_episodes self.defaults = { "n_units_1": 64, "n_units_2": 64, "batch_size": 64, "learning_rate": 1e-3, "discount": 0.99, "likelihood_ratio_clipping": 0.2, "activation_1": "tanh", "activation_2": "tanh", "optimizer_type": "adam", "optimization_steps": 10, "baseline_mode": "states", "baseline_n_units_1": 64, "baseline_n_units_2": 64, "baseline_learning_rate": 1e-3, "baseline_optimization_steps": 10, "baseline_optimizer_type": "adam" } if defaults is not None: self.defaults.update(defaults)
def __init__(self, dataset_name: str, rng: Union[np.random.RandomState, int, None] = None): """ Parameters ---------- dataset_name : str rng : np.random.RandomState, int, None """ self.rng = rng_helper.get_rng(rng) # Load dataset manager self.dataset_name = dataset_name self.datamanager = OutlierDetectionDataManager(dataset_name, self.rng) super(ODTraditional, self).__init__(rng=self.rng)
def objective_function(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[CS.Configuration, Dict, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Query the NAS-benchmark using a given configuration and a epoch (=budget). Parameters ---------- configuration : Dict, CS.Configuration fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None Random seed to use in the benchmark. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : validation error cost : runtime info : Dict fidelity : used fidelities in this evaluation """ self.benchmark.reset_tracker() self.rng = rng_helper.get_rng(rng, self_rng=self.rng) # Returns (valid_accuracy: 0, runtime: 0) if it is invalid, e.g. config not valid or # budget not in 4 12 36 108 data = self._query_benchmark(config=configuration, budget=fidelity['budget']) return { 'function_value': 1 - data['validation_accuracy'], 'cost': data['training_time'], 'info': { 'fidelity': fidelity, 'data': data } }
def objective_function_test(self, configuration: Union[Dict, CS.Configuration], fidelity: Union[Dict, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Parameters ---------- configuration : Dict, CS.Configuration fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None Random seed to use in the benchmark. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : validation loss cost : time to train and evaluate the model info : Dict valid_rmse_per_run runtime_per_run fidelity : used fidelities in this evaluation """ self.rng = rng_helper.get_rng(rng, self_rng=self.rng) default_fidelity = self.get_fidelity_space().get_default_configuration( ).get_dictionary() assert fidelity == default_fidelity, 'Test function works only on the highest budget.' result = self.benchmark.objective_function_test(configuration) return { 'function_value': float(result[0]), 'cost': float(result[1]), 'info': { 'fidelity': fidelity }, }
def __init__(self, rng: Union[int, np.random.RandomState, None] = None): """ Interface for benchmarks. A benchmark consists of two building blocks, the target function and the configuration space. Furthermore it can contain additional benchmark-specific information such as the location and the function value of the global optima. New benchmarks should be derived from this base class or one of its child classes. Parameters ---------- rng: int, np.random.RandomState, None The default random state for the benchmark. If type is int, a np.random.RandomState with seed `rng` is created. If type is None, create a new random state. """ self.rng = rng_helper.get_rng(rng=rng) self.configuration_space = self.get_configuration_space() self.fidelity_space = self.get_fidelity_space()
def __init__(self, dataset_name: str, rng: Union[np.random.RandomState, int, None] = None): """ Parameters ---------- dataset_name : str Must be one of [ "annthyroid", "arrhythmia", "breastw", "cardio", "ionosphere", "mammography", "musk", "optdigits", "pendigits", "pima", "satellite", "satimage-2", "thyroid", "vowels", "wbc"] rng : np.random.RandomState, int, None """ self.rng = rng_helper.get_rng(rng) pl.seed_everything(self.rng.randint(0, 10000)) # Load datamanager # It's important to call it before super # as AbstractBenchmark samples configuration space in which # the datamanager is needed self.dataset_name = dataset_name self.datamanager = OutlierDetectionDataManager(dataset_name, self.rng) super(ODAutoencoder, self).__init__(rng=self.rng)
def objective_function(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[CS.Configuration, Dict, None] = None, run_index: Union[int, Tuple, None] = (0, 1, 2), rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Query the NAS-benchmark using a given configuration and a epoch (=budget). Parameters ---------- configuration : Dict, CS.Configuration fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. run_index : int, Tuple, None The nas benchmark has for each configuration-budget-pair results from 3 different runs. - If multiple `run_id`s are given as Tuple, the benchmark returns the mean over the given runs. - By default (no parameter is specified) all runs are used. A specific run can be chosen by setting the `run_id` to a value from [0, 3]. While the performance is averaged across the `run_index`, the costs are the sum of the runtime per `run_index`. - When this value is explicitly set to `None`, the function will use a random seed. rng : np.random.RandomState, int, None Random seed to use in the benchmark. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : validation error cost : runtime info : Dict fidelity : used fidelities in this evaluation """ self.rng = rng_helper.get_rng(rng, self_rng=self.rng) if isinstance(run_index, int): assert 0 <= run_index <= 2, f'run_index must be in [0, 2], not {run_index}' run_index = (run_index, ) elif isinstance(run_index, (Tuple, List)): assert 0 < len(run_index) <= 3, 'run_index must not be empty' assert min(run_index) >= 0 and max(run_index) <= 2, \ f'all run_index values must be in [0, 2], but were {run_index}' if len(set(run_index)) != len(run_index): logger.debug('There are some values more than once in the run_index. We remove the redundant entries.') run_index = tuple(set(run_index)) elif run_index is None: logger.debug('The run index is explicitly set to None! A random seed will be selected.') run_index = tuple(self.rng.choice((0, 1, 2), size=1)) else: raise ValueError(f'run index must be one of Tuple or Int, but was {type(run_index)}') self.benchmark.reset_tracker() # Returns (valid_accuracy: 0, runtime: 0) if it is invalid, e.g. config not valid or # budget not in 4 12 36 108 train_accuracies = [] valid_accuracies = [] test_accuracies = [] training_times = [] additional = {} for run_id in run_index: data = self._query_benchmark(config=configuration, budget=fidelity['budget'], run_index=run_id) train_accuracies.append(data['train_accuracy']) valid_accuracies.append(data['validation_accuracy']) test_accuracies.append(data['test_accuracy']) training_times.append(data['training_time']) # Since those information are the same for all run ids, just store one of them. additional = {'trainable_parameters': data['trainable_parameters'], 'module_operations': data['module_operations']} return {'function_value': float(1 - np.mean(valid_accuracies)), 'cost': float(np.sum(training_times)), 'info': {'fidelity': fidelity, 'train_accuracies': train_accuracies, 'valid_accuracies': valid_accuracies, 'test_accuracies': test_accuracies, 'training_times': training_times, 'data': additional } }
def objective_function(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[Dict, None] = None, rng: Union[np.random.RandomState, int, None] = None, data_seed: Union[List, Tuple, int, None] = (777, 888, 999), **kwargs) -> Dict: """ Objective function for the NASBench201 benchmark. This functions sends a query to NASBench201 and evaluates the configuration. As already explained in the class definition, different data sets are trained on different splits. For example cifar10 is trained on the train and validation split and tested on the test split. Therefore, different entries are returned from the NASBench201 result. Overview of the used splits for training and testing and which are returned in the objective_function and which in the objective_function_test. |-------------------|-----------------------|---------------------------| | | Returned by | Returned by | | | objective_function | objective_function_test | | Data set | train_* | eval_* | |-------------------|-----------------------|---------------------------| | 'cifar10-valid' | train | valid | | 'cifar10' | train + valid | test | | 'cifar100' | train | valid + test | | 'ImageNet16-120' | train | valid + test | |-------------------|-----------------------|---------------------------| Legend: * = [losses, acc1es, times] Parameters ---------- configuration fidelity: Dict, None epoch: int - Values: [1, 200] Number of epochs an architecture was trained. Note: the number of epoch is 1 indexed! (Results after the first epoch: epoch = 1) Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None Random seed to use in the benchmark. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. data_seed : List, Tuple, None, int The nasbench_201 benchmark include for each run 3 different seeds: 777, 888, 999. The user can specify which seed to use. If more than one seed is given, the results are averaged across the seeds but then the time needed for training is the sum of the costs per seed. Note: For some architectures (configurations) no run was available. We've set missing values to an available value from another seed. Therefore, it is possible that run results are exactly the same for different seeds. kwargs Returns ------- Dict - function_value : training precision cost : time to train the network info : Dict train_precision : float train_losses : float train_cost : float Time needed to train the network for 'epoch' many epochs. If more than one seed is given, this field is the sum of the training time per network eval_precision : float eval_losses : float eval_cost : float Time needed to train the network for 'epoch many epochs plus the time to evaluate the network on the evaluation split. If more than one seed is given, this field is the sum of the eval cost per network fidelity : Dict used fidelities in this evaluation """ # Check if the data set seeds are valid assert isinstance(data_seed, List) or isinstance(data_seed, Tuple) or isinstance(data_seed, int), \ f'data seed has unknown data type {type(data_seed)}, but should be tuple or int (777,888,999)' if isinstance(data_seed, List): data_seed = tuple(data_seed) if isinstance(data_seed, int): data_seed = (data_seed, ) assert len(set(data_seed) - {777, 888, 999}) == 0,\ f'data seed can only contain the elements 777, 888, 999, but was {data_seed}' self.rng = rng_helper.get_rng(rng) structure = self.config_to_structure(configuration) structure_str = structure.tostr() epoch = fidelity['epoch'] - 1 train_accuracies = [ self.data[(seed, 'train_acc1es')][structure_str][epoch] for seed in data_seed ] train_losses = [ self.data[(seed, 'train_losses')][structure_str][epoch] for seed in data_seed ] train_times = [ np.sum(self.data[(seed, 'train_times')][structure_str][:epoch + 1]) for seed in data_seed ] valid_accuracies = [ self.data[(seed, 'valid_acc1es')][structure_str][epoch] for seed in data_seed ] valid_losses = [ self.data[(seed, 'valid_losses')][structure_str][epoch] for seed in data_seed ] valid_times = [ np.sum(self.data[(seed, 'valid_times')][structure_str][:epoch + 1]) for seed in data_seed ] # There is a single value for the eval data per seed. (only epoch 200) test_accuracies = [ self.data[(seed, 'test_acc1es')][structure_str] for seed in data_seed ] test_losses = [ self.data[(seed, 'test_losses')][structure_str] for seed in data_seed ] test_times = [ np.sum(self.data[(seed, 'test_times')][structure_str]) for seed in data_seed ] return { 'function_value': float(100 - np.mean(valid_accuracies)), 'cost': float(np.sum(valid_times) + np.sum(train_times)), 'info': { 'train_precision': float(100 - np.mean(train_accuracies)), 'train_losses': float(np.mean(train_losses)), 'train_cost': float(np.sum(train_times)), 'valid_precision': float(100 - np.mean(valid_accuracies)), 'valid_losses': float(np.mean(valid_losses)), 'valid_cost': float(np.sum(valid_times) + np.sum(train_times)), 'test_precision': float(100 - np.mean(test_accuracies)), 'test_losses': float(np.mean(test_losses)), 'test_cost': float(np.sum(train_times)) + float(np.sum(test_times)), 'fidelity': fidelity } }
def run_experiment(out_path, on_travis): settings = { 'min_budget': 1, 'max_budget': 9, # number of repetitions; this is the fidelity for this bench 'num_iterations': 10, # Set this to a low number for demonstration 'eta': 3, 'output_dir': Path(out_path) } if on_travis: settings.update(get_travis_settings('bohb')) b = Benchmark(rng=1) b.get_configuration_space(seed=1) settings.get('output_dir').mkdir(exist_ok=True) cs = b.get_configuration_space() seed = get_rng(rng=0) run_id = 'BOHB_on_cartpole' result_logger = hpres.json_result_logger(directory=str( settings.get('output_dir')), overwrite=True) ns = hpns.NameServer(run_id=run_id, host='localhost', working_directory=str(settings.get('output_dir'))) ns_host, ns_port = ns.start() worker = CustomWorker(seed=seed, nameserver=ns_host, nameserver_port=ns_port, run_id=run_id, max_budget=settings.get('max_budget')) worker.run(background=True) master = BOHB(configspace=cs, run_id=run_id, host=ns_host, nameserver=ns_host, nameserver_port=ns_port, eta=settings.get('eta'), min_budget=settings.get('min_budget'), max_budget=settings.get('max_budget'), result_logger=result_logger) result = master.run(n_iterations=settings.get('num_iterations')) master.shutdown(shutdown_workers=True) ns.shutdown() with open(settings.get('output_dir') / 'results.pkl', 'wb') as f: pickle.dump(result, f) id2config = result.get_id2config_mapping() incumbent = result.get_incumbent_id() inc_value = result.get_runs_by_id(incumbent)[-1]['loss'] inc_cfg = id2config[incumbent]['config'] logger.info(f'Inc Config:\n{inc_cfg}\n' f'with Performance: {inc_value:.2f}') if not on_travis: benchmark = Benchmark(container_source='library://phmueller/automl') incumbent_result = benchmark.objective_function_test( configuration=inc_cfg, fidelity={"budget": settings['max_budget']}) print(incumbent_result)
def shuffle_data(self, rng=None): """ Reshuffle the training data. If 'rng' is None, the training idx are shuffled according to the class-random-state""" random_state = rng_helper.get_rng(rng, self.rng) random_state.shuffle(self.train_idx)
def objective_function(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[CS.Configuration, Dict, None] = None, shuffle: bool = False, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Trains a XGBoost model given a hyperparameter configuration and evaluates the model on the validation set. Parameters ---------- configuration : Dict, CS.Configuration Configuration for the XGBoost model fidelity: Dict, None Fidelity parameters for the XGBoost model, check get_fidelity_space(). Uses default (max) value if None. shuffle : bool If ``True``, shuffle the training idx. If no parameter ``rng`` is given, use the class random state. Defaults to ``False``. rng : np.random.RandomState, int, None, Random seed for benchmark. By default the class level random seed. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : validation loss cost : time to train and evaluate the model info : Dict train_loss : trainings loss fidelity : used fidelities in this evaluation """ self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng) if shuffle: self.shuffle_data(self.rng) start = time.time() if self.lower_bound_train_size > fidelity['dataset_fraction']: train_data_fraction = self.lower_bound_train_size logger.warning( f'The given data set fraction is lower than the lower bound (10 * number of classes.) ' f'Increase the fidelity from {fidelity["dataset_fraction"]:.8f} to ' f'{self.lower_bound_train_size:.8f}') else: train_data_fraction = fidelity['dataset_fraction'] train_idx = self.train_idx[:int( len(self.train_idx) * train_data_fraction)] model = self._get_pipeline(n_estimators=fidelity["n_estimators"], **configuration) model.fit(X=self.x_train[train_idx], y=self.y_train[train_idx]) train_loss = 1 - self.accuracy_scorer(model, self.x_train[train_idx], self.y_train[train_idx]) val_loss = 1 - self.accuracy_scorer(model, self.x_valid, self.y_valid) cost = time.time() - start return { 'function_value': float(val_loss), 'cost': cost, 'info': { 'train_loss': float(train_loss), 'fidelity': fidelity } }
def objective_function(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[Dict, CS.Configuration, None] = None, rng: Union[np.random.RandomState, int, None] = None, data_seed: Union[List, Tuple, int, None] = (777, 888, 999), **kwargs) -> Dict: """ Objective function for the NASBench201 benchmark. This functions sends a query to NASBench201 and evaluates the configuration. As already explained in the class definition, different data sets are trained on different splits. The table above gives a detailed summary over the available splits, epochs, and which identifier are used per dataset. Parameters ---------- configuration fidelity: Dict, None epoch: int - Values: [1, 200] Number of epochs an architecture was trained. Note: the number of epoch is 1 indexed! (Results after the first epoch: epoch = 1) Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None Random seed to use in the benchmark. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. data_seed : List, Tuple, None, int The nasbench_201 benchmark include for each run 3 different seeds: 777, 888, 999. The user can specify which seed to use. If more than one seed is given, the results are averaged across the seeds but then the training time is the sum of the costs per seed. When this value is explicitly set to `None`, the function will chose randomly one out of [777, 888, 999]. Note: For some architectures (configurations) no run was available. We've set missing values to an available value from another seed. Therefore, it is possible that run results are exactly the same for different seeds. kwargs Returns ------- Dict - function_value : training precision cost : time to train the network info : Dict train_precision : float train_losses : float train_cost : float Time needed to train the network for 'epoch' many epochs. If more than one seed is given, this field is the sum of the training time per network eval_precision : float eval_losses : float eval_cost : float Time needed to train the network for 'epoch many epochs plus the time to evaluate the network on the evaluation split. If more than one seed is given, this field is the sum of the eval cost per network fidelity : Dict used fidelities in this evaluation """ self.rng = rng_helper.get_rng(rng) if isinstance(data_seed, (List, Tuple)): assert len(data_seed) != 0, 'data_seed must not be empty' if len(set(data_seed)) != len(data_seed): logger.debug( 'There are some values more than once in the run_index. We remove the redundant entries.' ) data_seed = tuple(set(data_seed)) elif isinstance(data_seed, int): data_seed = (data_seed, ) elif data_seed is None: logger.debug( 'The data seed is explicitly set to None! A random seed will be selected.' ) data_seed = tuple(self.rng.choice((777, 888, 999), size=1)) # Check if the data set seeds are valid else: raise ValueError( f'data seed has unknown data type {type(data_seed)}, ' f'but should be tuple or int (777,888,999)') assert len(set(data_seed) - {777, 888, 999}) == 0,\ f'data seed can only contain the elements 777, 888, 999, but was {data_seed}' structure = self.config_to_structure(configuration) structure_str = structure.tostr() epoch = fidelity['epoch'] - 1 data_seed = [str(seed) for seed in data_seed] valid_key, test_key = self.dataset_mapping(self.dataset) train_accuracies = [ self.data[seed][structure_str]['train_acc1es'][f'{epoch}'] for seed in data_seed ] train_losses = [ self.data[seed][structure_str]['train_losses'][f'{epoch}'] for seed in data_seed ] train_times = [ np.sum((self.data[seed][structure_str]['train_times'][f'{e}']) for e in range(1, epoch + 1)) for seed in data_seed ] valid_accuracies = [ self.data[seed][structure_str]['eval_acc1es'] [f'{valid_key}@{epoch}'] for seed in data_seed ] valid_losses = [ self.data[seed][structure_str]['eval_losses'] [f'{valid_key}@{epoch}'] for seed in data_seed ] valid_times = [ np.sum((self.data[seed][structure_str]['eval_times'] [f'{valid_key}@{e}']) for e in range(1, epoch + 1)) for seed in data_seed ] # There is a single value for the eval data per seed. (only epoch 200) test_accuracies = [ self.data[seed][structure_str]['eval_acc1es'][f'{valid_key}@{199}'] for seed in data_seed ] test_losses = [ self.data[seed][structure_str]['eval_losses'][f'{valid_key}@{199}'] for seed in data_seed ] test_times = [ np.sum((self.data[seed][structure_str]['eval_times'] [f'{test_key}@{199}']) for e in range(1, epoch + 1)) for seed in data_seed ] return { 'function_value': float(100 - np.mean(valid_accuracies)), 'cost': float(np.sum(valid_times) + np.sum(train_times)), 'info': { 'train_precision': float(100 - np.mean(train_accuracies)), 'train_losses': float(np.mean(train_losses)), 'train_cost': float(np.sum(train_times)), 'valid_precision': float(100 - np.mean(valid_accuracies)), 'valid_losses': float(np.mean(valid_losses)), 'valid_cost': float(np.sum(valid_times) + np.sum(train_times)), 'test_precision': float(100 - np.mean(test_accuracies)), 'test_losses': float(np.mean(test_losses)), 'test_cost': float(np.sum(train_times)) + float(np.sum(test_times)), 'fidelity': fidelity } }
def objective_function(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[Dict, CS.Configuration, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Trains a bayesian neural network with 3 layers on the defined data set and evaluates the trained model on the validation split. Parameters ---------- configuration : Dict, CS.Configuration Configuration for the pyBNN model fidelity: Dict, None budget : int [500 - 10000] number of epochs to train the model Fidelity parameters for the pyBNN model, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None, Random seed for benchmark. By default the class level random seed. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : validation loss cost : time to train and evaluate the model info : Dict fidelity : used fidelities in this evaluation """ start = time.time() self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng) np.random.seed(self.rng.randint(1, 10000)) # See comment in objective function test burn_in_steps = max(1, int(configuration['burn_in'] * fidelity['budget'])) net = partial(_get_net, n_units_1=configuration['n_units_1'], n_units_2=configuration['n_units_2']) model = BayesianNeuralNetwork(sampling_method="sghmc", get_net=net, l_rate=configuration['l_rate'], mdecay=configuration['mdecay'], burn_in=burn_in_steps, n_iters=fidelity['budget'], precondition=True, normalize_input=True, normalize_output=True, rng=self.rng) model.train(self.train, self.train_targets, valid=self.valid, valid_targets=self.valid_targets, valid_after_n_steps=100) mean_pred, var_pred = model.predict(self.valid) # Negative log-likelihood valid_loss = self._neg_log_likelihood(self.valid_targets, mean_pred, var_pred) cost = time.time() - start return { 'function_value': float(valid_loss), 'cost': cost, 'info': { 'fidelity': fidelity } }
def objective_function_test(self, configuration: Union[Dict, CS.Configuration], fidelity: Union[Dict, None] = None, shuffle: bool = False, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Trains a SVM model with a given configuration on both the X_train and validation data set and evaluates the model on the X_test data set. Parameters ---------- configuration : Dict, CS.Configuration Configuration for the SVM Model fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None, Random seed for benchmark. By default the class level random seed. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : X_test loss cost : time to X_train and evaluate the model info : Dict train_valid_loss: Loss on the train+valid data set fidelity : used fidelities in this evaluation """ assert np.isclose(fidelity['dataset_fraction'], 1), \ f'Data set fraction must be 1 but was {fidelity["dataset_fraction"]}' self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng) start_time = time.time() # Concatenate training and validation dataset if isinstance(self.X_train, sparse.csr.csr_matrix) or isinstance( self.X_valid, sparse.csr.csr_matrix): data = sparse.vstack((self.X_train, self.X_valid)) else: data = np.concatenate((self.X_train, self.X_valid)) targets = np.concatenate((self.y_train, self.y_valid)) # Transform hyperparameters to linear scale hp_c = np.exp(float(configuration['C'])) hp_gamma = np.exp(float(configuration['gamma'])) model = self.get_pipeline(hp_c, hp_gamma) model.fit(data, targets) # Compute validation error train_valid_loss = 1 - self.accuracy_scorer(model, data, targets) # Compute test error test_loss = 1 - self.accuracy_scorer(model, self.X_test, self.y_test) cost = time.time() - start_time return { 'function_value': test_loss, "cost": cost, 'info': { 'train_valid_loss': train_valid_loss, 'fidelity': fidelity } }
def objective_function(self, configuration: Union[Dict, CS.Configuration], fidelity: Union[Dict, None] = None, shuffle: bool = False, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Trains a SVM model given a hyperparameter configuration and evaluates the model on the validation set. Parameters ---------- configuration : Dict, CS.Configuration Configuration for the SVM model fidelity: Dict, None Fidelity parameters for the SVM model, check get_fidelity_space(). Uses default (max) value if None. shuffle : bool If ``True``, shuffle the training idx. If no parameter ``rng`` is given, use the class random state. Defaults to ``False``. rng : np.random.RandomState, int, None, Random seed for benchmark. By default the class level random seed. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : validation loss cost : time to train and evaluate the model info : Dict train_loss : training loss fidelity : used fidelities in this evaluation """ start_time = time.time() self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng) if shuffle: self.shuffle_data(self.rng) # Split of dataset subset if self.lower_bound_train_size > fidelity['dataset_fraction']: train_size = self.lower_bound_train_size logger.warning( f'The given data set fraction is lower than the lower bound (10 * number of classes.) ' f'Increase the fidelity from {fidelity["dataset_fraction"]:.2f} to ' f'{self.lower_bound_train_size:.2f}') else: train_size = fidelity['dataset_fraction'] train_size = int(train_size * len(self.train_idx)) train_idx = self.train_idx[:train_size] # Transform hyperparameters to linear scale hp_c = np.exp(float(configuration['C'])) hp_gamma = np.exp(float(configuration['gamma'])) # Train support vector machine model = self.get_pipeline(hp_c, hp_gamma) model.fit(self.X_train[train_idx], self.y_train[train_idx]) # Compute validation error train_loss = 1 - self.accuracy_scorer(model, self.X_train[train_idx], self.y_train[train_idx]) val_loss = 1 - self.accuracy_scorer(model, self.X_valid, self.y_valid) cost = time.time() - start_time return { 'function_value': val_loss, "cost": cost, 'info': { 'train_loss': train_loss, 'fidelity': fidelity } }
def objective_function(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[CS.Configuration, Dict, None] = None, run_index: Union[int, Tuple, None] = (0, 1, 2, 3), rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Query the NAS-benchmark using a given configuration and a epoch (=budget). Parameters ---------- configuration : Dict, CS.Configuration fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. run_index : int, Tuple, None The nas benchmark has for each configuration-budget-pair results from 4 different runs. If multiple `run_id`s are given, the benchmark returns the mean over the given runs. By default (no parameter is specified) all runs are used. A specific run can be chosen by setting the `run_id` to a value from [0, 3]. When this value is explicitly set to `None`, the function will use a random seed. rng : np.random.RandomState, int, None Random seed to use in the benchmark. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : validation loss cost : time to train and evaluate the model info : Dict with valid_rmse_per_run, runtime_per_run info : Dict valid_rmse_per_run runtime_per_run fidelity : used fidelities in this evaluation """ self.rng = rng_helper.get_rng(rng) if isinstance(run_index, int): assert 0 <= run_index <= 3, f'run_index must be in [0, 3], not {run_index}' run_index = (run_index, ) elif isinstance(run_index, (Tuple, List)): assert len(run_index) != 0, 'run_index must not be empty' if len(set(run_index)) != len(run_index): logger.debug( 'There are some values more than once in the run_index. We remove the redundant entries.' ) run_index = tuple(set(run_index)) assert min(run_index) >= 0 and max(run_index) <= 3, \ f'all run_index values must be in [0, 3], but were {run_index}' elif run_index is None: logger.debug( 'The run index is explicitly set to None! A random seed will be selected.' ) run_index = tuple(self.rng.choice((0, 1, 2, 3), size=1)) else: raise ValueError( f'run index must be one of Tuple or Int, but was {type(run_index)}' ) self._reset_tracker() valid_rmse_list, runtime_list = [], [] for run_id in run_index: valid_rmse, runtime = self.benchmark.objective_function_deterministic( config=configuration, budget=fidelity["budget"], index=run_id) valid_rmse_list.append(float(valid_rmse)) runtime_list.append(float(runtime)) valid_rmse = sum(valid_rmse_list) / len(valid_rmse_list) runtime = sum(runtime_list) return { 'function_value': float(valid_rmse), 'cost': float(runtime), 'info': { 'valid_rmse_per_run': valid_rmse_list, 'runtime_per_run': runtime_list, 'fidelity': fidelity }, }
def objective_function_test(self, configuration: Union[Dict, CS.Configuration], fidelity: Union[Dict, CS.Configuration, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Trains a bayesian neural network with 3 layers on the train and valid data split and evaluates it on the test split. Parameters ---------- configuration : Dict, CS.Configuration Configuration for the pyBNN model fidelity: Dict, None budget : int [500 - 10000] number of epochs to train the model Fidelity parameters for the pyBNN model, check get_fidelity_space(). Uses default (max) value if None. Note: The fidelity should be here the max budget (= 10000). By leaving this field empty, the maximum budget will be used by default. rng : np.random.RandomState, int, None, Random seed for benchmark. By default the class level random seed. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : validation loss cost : time to train and evaluate the model info : Dict fidelity : used fidelities in this evaluation """ start = time.time() self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng) np.random.seed(self.rng.randint(1, 10000)) # `burn_in_steps` must be at least 1, otherwise, theano will raise an RuntimeError. (Actually, the definition of # the config space allows as lower limit a zero. In this case, set the number of steps to 1.) burn_in_steps = max(1, int(configuration['burn_in'] * fidelity['budget'])) net = partial(_get_net, n_units_1=configuration['n_units_1'], n_units_2=configuration['n_units_2']) model = BayesianNeuralNetwork(sampling_method="sghmc", get_net=net, l_rate=configuration['l_rate'], mdecay=configuration['mdecay'], burn_in=burn_in_steps, n_iters=fidelity['budget'], precondition=True, normalize_input=True, normalize_output=True, rng=self.rng) train = np.concatenate((self.train, self.valid)) train_targets = np.concatenate( (self.train_targets, self.valid_targets)) model.train(train, train_targets) mean_pred, var_pred = model.predict(self.test) test_loss = self._neg_log_likelihood(self.test_targets, mean_pred, var_pred) cost = time.time() - start return { 'function_value': float(test_loss), 'cost': cost, 'info': { 'fidelity': fidelity } }
def objective_function(self, configuration: Union[Dict, CS.Configuration], fidelity: Union[Dict, CS.Configuration, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Trains a Tensorforce RL agent on the cartpole experiment. This benchmark was used in the experiments for the BOHB-paper (see references). A more detailed explanations can be found there. The budget describes how often the agent is trained on the experiment. It returns the average number of the length of episodes. Parameters ---------- configuration : Dict, CS.Configuration fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None Random seed to use in the benchmark. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : average episode length cost : time to run all agents info : Dict max_episodes : the maximum length of an episode budget : number of agents used all_runs : the episode length of all runs of all agents fidelity : the used fidelities in this evaluation """ self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng) tf.random.set_random_seed(self.rng.randint(1, 100000)) np.random.seed(self.rng.randint(1, 100000)) # fill in missing entries with default values for 'incomplete/reduced' configspaces new_config = self.defaults new_config.update(configuration) configuration = new_config start_time = time.time() network_spec = [{ 'type': 'dense', 'size': configuration["n_units_1"], 'activation': configuration['activation_1'] }, { 'type': 'dense', 'size': configuration["n_units_2"], 'activation': configuration['activation_2'] }] converged_episodes = [] for _ in range(fidelity["budget"]): agent = PPOAgent( states=self.env.states, actions=self.env.actions, network=network_spec, update_mode={ 'unit': 'episodes', 'batch_size': configuration["batch_size"] }, step_optimizer={ 'type': configuration["optimizer_type"], 'learning_rate': configuration["learning_rate"] }, optimization_steps=configuration["optimization_steps"], discount=configuration["discount"], baseline_mode=configuration["baseline_mode"], baseline={ "type": "mlp", "sizes": [ configuration["baseline_n_units_1"], configuration["baseline_n_units_2"] ] }, baseline_optimizer={ "type": "multi_step", "optimizer": { "type": configuration["baseline_optimizer_type"], "learning_rate": configuration["baseline_learning_rate"] }, "num_steps": configuration["baseline_optimization_steps"] }, likelihood_ratio_clipping=configuration[ "likelihood_ratio_clipping"], entropy_regularization=configuration["entropy_regularization"], ) def episode_finished(record): # Check if we have converged return np.mean( record.episode_rewards[-self.avg_n_episodes:]) != 200 runner = Runner(agent=agent, environment=self.env) runner.run(episodes=self.max_episodes, max_episode_timesteps=200, episode_finished=episode_finished) converged_episodes.append(len(runner.episode_rewards)) cost = time.time() - start_time return { 'function_value': np.mean(converged_episodes), 'cost': cost, 'info': { 'max_episodes': self.max_episodes, 'all_runs': converged_episodes, 'fidelity': fidelity } }
def objective_function(self, configuration: Union[Dict, CS.Configuration], fidelity: Union[Dict, None] = None, shuffle: bool = False, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Trains a XGBoost model given a hyperparameter configuration and evaluates the model on the validation set. Parameters ---------- configuration : Dict, CS.Configuration Configuration for the XGBoost model fidelity: Dict, None Fidelity parameters for the XGBoost model, check get_fidelity_space(). Uses default (max) value if None. shuffle : bool If ``True``, shuffle the training idx. If no parameter ``rng`` is given, use the class random state. Defaults to ``False``. rng : np.random.RandomState, int, None, Random seed for benchmark. By default the class level random seed. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : validation loss cost : time to train and evaluate the model info : Dict train_loss : trainings loss fidelity : used fidelities in this evaluation """ self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng) if shuffle: self.shuffle_data(self.rng) start = time.time() train_idx = self.train_idx[:int( len(self.train_idx) * fidelity["subsample"])] model = self._get_pipeline(n_estimators=fidelity["n_estimators"], **configuration) model.fit(X=self.X_train[train_idx], y=self.y_train[train_idx]) train_loss = 1 - self.accuracy_scorer(model, self.X_train[train_idx], self.y_train[train_idx]) val_loss = 1 - self.accuracy_scorer(model, self.X_valid, self.y_valid) cost = time.time() - start return { 'function_value': val_loss, 'cost': cost, 'info': { 'train_loss': train_loss, 'fidelity': fidelity } }
def objective_function(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[CS.Configuration, Dict, None] = None, run_index: Union[int, Tuple, List, None] = (0, 1, 2), rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ Query the NAS1shot1-benchmark using a given configuration and an epoch (=budget). Only data for the budgets 4, 12, 36, 108 are available. Parameters ---------- configuration : Dict, CS.Configuration fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. run_index : int, Tuple, None The nas benchmark has for each configuration-budget-pair results from 3 different runs. - If multiple `run_id`s are given as Tuple/List, the benchmark returns the mean over the given runs. - By default (no parameter is specified) all runs are used. A specific run can be chosen by setting the `run_id` to a value from [0, 3]. While the performance is averaged across the `run_index`, the costs are the sum of the runtime per `run_index`. - When this value is explicitly set to `None`, the function will use a random seed. rng : np.random.RandomState, int, None Random seed to use in the benchmark. To prevent overfitting on a single seed, it is possible to pass a parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. If this parameter is not given, the default random state is used. kwargs Returns ------- Dict - function_value : validation error cost : runtime info : Dict train_accuracies test_accuracies valid_accuracies training_times fidelity : used fidelities in this evaluation data : additional data such as trainable parameters and used operations """ self.rng = rng_helper.get_rng(rng, self_rng=self.rng) run_index = self._check_run_index(run_index) configuration = self._parse_configuration(configuration) train_accuracies = [] valid_accuracies = [] test_accuracies = [] training_times = [] additional = {} failure = False for run_id in run_index: data = self._query_benchmark(config=configuration, fidelity=fidelity, run_index=run_id) train_accuracies.append(data['train_accuracy']) valid_accuracies.append(data['validation_accuracy']) test_accuracies.append(data['test_accuracy']) training_times.append(data['training_time']) # Since those information are the same for all run ids, just store one of them. additional = { 'trainable_parameters': data['trainable_parameters'], 'module_operations': data['module_operations'] } failure = failure or ('info' in data and data['info'] == 'failure') return { 'function_value': float(1 - np.mean(valid_accuracies)), 'cost': float(np.sum(training_times)), 'info': { 'fidelity': fidelity, 'train_accuracies': train_accuracies, 'valid_accuracies': valid_accuracies, 'test_accuracies': test_accuracies, 'training_times': training_times, 'data': additional, 'failure': 'False' if not failure else 'True' } }