def generate_variants_compatible( unresolved_spec: Dict, constant_grid_search: bool = False, random_state=None ) -> Generator[Tuple[Dict, Dict], None, None]: try: return generate_variants(unresolved_spec, constant_grid_search, random_state) except TypeError: return generate_variants(unresolved_spec, constant_grid_search)
def _generate_resolved_specs(self, num_samples, unresolved_spec): """Needed for slurm_cluster.py """ for _ in range(num_samples): # Iterate over list of configs for unresolved_cfg in unresolved_spec["config"]: unresolved_spec_variant = deepcopy(unresolved_spec) unresolved_spec_variant["config"] = unresolved_cfg resolved_base_vars = CustomVariantGenerator._extract_resolved_base_vars(unresolved_cfg, unresolved_spec["config"]) print("Resolved base cfg vars", resolved_base_vars) for resolved_vars, spec in generate_variants(unresolved_spec_variant): resolved_vars.update(resolved_base_vars) print("Resolved vars", resolved_vars) trial_id = "%05d" % self._counter experiment_tag = str(self._counter) if resolved_vars: experiment_tag += "_{}".format( format_vars({k: v for k, v in resolved_vars.items() if "tag" in k})) self._counter += 1 yield { "spec": spec, "evaluated_params": flatten_resolved_vars(resolved_vars), "trial_id": trial_id, "experiment_tag": experiment_tag }
def _generate_trials(self, unresolved_spec, output_path=""): """Generates Trial objects with the variant generation process. Uses a fixed point iteration to resolve variants. All trials should be able to be generated at once. See also: `ray.tune.suggest.variant_generator`. Yields: Trial object """ if "run" not in unresolved_spec: raise TuneError("Must specify `run` in {}".format(unresolved_spec)) for _ in range(unresolved_spec.get("num_samples", 1)): for resolved_vars, spec in generate_variants(unresolved_spec): experiment_tag = str(self._counter) if resolved_vars: experiment_tag += "_{}".format(resolved_vars) self._counter += 1 yield create_trial_from_spec( spec, output_path, self._parser, evaluated_params=resolved_vars, experiment_tag=experiment_tag)
def _generate_trials(self, unresolved_spec, output_path=""): """Generates Trial objects with the variant generation process. Uses a fixed point iteration to resolve variants. All trials should be able to be generated at once. See also: `ray.tune.suggest.variant_generator`. Yields: Trial object """ if "run" not in unresolved_spec: raise TuneError("Must specify `run` in {}".format(unresolved_spec)) for _ in range(unresolved_spec.get("num_samples", 1)): for resolved_vars, spec in generate_variants(unresolved_spec): experiment_tag = str(self._counter) if resolved_vars: experiment_tag += "_{}".format(resolved_vars) self._counter += 1 yield create_trial_from_spec( spec, output_path, self._parser, experiment_tag=experiment_tag)
def _select(self): choices = self.search_space.to_random_choice() configs = [] for _ in range(self.repeat): for _, config in generate_variants(choices): configs.append(config) return configs, None
def testTuneSampleAPI(self): config = { "func": tune.sample_from(lambda spec: spec.config.uniform * 0.01), "uniform": tune.uniform(-5, -1), "quniform": tune.quniform(3.2, 5.4, 0.2), "loguniform": tune.loguniform(1e-4, 1e-2), "qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-5), "choice": tune.choice([2, 3, 4]), "randint": tune.randint(-9, 15), "qrandint": tune.qrandint(-21, 12, 3), "randn": tune.randn(10, 2), "qrandn": tune.qrandn(10, 2, 0.2), } for _, (_, generated) in zip( range(1000), generate_variants({ "config": config })): out = generated["config"] self.assertAlmostEqual(out["func"], out["uniform"] * 0.01) self.assertGreaterEqual(out["uniform"], -5) self.assertLess(out["uniform"], -1) self.assertGreaterEqual(out["quniform"], 3.2) self.assertLessEqual(out["quniform"], 5.4) self.assertAlmostEqual(out["quniform"] / 0.2, round(out["quniform"] / 0.2)) self.assertGreaterEqual(out["loguniform"], 1e-4) self.assertLess(out["loguniform"], 1e-2) self.assertGreaterEqual(out["qloguniform"], 1e-4) self.assertLessEqual(out["qloguniform"], 1e-1) self.assertAlmostEqual(out["qloguniform"] / 5e-5, round(out["qloguniform"] / 5e-5)) self.assertIn(out["choice"], [2, 3, 4]) self.assertGreaterEqual(out["randint"], -9) self.assertLess(out["randint"], 15) self.assertGreaterEqual(out["qrandint"], -21) self.assertLessEqual(out["qrandint"], 12) self.assertEqual(out["qrandint"] % 3, 0) # Very improbable self.assertGreater(out["randn"], 0) self.assertLess(out["randn"], 20) self.assertGreater(out["qrandn"], 0) self.assertLess(out["qrandn"], 20) self.assertAlmostEqual(out["qrandn"] / 0.2, round(out["qrandn"] / 0.2))
def _generate_trials(self, num_samples, unresolved_spec, output_path="", points_to_evaluate=None): """Generates Trial objects with the variant generation process. Uses a fixed point iteration to resolve variants. All trials should be able to be generated at once. See also: `ray.tune.suggest.variant_generator`. Yields: Trial object """ if "run" not in unresolved_spec: raise TuneError("Must specify `run` in {}".format(unresolved_spec)) points_to_evaluate = points_to_evaluate or [] while points_to_evaluate: config = points_to_evaluate.pop(0) for resolved_vars, spec in get_preset_variants( unresolved_spec, config): trial_id = self._uuid_prefix + ("%05d" % self._counter) experiment_tag = str(self._counter) self._counter += 1 yield create_trial_from_spec( spec, output_path, self._parser, evaluated_params=flatten_resolved_vars(resolved_vars), trial_id=trial_id, experiment_tag=experiment_tag) num_samples -= 1 if num_samples <= 0: return for _ in range(num_samples): for resolved_vars, spec in generate_variants(unresolved_spec): trial_id = self._uuid_prefix + ("%05d" % self._counter) experiment_tag = str(self._counter) if resolved_vars: experiment_tag += "_{}".format(format_vars(resolved_vars)) self._counter += 1 yield create_trial_from_spec( spec, output_path, self._parser, evaluated_params=flatten_resolved_vars(resolved_vars), trial_id=trial_id, experiment_tag=experiment_tag)
def __next__(self): """Generates Trial objects with the variant generation process. Uses a fixed point iteration to resolve variants. All trials should be able to be generated at once. See also: `ray.tune.suggest.variant_generator`. Returns: Trial object """ if "run" not in self.unresolved_spec: raise TuneError("Must specify `run` in {}".format( self.unresolved_spec)) if self.variants and self.variants.has_next(): # This block will be skipped upon instantiation. # `variants` will be set later after the first loop. resolved_vars, spec = next(self.variants) return self.create_trial(resolved_vars, spec) if self.points_to_evaluate: config = self.points_to_evaluate.pop(0) self.num_samples_left -= 1 self.variants = _VariantIterator( get_preset_variants( self.unresolved_spec, config, constant_grid_search=self.constant_grid_search, random_state=self.random_state, ), lazy_eval=self.lazy_eval, ) resolved_vars, spec = next(self.variants) return self.create_trial(resolved_vars, spec) elif self.num_samples_left > 0: self.variants = _VariantIterator( generate_variants( self.unresolved_spec, constant_grid_search=self.constant_grid_search, random_state=self.random_state, ), lazy_eval=self.lazy_eval, ) self.num_samples_left -= 1 resolved_vars, spec = next(self.variants) return self.create_trial(resolved_vars, spec) else: raise StopIteration
def complete_config(self, partial_config: Dict, lower: Optional[Dict] = None, upper: Optional[Dict] = None) -> Dict: ''' generate a complete config from the partial config input add minimal resource to config if available ''' if self._reset_times and partial_config == self.init_config: # not the first time to complete init_config, use random gaussian normalized = self.normalize(partial_config) for key in normalized: # don't change unordered cat choice if key not in self._unordered_cat_hp: if upper and lower: u, l = upper[key], lower[key] gauss_std = u - l or self.STEPSIZE # allowed bound u += self.STEPSIZE l -= self.STEPSIZE elif key in self._bounded_keys: u, l, gauss_std = 1, 0, 1.0 else: u, l, gauss_std = np.Inf, -np.Inf, 1.0 if key in self._bounded_keys: u = min(u, 1) l = max(l, 0) delta = self.rand_vector_gaussian(1, gauss_std)[0] normalized[key] = max(l, min(u, normalized[key] + delta)) # use best config for unordered cat choice config = self.denormalize(normalized) else: # first time init_config, or other configs, take as is config = partial_config.copy() if partial_config == self.init_config: self._reset_times += 1 config = flatten_dict(config) for key, value in self.space.items(): if key not in config: config[key] = value # logger.debug(f'before random {config}') for _, generated in generate_variants({'config': config}): config = generated['config'] break # logger.debug(f'after random {config}') if self._resource: config[self.prune_attr] = self.min_resource return unflatten_dict(config)
def generate_trial_variants(config): """ Generate configuration for each trial variant evaluating 'ray.tune' functions (grid_search, sample_from, ...) into its final values. :param config: Ray tune configuration with 'ray.tune' functions :return: list of dict for each trial configuration variant """ trials = [] num_samples = config["num_samples"] for i in range(num_samples): for variables, variant in generate_variants(config): # Update experiment tag with variant vars if len(variables) > 0: variant["experiment_tag"] = f"{i}_{format_vars(variables)}" else: variant["experiment_tag"] = str(i) trials.append(variant) return trials
def _generate_trials(self, num_samples, unresolved_spec, output_path=""): """Generates Trial objects with the variant generation process. Uses a fixed point iteration to resolve variants. All trials should be able to be generated at once. See also: `ray.tune.suggest.variant_generator`. Yields: Trial object """ if "run" not in unresolved_spec: raise TuneError("Must specify `run` in {}".format(unresolved_spec)) for _ in range(num_samples): # Iterate over list of configs for unresolved_cfg in unresolved_spec["config"]: unresolved_spec_variant = deepcopy(unresolved_spec) unresolved_spec_variant["config"] = unresolved_cfg resolved_base_vars = CustomVariantGenerator._extract_resolved_base_vars(unresolved_cfg, unresolved_spec["config"]) print("Resolved base cfg vars", resolved_base_vars) for resolved_vars, spec in generate_variants(unresolved_spec_variant): resolved_vars.update(resolved_base_vars) print("Resolved vars", resolved_vars) trial_id = "%05d" % self._counter experiment_tag = str(self._counter) if resolved_vars: experiment_tag += "_{}".format( format_vars({k: v for k, v in resolved_vars.items() if "tag" in k})) self._counter += 1 yield create_trial_from_spec( spec, output_path, self._parser, evaluated_params=flatten_resolved_vars(resolved_vars), trial_id=trial_id, experiment_tag=experiment_tag)
def _generate_trials(self, num_samples, unresolved_spec, output_path=""): """Generates Trial objects with the variant generation process. Uses a fixed point iteration to resolve variants. All trials should be able to be generated at once. See also: `ray.tune.suggest.variant_generator`. Yields: Trial object """ if "run" not in unresolved_spec: raise TuneError("Must specify `run` in {}".format(unresolved_spec)) for _ in range(num_samples): for resolved_vars, spec in generate_variants(unresolved_spec): while True: if self._num_live_trials() >= self._max_concurrent: yield None else: break trial_id = "%05d" % self._counter experiment_tag = str(self._counter) if resolved_vars: experiment_tag += "_{}".format(format_vars(resolved_vars)) self._counter += 1 self._live_trials.add(trial_id) yield create_trial_from_spec( spec, output_path, self._parser, evaluated_params=flatten_resolved_vars(resolved_vars), trial_id=trial_id, experiment_tag=experiment_tag, )
def _select(self): grid = self.search_space.to_grid_search() configs = [] for _, config in generate_variants(grid): configs.append(config) return configs, None
def suggest(self, trial_id: str) -> Optional[Dict]: ''' choose thread, suggest a valid config ''' if self._init_used and not self._points_to_evaluate: choice, backup = self._select_thread() # logger.debug(f"choice={choice}, backup={backup}") if choice < 0: return None # timeout self._use_rs = False config = self._search_thread_pool[choice].suggest(trial_id) skip = self._should_skip(choice, trial_id, config) if skip: if choice: # logger.info(f"skipping choice={choice}, config={config}") return None # use rs self._use_rs = True for _, generated in generate_variants( {'config': self._ls.space}): config = generated['config'] break # logger.debug(f"random config {config}") skip = self._should_skip(choice, trial_id, config) if skip: return None # if not choice: logger.info(config) if choice or backup == choice or self._valid(config): # LS or valid or no backup choice self._trial_proposed_by[trial_id] = choice else: # invalid config proposed by GS if not self._use_rs: self._search_thread_pool[choice].on_trial_complete( trial_id, {}, error=True) # tell GS there is an error self._use_rs = False config = self._search_thread_pool[backup].suggest(trial_id) skip = self._should_skip(backup, trial_id, config) if skip: return None self._trial_proposed_by[trial_id] = backup choice = backup # if choice: self._pending.add(choice) # local search thread pending if not choice: if self._ls._resource: # TODO: add resource to config proposed by GS, min or median? config[self._ls.prune_attr] = self._ls.min_resource self._result[self._ls.config_signature(config)] = {} else: # use init config init_config = self._points_to_evaluate.pop( 0) if self._points_to_evaluate else self._ls.init_config config = self._ls.complete_config(init_config, self._admissible_min, self._admissible_max) # logger.info(f"reset config to {config}") config_signature = self._ls.config_signature(config) result = self._result.get(config_signature) if result: # tried before # self.on_trial_complete(trial_id, result) return None elif result is None: # not tried before self._result[config_signature] = {} else: return None # running but no result yet self._init_used = True # logger.info(f"config={config}") return config
def suggest(self, trial_id: str) -> Optional[Dict]: ''' choose thread, suggest a valid config ''' if self._init_used and not self._points_to_evaluate: choice, backup = self._select_thread() # print(f"choice={choice}, backup={backup}") if choice < 0: return None # timeout self._use_rs = False config = self._search_thread_pool[choice].suggest(trial_id) # preliminary check; not checking config validation skip = self._should_skip(choice, trial_id, config) if skip: if choice: # print(f"skipping choice={choice}, config={config}") return None # use rs when BO fails to suggest a config self._use_rs = True for _, generated in generate_variants( {'config': self._ls.space}): config = generated['config'] break # get one random config # logger.debug(f"random config {config}") skip = self._should_skip(choice, trial_id, config) if skip: return None # if not choice: print(config) if choice or self._valid(config): # LS or valid or no backup choice self._trial_proposed_by[trial_id] = choice else: # invalid config proposed by GS # if not self._use_rs: # self._search_thread_pool[choice].on_trial_complete( # trial_id, {}, error=True) # tell GS there is an error self._use_rs = False if choice == backup: # use CFO's init point init_config = self._ls.init_config config = self._ls.complete_config(init_config, self._ls_bound_min, self._ls_bound_max) self._trial_proposed_by[trial_id] = choice else: config = self._search_thread_pool[backup].suggest(trial_id) skip = self._should_skip(backup, trial_id, config) if skip: return None self._trial_proposed_by[trial_id] = backup choice = backup if not choice: # global search if self._ls._resource: # TODO: add resource to config proposed by GS, min or median? config[self._ls.prune_attr] = self._ls.min_resource # temporarily relax admissible region for parallel proposals self._update_admissible_region(config, self._gs_admissible_min, self._gs_admissible_max) else: self._update_admissible_region(config, self._ls_bound_min, self._ls_bound_max) self._gs_admissible_min.update(self._ls_bound_min) self._gs_admissible_max.update(self._ls_bound_max) self._result[self._ls.config_signature(config)] = {} else: # use init config # print("use init config") init_config = self._points_to_evaluate.pop( 0) if self._points_to_evaluate else self._ls.init_config config = self._ls.complete_config(init_config, self._ls_bound_min, self._ls_bound_max) # logger.info(f"reset config to {config}") config_signature = self._ls.config_signature(config) result = self._result.get(config_signature) if result: # tried before # self.on_trial_complete(trial_id, result) return None elif result is None: # not tried before self._result[config_signature] = {} else: return None # running but no result yet self._init_used = True self._trial_proposed_by[trial_id] = 0 # logger.info(f"config={config}") return config
def train_model_on_task(self, task, task_viz, exp_dir, use_ray, use_ray_logging, grace_period, num_hp_samplings, local_mode, redis_address, lca_n, **training_params): logger.info("Training dashboard: {}".format(get_env_url(task_viz))) t_id = task['id'] trainable = self.get_trainable(use_ray_logging=use_ray_logging) past_tasks = training_params.pop('past_tasks') normalize = training_params.pop('normalize') augment_data = training_params.pop('augment_data') transformations = [] if augment_data: transformations.extend([ transforms.ToPILImage(), transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, 4), transforms.ToTensor() ]) t_trans = [[] for _ in range(len(task['split_names']))] t_trans[0] = transformations datasets = trainable._load_datasets(task, task['loss_fn'], past_tasks, t_trans, normalize) train_loader, eval_loaders = get_classic_dataloaders(datasets, training_params.pop( 'batch_sizes')) model = self.get_model(task_id=t_id, x_dim=task['x_dim'], n_classes=task['n_classes'], descriptor=task['descriptor'], dataset=eval_loaders[:2]) if use_ray: if not ray.is_initialized(): ray.init(address=redis_address) scheduler = None training_params['loss_fn'] = tune.function( training_params['loss_fn']) training_params['optim_func'] = tune.function(self.optim_func) init_model_path = os.path.join(exp_dir, 'model_initializations') model_file_name = '{}_init.pth'.format(training_params['name']) model_path = os.path.join(init_model_path, model_file_name) torch.save(model, model_path) training_params['model_path'] = model_path config = {**self.get_search_space(), 'training-params': training_params} if use_ray_logging: stop_condition = {'training_iteration': training_params['n_it_max']} checkpoint_at_end = False keep_checkpoints_num = 1 checkpoint_score_attr = 'min-Val nll' else: stop_condition = None # loggers = [JsonLogger, MyCSVLogger] checkpoint_at_end = False keep_checkpoints_num = None checkpoint_score_attr = None trainable = rename_class(trainable, training_params['name']) experiment = Experiment( name=training_params['name'], run=trainable, stop=stop_condition, config=config, resources_per_trial=self.ray_resources, num_samples=num_hp_samplings, local_dir=exp_dir, loggers=(JsonLogger, CSVLogger), checkpoint_at_end=checkpoint_at_end, keep_checkpoints_num=keep_checkpoints_num, checkpoint_score_attr=checkpoint_score_attr) analysis = tune.run(experiment, scheduler=scheduler, verbose=1, raise_on_failed_trial=True, # max_failures=-1, # with_server=True, # server_port=4321 ) os.remove(model_path) logger.info("Training dashboard: {}".format(get_env_url(task_viz))) all_trials = {t.logdir: t for t in analysis.trials} best_logdir = analysis.get_best_logdir('Val nll', 'min') best_trial = all_trials[best_logdir] # picked_metric = 'accuracy_0' # metric_names = {s: '{} {}'.format(s, picked_metric) for s in # ['Train', 'Val', 'Test']} logger.info('Best trial: {}'.format(best_trial)) best_res = best_trial.checkpoint.result best_point = (best_res['training_iteration'], best_res['Val nll']) # y_keys = ['mean_loss' if use_ray_logging else 'Val nll', 'train_loss'] y_keys = ['Val nll', 'Train nll'] epoch_key = 'training_epoch' it_key = 'training_iteration' plot_res_dataframe(analysis, training_params['name'], best_point, task_viz, epoch_key, it_key, y_keys) if 'entropy' in next(iter(analysis.trial_dataframes.values())): plot_res_dataframe(analysis, training_params['name'], None, task_viz, epoch_key, it_key, ['entropy']) best_model = self.get_model(task_id=t_id) best_model.load_state_dict(torch.load(best_trial.checkpoint.value)) train_accs = analysis.trial_dataframes[best_logdir]['Train accuracy_0'] best_t = best_res['training_iteration'] t = best_trial.last_result['training_iteration'] else: search_space = self.get_search_space() rand_config = list(generate_variants(search_space))[0][1] learner_params = rand_config.pop('learner-params', {}) optim_params = rand_config.pop('optim') split_optims = training_params.pop('split_optims') if hasattr(model, 'set_h_params'): model.set_h_params(**learner_params) if hasattr(model, 'train_loader_wrapper'): train_loader = model.train_loader_wrapper(train_loader) loss_fn = task['loss_fn'] if hasattr(model, 'loss_wrapper'): loss_fn = model.loss_wrapper(task['loss_fn']) prepare_batch = _prepare_batch if hasattr(model, 'prepare_batch_wrapper'): prepare_batch = model.prepare_batch_wrapper(prepare_batch, t_id) optim_fact = partial(set_optim_params, optim_func=self.optim_func, optim_params=optim_params, split_optims=split_optims) if hasattr(model, 'train_func'): f = model.train_func t, metrics, b_state_dict = f(train_loader=train_loader, eval_loaders=eval_loaders, optim_fact=optim_fact, loss_fn=loss_fn, split_names=task['split_names'], viz=task_viz, prepare_batch=prepare_batch, **training_params) else: optim = optim_fact(model=model) t, metrics, b_state_dict = train(model=model, train_loader=train_loader, eval_loaders=eval_loaders, optimizer=optim, loss_fn=loss_fn, split_names=task['split_names'], viz=task_viz, prepare_batch=prepare_batch, **training_params) train_accs = metrics['Train accuracy_0'] best_t = b_state_dict['iter'] if 'training_archs' in metrics: plot_trajectory(model.ssn.graph, metrics['training_archs'], model.ssn.stochastic_node_ids, task_viz) weights = model.arch_sampler().squeeze() archs = model.ssn.get_top_archs(weights, 5) list_top_archs(archs, task_viz) list_arch_scores(self.arch_scores[t_id], task_viz) update_summary(self.arch_scores[t_id], task_viz, 'scores') if len(train_accs) > lca_n: lca_accs = [] for i in range(lca_n + 1): if i in train_accs: lca_accs.append(train_accs[i]) else: logger.warning('Missing step for {}/{} for lca computation' .format(i, lca_n)) lca = np.mean(lca_accs) else: lca = np.float('nan') stats = {} start = time.time() # train_idx = task['split_names'].index('Train') # train_path = task['data_path'][train_idx] # train_dataset = _load_datasets([train_path])[0] train_dataset = _load_datasets(task, 'Train')[0] stats.update(self.finish_task(train_dataset, t_id, task_viz, path='drawings')) stats['duration'] = {'iterations': t, 'finish': time.time() - start, 'best_iterations': best_t} stats['params'] = {'total': self.n_params(t_id), 'new': self.new_params(t_id)} stats['lca'] = lca return stats
def suggest(self, trial_id: str) -> Optional[Dict]: ''' choose thread, suggest a valid config ''' if self._init_used and not self._points_to_evaluate: choice, backup = self._select_thread() if choice < 0: # timeout return None self._use_rs = False config = self._search_thread_pool[choice].suggest(trial_id) if choice and config is None: # local search thread finishes if self._search_thread_pool[choice].converged: self._expand_admissible_region() del self._search_thread_pool[choice] return None # preliminary check; not checking config validation skip = self._should_skip(choice, trial_id, config) if skip: if choice: return None # use rs when BO fails to suggest a config self._use_rs = True for _, generated in generate_variants({'config': self._ls.space}): config = generated['config'] break # get one random config skip = self._should_skip(choice, trial_id, config) if skip: return None if choice or self._valid(config): # LS or valid or no backup choice self._trial_proposed_by[trial_id] = choice else: # invalid config proposed by GS self._use_rs = False if choice == backup: # use CFO's init point init_config = self._ls.init_config config = self._ls.complete_config( init_config, self._ls_bound_min, self._ls_bound_max) self._trial_proposed_by[trial_id] = choice else: config = self._search_thread_pool[backup].suggest(trial_id) skip = self._should_skip(backup, trial_id, config) if skip: return None self._trial_proposed_by[trial_id] = backup choice = backup if not choice: # global search if self._ls._resource: # TODO: min or median? config[self._ls.prune_attr] = self._ls.min_resource # temporarily relax admissible region for parallel proposals self._update_admissible_region( config, self._gs_admissible_min, self._gs_admissible_max) else: self._update_admissible_region( config, self._ls_bound_min, self._ls_bound_max) self._gs_admissible_min.update(self._ls_bound_min) self._gs_admissible_max.update(self._ls_bound_max) self._result[self._ls.config_signature(config)] = {} else: # use init config init_config = self._points_to_evaluate.pop( 0) if self._points_to_evaluate else self._ls.init_config config = self._ls.complete_config( init_config, self._ls_bound_min, self._ls_bound_max) config_signature = self._ls.config_signature(config) result = self._result.get(config_signature) if result: # tried before return None elif result is None: # not tried before self._result[config_signature] = {} else: # running but no result yet return None self._init_used = True self._trial_proposed_by[trial_id] = 0 return config