class BlendSearch(Searcher):
'''class for BlendSearch algorithm
'''
cost_attr = "time_total_s" # cost attribute in result
lagrange = '_lagrange' # suffix for lagrange-modified metric
penalty = 1e+10 # penalty term for constraints
def __init__(self,
metric: Optional[str] = None,
mode: Optional[str] = None,
space: Optional[dict] = None,
points_to_evaluate: Optional[List[dict]] = None,
low_cost_partial_config: Optional[dict] = None,
cat_hp_cost: Optional[dict] = None,
prune_attr: Optional[str] = None,
min_resource: Optional[float] = None,
max_resource: Optional[float] = None,
reduction_factor: Optional[float] = None,
global_search_alg: Optional[Searcher] = None,
config_constraints: Optional[
List[Tuple[Callable[[dict], float], str, float]]] = None,
metric_constraints: Optional[
List[Tuple[str, str, float]]] = None,
seed: Optional[int] = 20):
'''Constructor
Args:
metric: A string of the metric name to optimize for.
mode: A string in ['min', 'max'] to specify the objective as
minimization or maximization.
space: A dictionary to specify the search space.
points_to_evaluate: Initial parameter suggestions to be run first.
low_cost_partial_config: A dictionary from a subset of
controlled dimensions to the initial low-cost values.
e.g.,
.. code-block:: python
{'n_estimators': 4, 'max_leaves': 4}
cat_hp_cost: A dictionary from a subset of categorical dimensions
to the relative cost of each choice.
e.g.,
.. code-block:: python
{'tree_method': [1, 1, 2]}
i.e., the relative cost of the
three choices of 'tree_method' is 1, 1 and 2 respectively.
prune_attr: A string of the attribute used for pruning.
Not necessarily in space.
When prune_attr is in space, it is a hyperparameter, e.g.,
'n_iters', and the best value is unknown.
When prune_attr is not in space, it is a resource dimension,
e.g., 'sample_size', and the peak performance is assumed
to be at the max_resource.
min_resource: A float of the minimal resource to use for the
prune_attr; only valid if prune_attr is not in space.
max_resource: A float of the maximal resource to use for the
prune_attr; only valid if prune_attr is not in space.
reduction_factor: A float of the reduction factor used for
incremental pruning.
global_search_alg: A Searcher instance as the global search
instance. If omitted, Optuna is used. The following algos have
known issues when used as global_search_alg:
- HyperOptSearch raises exception sometimes
- TuneBOHB has its own scheduler
config_constraints: A list of config constraints to be satisfied.
e.g.,
.. code-block: python
config_constraints = [(mem_size, '<=', 1024**3)]
mem_size is a function which produces a float number for the bytes
needed for a config.
It is used to skip configs which do not fit in memory.
metric_constraints: A list of metric constraints to be satisfied.
e.g., `['precision', '>=', 0.9]`
seed: An integer of the random seed.
'''
self._metric, self._mode = metric, mode
init_config = low_cost_partial_config or {}
if not init_config:
logger.warning(
"No low-cost partial config given to the search algorithm. "
"For cost-frugal search, "
"consider providing low-cost values for cost-related hps via "
"'low_cost_partial_config'."
)
self._points_to_evaluate = points_to_evaluate or []
self._config_constraints = config_constraints
self._metric_constraints = metric_constraints
if self._metric_constraints:
# metric modified by lagrange
metric += self.lagrange
if global_search_alg is not None:
self._gs = global_search_alg
elif getattr(self, '__name__', None) != 'CFO':
self._gs = GlobalSearch(space=space, metric=metric, mode=mode)
else:
self._gs = None
self._ls = LocalSearch(
init_config, metric, mode, cat_hp_cost, space,
prune_attr, min_resource, max_resource, reduction_factor, seed)
self._init_search()
def set_search_properties(self,
metric: Optional[str] = None,
mode: Optional[str] = None,
config: Optional[Dict] = None) -> bool:
if self._ls.space:
if 'time_budget_s' in config:
self._deadline = config.get('time_budget_s') + time.time()
if 'metric_target' in config:
self._metric_target = config.get('metric_target')
else:
if metric:
self._metric = metric
if self._metric_constraints:
# metric modified by lagrange
metric += self.lagrange
# TODO: don't change metric for global search methods that
# can handle constraints already
if mode:
self._mode = mode
self._ls.set_search_properties(metric, mode, config)
if self._gs is not None:
self._gs.set_search_properties(metric, mode, config)
self._init_search()
return True
def _init_search(self):
'''initialize the search
'''
self._metric_target = np.inf * self._ls.metric_op
self._search_thread_pool = {
# id: int -> thread: SearchThread
0: SearchThread(self._ls.mode, self._gs)
}
self._thread_count = 1 # total # threads created
self._init_used = self._ls.init_config is None
self._trial_proposed_by = {} # trial_id: str -> thread_id: int
self._ls_bound_min = self._ls.normalize(self._ls.init_config)
self._ls_bound_max = self._ls_bound_min.copy()
self._gs_admissible_min = self._ls_bound_min.copy()
self._gs_admissible_max = self._ls_bound_max.copy()
self._result = {} # config_signature: tuple -> result: Dict
self._deadline = np.inf
if self._metric_constraints:
self._metric_constraint_satisfied = False
self._metric_constraint_penalty = [
self.penalty for _ in self._metric_constraints]
else:
self._metric_constraint_satisfied = True
self._metric_constraint_penalty = None
def save(self, checkpoint_path: str):
save_object = self
with open(checkpoint_path, "wb") as outputFile:
pickle.dump(save_object, outputFile)
def restore(self, checkpoint_path: str):
with open(checkpoint_path, "rb") as inputFile:
state = pickle.load(inputFile)
self._metric_target = state._metric_target
self._search_thread_pool = state._search_thread_pool
self._thread_count = state._thread_count
self._init_used = state._init_used
self._trial_proposed_by = state._trial_proposed_by
self._ls_bound_min = state._ls_bound_min
self._ls_bound_max = state._ls_bound_max
self._gs_admissible_min = state._gs_admissible_min
self._gs_admissible_max = state._gs_admissible_max
self._result = state._result
self._deadline = state._deadline
self._metric, self._mode = state._metric, state._mode
self._points_to_evaluate = state._points_to_evaluate
self._gs = state._gs
self._ls = state._ls
self._config_constraints = state._config_constraints
self._metric_constraints = state._metric_constraints
self._metric_constraint_satisfied = state._metric_constraint_satisfied
self._metric_constraint_penalty = state._metric_constraint_penalty
@property
def metric_target(self):
return self._metric_target
def restore_from_dir(self, checkpoint_dir: str):
super.restore_from_dir(checkpoint_dir)
def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None,
error: bool = False):
''' search thread updater and cleaner
'''
metric_constraint_satisfied = True
if result and not error and self._metric_constraints:
# account for metric constraints if any
objective = result[self._metric]
for i, constraint in enumerate(self._metric_constraints):
metric_constraint, sign, threshold = constraint
value = result.get(metric_constraint)
if value:
# sign is <= or >=
sign_op = 1 if sign == '<=' else -1
violation = (value - threshold) * sign_op
if violation > 0:
# add penalty term to the metric
objective += self._metric_constraint_penalty[
i] * violation * self._ls.metric_op
metric_constraint_satisfied = False
if self._metric_constraint_penalty[i] < self.penalty:
self._metric_constraint_penalty[i] += violation
result[self._metric + self.lagrange] = objective
if metric_constraint_satisfied and not self._metric_constraint_satisfied:
# found a feasible point
self._metric_constraint_penalty = [1 for _ in self._metric_constraints]
self._metric_constraint_satisfied |= metric_constraint_satisfied
thread_id = self._trial_proposed_by.get(trial_id)
if thread_id in self._search_thread_pool:
self._search_thread_pool[thread_id].on_trial_complete(
trial_id, result, error)
del self._trial_proposed_by[trial_id]
if result:
config = {}
for key, value in result.items():
if key.startswith('config/'):
config[key[7:]] = value
if error: # remove from result cache
del self._result[self._ls.config_signature(config)]
else: # add to result cache
self._result[self._ls.config_signature(config)] = result
# update target metric if improved
objective = result[
self._metric + self.lagrange] if self._metric_constraints \
else result[self._metric]
if (objective - self._metric_target) * self._ls.metric_op < 0:
self._metric_target = objective
if not thread_id and metric_constraint_satisfied \
and self._create_condition(result):
# thread creator
self._search_thread_pool[self._thread_count] = SearchThread(
self._ls.mode,
self._ls.create(
config, objective, cost=result[self.cost_attr])
)
thread_id = self._thread_count
self._thread_count += 1
self._update_admissible_region(
config, self._ls_bound_min, self._ls_bound_max)
elif thread_id and not self._metric_constraint_satisfied:
# no point has been found to satisfy metric constraint
self._expand_admissible_region()
# reset admissible region to ls bounding box
self._gs_admissible_min.update(self._ls_bound_min)
self._gs_admissible_max.update(self._ls_bound_max)
# cleaner
if thread_id and thread_id in self._search_thread_pool:
# local search thread
self._clean(thread_id)
def _update_admissible_region(self, config, admissible_min, admissible_max):
# update admissible region
normalized_config = self._ls.normalize(config)
for key in admissible_min:
value = normalized_config[key]
if value > admissible_max[key]:
admissible_max[key] = value
elif value < admissible_min[key]:
admissible_min[key] = value
def _create_condition(self, result: Dict) -> bool:
''' create thread condition
'''
if len(self._search_thread_pool) < 2:
return True
obj_median = np.median(
[thread.obj_best1 for id, thread in self._search_thread_pool.items()
if id])
return result[self._metric] * self._ls.metric_op < obj_median
def _clean(self, thread_id: int):
''' delete thread and increase admissible region if converged,
merge local threads if they are close
'''
assert thread_id
todelete = set()
for id in self._search_thread_pool:
if id and id != thread_id:
if self._inferior(id, thread_id):
todelete.add(id)
for id in self._search_thread_pool:
if id and id != thread_id:
if self._inferior(thread_id, id):
todelete.add(thread_id)
break
if self._search_thread_pool[thread_id].converged:
todelete.add(thread_id)
self._expand_admissible_region()
for id in todelete:
del self._search_thread_pool[id]
def _expand_admissible_region(self):
for key in self._ls_bound_max:
self._ls_bound_max[key] += self._ls.STEPSIZE
self._ls_bound_min[key] -= self._ls.STEPSIZE
def _inferior(self, id1: int, id2: int) -> bool:
''' whether thread id1 is inferior to id2
'''
t1 = self._search_thread_pool[id1]
t2 = self._search_thread_pool[id2]
if t1.obj_best1 < t2.obj_best2:
return False
elif t1.resource and t1.resource < t2.resource:
return False
elif t2.reach(t1):
return True
return False
def on_trial_result(self, trial_id: str, result: Dict):
if trial_id not in self._trial_proposed_by:
return
thread_id = self._trial_proposed_by[trial_id]
if thread_id not in self._search_thread_pool:
return
if result and self._metric_constraints:
result[self._metric + self.lagrange] = result[self._metric]
self._search_thread_pool[thread_id].on_trial_result(trial_id, result)
def suggest(self, trial_id: str) -> Optional[Dict]:
''' choose thread, suggest a valid config
'''
if self._init_used and not self._points_to_evaluate:
choice, backup = self._select_thread()
if choice < 0: # timeout
return None
self._use_rs = False
config = self._search_thread_pool[choice].suggest(trial_id)
if choice and config is None:
# local search thread finishes
if self._search_thread_pool[choice].converged:
self._expand_admissible_region()
del self._search_thread_pool[choice]
return None
# preliminary check; not checking config validation
skip = self._should_skip(choice, trial_id, config)
if skip:
if choice:
return None
# use rs when BO fails to suggest a config
self._use_rs = True
for _, generated in generate_variants({'config': self._ls.space}):
config = generated['config']
break # get one random config
skip = self._should_skip(choice, trial_id, config)
if skip:
return None
if choice or self._valid(config):
# LS or valid or no backup choice
self._trial_proposed_by[trial_id] = choice
else: # invalid config proposed by GS
self._use_rs = False
if choice == backup:
# use CFO's init point
init_config = self._ls.init_config
config = self._ls.complete_config(
init_config, self._ls_bound_min, self._ls_bound_max)
self._trial_proposed_by[trial_id] = choice
else:
config = self._search_thread_pool[backup].suggest(trial_id)
skip = self._should_skip(backup, trial_id, config)
if skip:
return None
self._trial_proposed_by[trial_id] = backup
choice = backup
if not choice: # global search
if self._ls._resource:
# TODO: min or median?
config[self._ls.prune_attr] = self._ls.min_resource
# temporarily relax admissible region for parallel proposals
self._update_admissible_region(
config, self._gs_admissible_min, self._gs_admissible_max)
else:
self._update_admissible_region(
config, self._ls_bound_min, self._ls_bound_max)
self._gs_admissible_min.update(self._ls_bound_min)
self._gs_admissible_max.update(self._ls_bound_max)
self._result[self._ls.config_signature(config)] = {}
else: # use init config
init_config = self._points_to_evaluate.pop(
0) if self._points_to_evaluate else self._ls.init_config
config = self._ls.complete_config(
init_config, self._ls_bound_min, self._ls_bound_max)
config_signature = self._ls.config_signature(config)
result = self._result.get(config_signature)
if result: # tried before
return None
elif result is None: # not tried before
self._result[config_signature] = {}
else: # running but no result yet
return None
self._init_used = True
self._trial_proposed_by[trial_id] = 0
return config
def _should_skip(self, choice, trial_id, config) -> bool:
''' if config is None or config's result is known or constraints are violated
return True; o.w. return False
'''
if config is None:
return True
config_signature = self._ls.config_signature(config)
exists = config_signature in self._result
# check constraints
if not exists and self._config_constraints:
for constraint in self._config_constraints:
func, sign, threshold = constraint
value = func(config)
if (sign == '<=' and value > threshold
or sign == '>=' and value < threshold):
self._result[config_signature] = {
self._metric: np.inf * self._ls.metric_op,
'time_total_s': 1,
}
exists = True
break
if exists:
if not self._use_rs:
result = self._result.get(config_signature)
if result:
self._search_thread_pool[choice].on_trial_complete(
trial_id, result, error=False)
if choice:
# local search thread
self._clean(choice)
# else:
# # tell the thread there is an error
# self._search_thread_pool[choice].on_trial_complete(
# trial_id, {}, error=True)
return True
return False
def _select_thread(self) -> Tuple:
''' thread selector; use can_suggest to check LS availability
'''
# update priority
min_eci = self._deadline - time.time()
if min_eci <= 0:
return -1, -1
max_speed = 0
for thread in self._search_thread_pool.values():
if thread.speed > max_speed:
max_speed = thread.speed
for thread in self._search_thread_pool.values():
thread.update_eci(self._metric_target, max_speed)
if thread.eci < min_eci:
min_eci = thread.eci
for thread in self._search_thread_pool.values():
thread.update_priority(min_eci)
top_thread_id = backup_thread_id = 0
priority1 = priority2 = self._search_thread_pool[0].priority
for thread_id, thread in self._search_thread_pool.items():
# if thread_id:
# print(
# f"priority of thread {thread_id}={thread.priority}")
# logger.debug(
# f"thread {thread_id}.can_suggest={thread.can_suggest}")
if thread_id and thread.can_suggest:
priority = thread.priority
if priority > priority1:
priority1 = priority
top_thread_id = thread_id
if priority > priority2 or backup_thread_id == 0:
priority2 = priority
backup_thread_id = thread_id
return top_thread_id, backup_thread_id
def _valid(self, config: Dict) -> bool:
''' config validator
'''
normalized_config = self._ls.normalize(config)
for key in self._gs_admissible_min:
if key in config:
value = normalized_config[key]
if value + self._ls.STEPSIZE < self._gs_admissible_min[key] \
or value > self._gs_admissible_max[key] + self._ls.STEPSIZE:
return False
return True
class BlendSearch(Searcher):
'''class for BlendSearch algorithm
'''
def __init__(self,
metric: Optional[str] = None,
mode: Optional[str] = None,
space: Optional[dict] = None,
points_to_evaluate: Optional[List[Dict]] = None,
cat_hp_cost: Optional[dict] = None,
prune_attr: Optional[str] = None,
min_resource: Optional[float] = None,
max_resource: Optional[float] = None,
reduction_factor: Optional[float] = None,
resources_per_trial: Optional[dict] = None,
global_search_alg: Optional[Searcher] = None,
mem_size=None):
'''Constructor
Args:
metric: A string of the metric name to optimize for.
minimization or maximization.
mode: A string in ['min', 'max'] to specify the objective as
space: A dictionary to specify the search space.
points_to_evaluate: Initial parameter suggestions to be run first.
The first element needs to be a dictionary from a subset of
controlled dimensions to the initial low-cost values.
e.g.,
.. code-block:: python
[{'epochs': 1}]
cat_hp_cost: A dictionary from a subset of categorical dimensions
to the relative cost of each choice.
e.g.,
.. code-block:: python
{'tree_method': [1, 1, 2]}
i.e., the relative cost of the
three choices of 'tree_method' is 1, 1 and 2 respectively.
prune_attr: A string of the attribute used for pruning.
Not necessarily in space.
When prune_attr is in space, it is a hyperparameter, e.g.,
'n_iters', and the best value is unknown.
When prune_attr is not in space, it is a resource dimension,
e.g., 'sample_size', and the peak performance is assumed
to be at the max_resource.
min_resource: A float of the minimal resource to use for the
prune_attr; only valid if prune_attr is not in space.
max_resource: A float of the maximal resource to use for the
prune_attr; only valid if prune_attr is not in space.
reduction_factor: A float of the reduction factor used for
incremental pruning.
resources_per_trial: A dictionary of the resources permitted per
trial, such as 'mem'.
global_search_alg: A Searcher instance as the global search
instance. If omitted, Optuna is used. The following algos have
known issues when used as global_search_alg:
- HyperOptSearch raises exception sometimes
- TuneBOHB has its own scheduler
mem_size: A function to estimate the memory size for a given config.
'''
self._metric, self._mode = metric, mode
if points_to_evaluate: init_config = points_to_evaluate[0]
else: init_config = {}
self._points_to_evaluate = points_to_evaluate
if global_search_alg is not None:
self._gs = global_search_alg
elif getattr(self, '__name__', None) != 'CFO':
self._gs = GlobalSearch(space=space, metric=metric, mode=mode)
else:
self._gs = None
self._ls = LocalSearch(init_config, metric, mode, cat_hp_cost, space,
prune_attr, min_resource, max_resource,
reduction_factor)
self._resources_per_trial = resources_per_trial
self._mem_size = mem_size
self._mem_threshold = resources_per_trial.get(
'mem') if resources_per_trial else None
self._init_search()
def set_search_properties(self,
metric: Optional[str] = None,
mode: Optional[str] = None,
config: Optional[Dict] = None) -> bool:
if self._ls.space:
if 'time_budget_s' in config:
self._deadline = config.get('time_budget_s') + time.time()
if 'metric_target' in config:
self._metric_target = config.get('metric_target')
else:
if metric: self._metric = metric
if mode: self._mode = mode
self._ls.set_search_properties(metric, mode, config)
if self._gs is not None:
self._gs.set_search_properties(metric, mode, config)
self._init_search()
return True
def _init_search(self):
'''initialize the search
'''
self._metric_target = np.inf * self._ls.metric_op
self._search_thread_pool = {
# id: int -> thread: SearchThread
0: SearchThread(self._ls.mode, self._gs)
}
self._thread_count = 1 # total # threads created
self._init_used = self._ls.init_config is None
self._trial_proposed_by = {} # trial_id: str -> thread_id: int
self._admissible_min = self._ls.normalize(self._ls.init_config)
self._admissible_max = self._admissible_min.copy()
self._result = {} # config_signature: tuple -> result: Dict
self._deadline = np.inf
def save(self, checkpoint_path: str):
save_object = (self._metric_target, self._search_thread_pool,
self._thread_count, self._init_used,
self._trial_proposed_by, self._admissible_min,
self._admissible_max, self._result, self._deadline)
with open(checkpoint_path, "wb") as outputFile:
pickle.dump(save_object, outputFile)
def restore(self, checkpoint_path: str):
with open(checkpoint_path, "rb") as inputFile:
save_object = pickle.load(inputFile)
self._metric_target, self._search_thread_pool, \
self._thread_count, self._init_used, self._trial_proposed_by, \
self._admissible_min, self._admissible_max, self._result, \
self._deadline = save_object
def restore_from_dir(self, checkpoint_dir: str):
super.restore_from_dir(checkpoint_dir)
def on_trial_complete(self,
trial_id: str,
result: Optional[Dict] = None,
error: bool = False):
''' search thread updater and cleaner
'''
thread_id = self._trial_proposed_by.get(trial_id)
if thread_id in self._search_thread_pool:
self._search_thread_pool[thread_id].on_trial_complete(
trial_id, result, error)
del self._trial_proposed_by[trial_id]
# if not thread_id: logger.info(f"result {result}")
if result:
config = {}
for key, value in result.items():
if key.startswith('config/'):
config[key[7:]] = value
if error: # remove from result cache
del self._result[self._ls.config_signature(config)]
else: # add to result cache
self._result[self._ls.config_signature(config)] = result
# update target metric if improved
if (result[self._metric] -
self._metric_target) * self._ls.metric_op < 0:
self._metric_target = result[self._metric]
if thread_id: # from local search
# update admissible region
normalized_config = self._ls.normalize(config)
for key in self._admissible_min:
value = normalized_config[key]
if value > self._admissible_max[key]:
self._admissible_max[key] = value
elif value < self._admissible_min[key]:
self._admissible_min[key] = value
elif self._create_condition(result):
# thread creator
self._search_thread_pool[self._thread_count] = SearchThread(
self._ls.mode,
self._ls.create(config,
result[self._metric],
cost=result["time_total_s"]))
thread_id = self._thread_count
self._thread_count += 1
# cleaner
# logger.info(f"thread {thread_id} in search thread pool="
# f"{thread_id in self._search_thread_pool}")
if thread_id and thread_id in self._search_thread_pool:
# local search thread
self._clean(thread_id)
def _create_condition(self, result: Dict) -> bool:
''' create thread condition
'''
if len(self._search_thread_pool) < 2: return True
obj_median = np.median([
thread.obj_best1
for id, thread in self._search_thread_pool.items() if id
])
return result[self._metric] * self._ls.metric_op < obj_median
def _clean(self, thread_id: int):
''' delete thread and increase admissible region if converged,
merge local threads if they are close
'''
assert thread_id
todelete = set()
for id in self._search_thread_pool:
if id and id != thread_id:
if self._inferior(id, thread_id):
todelete.add(id)
for id in self._search_thread_pool:
if id and id != thread_id:
if self._inferior(thread_id, id):
todelete.add(thread_id)
break
# logger.info(f"thead {thread_id}.converged="
# f"{self._search_thread_pool[thread_id].converged}")
if self._search_thread_pool[thread_id].converged:
todelete.add(thread_id)
for key in self._admissible_min:
self._admissible_max[key] += self._ls.STEPSIZE
self._admissible_min[key] -= self._ls.STEPSIZE
for id in todelete:
del self._search_thread_pool[id]
def _inferior(self, id1: int, id2: int) -> bool:
''' whether thread id1 is inferior to id2
'''
t1 = self._search_thread_pool[id1]
t2 = self._search_thread_pool[id2]
if t1.obj_best1 < t2.obj_best2: return False
elif t1.resource and t1.resource < t2.resource: return False
elif t2.reach(t1): return True
else: return False
def on_trial_result(self, trial_id: str, result: Dict):
if trial_id not in self._trial_proposed_by: return
thread_id = self._trial_proposed_by[trial_id]
if not thread_id in self._search_thread_pool: return
self._search_thread_pool[thread_id].on_trial_result(trial_id, result)
def suggest(self, trial_id: str) -> Optional[Dict]:
''' choose thread, suggest a valid config
'''
if self._init_used and not self._points_to_evaluate:
choice, backup = self._select_thread()
# logger.debug(f"choice={choice}, backup={backup}")
if choice < 0: return None # timeout
self._use_rs = False
config = self._search_thread_pool[choice].suggest(trial_id)
skip = self._should_skip(choice, trial_id, config)
if skip:
if choice:
# logger.info(f"skipping choice={choice}, config={config}")
return None
# use rs
self._use_rs = True
for _, generated in generate_variants(
{'config': self._ls.space}):
config = generated['config']
break
# logger.debug(f"random config {config}")
skip = self._should_skip(choice, trial_id, config)
if skip: return None
# if not choice: logger.info(config)
if choice or backup == choice or self._valid(config):
# LS or valid or no backup choice
self._trial_proposed_by[trial_id] = choice
else: # invalid config proposed by GS
if not self._use_rs:
self._search_thread_pool[choice].on_trial_complete(
trial_id, {}, error=True) # tell GS there is an error
self._use_rs = False
config = self._search_thread_pool[backup].suggest(trial_id)
skip = self._should_skip(backup, trial_id, config)
if skip:
return None
self._trial_proposed_by[trial_id] = backup
choice = backup
# if choice: self._pending.add(choice) # local search thread pending
if not choice:
if self._ls._resource:
# TODO: add resource to config proposed by GS, min or median?
config[self._ls.prune_attr] = self._ls.min_resource
self._result[self._ls.config_signature(config)] = {}
else: # use init config
init_config = self._points_to_evaluate.pop(
0) if self._points_to_evaluate else self._ls.init_config
config = self._ls.complete_config(init_config,
self._admissible_min,
self._admissible_max)
# logger.info(f"reset config to {config}")
config_signature = self._ls.config_signature(config)
result = self._result.get(config_signature)
if result: # tried before
# self.on_trial_complete(trial_id, result)
return None
elif result is None: # not tried before
self._result[config_signature] = {}
else:
return None # running but no result yet
self._init_used = True
# logger.info(f"config={config}")
return config
def _should_skip(self, choice, trial_id, config) -> bool:
''' if config is None or config's result is known or above mem threshold
return True; o.w. return False
'''
if config is None: return True
config_signature = self._ls.config_signature(config)
exists = config_signature in self._result
# check mem constraint
if not exists and self._mem_threshold and self._mem_size(
config) > self._mem_threshold:
self._result[config_signature] = {
self._metric: np.inf * self._ls.metric_op,
'time_total_s': 1
}
exists = True
if exists:
if not self._use_rs:
result = self._result.get(config_signature)
if result:
self._search_thread_pool[choice].on_trial_complete(
trial_id, result, error=False)
if choice:
# local search thread
self._clean(choice)
else:
# tell the thread there is an error
self._search_thread_pool[choice].on_trial_complete(
trial_id, {}, error=True)
return True
return False
def _select_thread(self) -> Tuple:
''' thread selector; use can_suggest to check LS availability
'''
# update priority
min_eci = self._deadline - time.time()
if min_eci <= 0: return -1, -1
max_speed = 0
for thread in self._search_thread_pool.values():
if thread.speed > max_speed: max_speed = thread.speed
for thread in self._search_thread_pool.values():
thread.update_eci(self._metric_target, max_speed)
if thread.eci < min_eci: min_eci = thread.eci
for thread in self._search_thread_pool.values():
thread.update_priority(min_eci)
top_thread_id = backup_thread_id = 0
priority1 = priority2 = self._search_thread_pool[0].priority
# logger.debug(f"priority of thread 0={priority1}")
for thread_id, thread in self._search_thread_pool.items():
# if thread_id:
# logger.debug(
# f"priority of thread {thread_id}={thread.priority}")
# logger.debug(
# f"thread {thread_id}.can_suggest={thread.can_suggest}")
if thread_id and thread.can_suggest:
priority = thread.priority
if priority > priority1:
priority1 = priority
top_thread_id = thread_id
if priority > priority2 or backup_thread_id == 0:
priority2 = priority
backup_thread_id = thread_id
return top_thread_id, backup_thread_id
def _valid(self, config: Dict) -> bool:
''' config validator
'''
for key in self._admissible_min:
if key in config:
value = config[key]
# logger.info(
# f"{key},{value},{self._admissible_min[key]},{self._admissible_max[key]}")
if value < self._admissible_min[
key] or value > self._admissible_max[key]:
return False
return True
class BlendSearch(Searcher):
"""class for BlendSearch algorithm."""
cost_attr = "time_total_s" # cost attribute in result
lagrange = "_lagrange" # suffix for lagrange-modified metric
penalty = 1e10 # penalty term for constraints
LocalSearch = FLOW2
def __init__(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
space: Optional[dict] = None,
low_cost_partial_config: Optional[dict] = None,
cat_hp_cost: Optional[dict] = None,
points_to_evaluate: Optional[List[dict]] = None,
evaluated_rewards: Optional[List] = None,
time_budget_s: Union[int, float] = None,
num_samples: Optional[int] = None,
resource_attr: Optional[str] = None,
min_resource: Optional[float] = None,
max_resource: Optional[float] = None,
reduction_factor: Optional[float] = None,
global_search_alg: Optional[Searcher] = None,
config_constraints: Optional[List[Tuple[Callable[[dict], float], str,
float]]] = None,
metric_constraints: Optional[List[Tuple[str, str, float]]] = None,
seed: Optional[int] = 20,
experimental: Optional[bool] = False,
use_incumbent_result_in_evaluation=False,
):
"""Constructor.
Args:
metric: A string of the metric name to optimize for.
mode: A string in ['min', 'max'] to specify the objective as
minimization or maximization.
space: A dictionary to specify the search space.
low_cost_partial_config: A dictionary from a subset of
controlled dimensions to the initial low-cost values.
E.g., ```{'n_estimators': 4, 'max_leaves': 4}```.
cat_hp_cost: A dictionary from a subset of categorical dimensions
to the relative cost of each choice.
E.g., ```{'tree_method': [1, 1, 2]}```.
I.e., the relative cost of the three choices of 'tree_method'
is 1, 1 and 2 respectively.
points_to_evaluate: Initial parameter suggestions to be run first.
evaluated_rewards (list): If you have previously evaluated the
parameters passed in as points_to_evaluate you can avoid
re-running those trials by passing in the reward attributes
as a list so the optimiser can be told the results without
needing to re-compute the trial. Must be the same length as
points_to_evaluate.
time_budget_s: int or float | Time budget in seconds.
num_samples: int | The number of configs to try.
resource_attr: A string to specify the resource dimension and the best
performance is assumed to be at the max_resource.
min_resource: A float of the minimal resource to use for the resource_attr.
max_resource: A float of the maximal resource to use for the resource_attr.
reduction_factor: A float of the reduction factor used for
incremental pruning.
global_search_alg: A Searcher instance as the global search
instance. If omitted, Optuna is used. The following algos have
known issues when used as global_search_alg:
- HyperOptSearch raises exception sometimes
- TuneBOHB has its own scheduler
config_constraints: A list of config constraints to be satisfied.
E.g., ```config_constraints = [(mem_size, '<=', 1024**3)]```.
`mem_size` is a function which produces a float number for the bytes
needed for a config.
It is used to skip configs which do not fit in memory.
metric_constraints: A list of metric constraints to be satisfied.
E.g., `['precision', '>=', 0.9]`. The sign can be ">=" or "<=".
seed: An integer of the random seed.
experimental: A bool of whether to use experimental features.
"""
self._metric, self._mode = metric, mode
self._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
init_config = low_cost_partial_config or {}
if not init_config:
logger.info(
"No low-cost partial config given to the search algorithm. "
"For cost-frugal search, "
"consider providing low-cost values for cost-related hps via "
"'low_cost_partial_config'. More info can be found at "
"https://microsoft.github.io/FLAML/docs/FAQ#about-low_cost_partial_config-in-tune"
)
if evaluated_rewards and mode:
self._points_to_evaluate = []
self._evaluated_rewards = []
best = max(evaluated_rewards) if mode == "max" else min(
evaluated_rewards)
# only keep the best points as start points
for i, r in enumerate(evaluated_rewards):
if r == best:
p = points_to_evaluate[i]
self._points_to_evaluate.append(p)
self._evaluated_rewards.append(r)
else:
self._points_to_evaluate = points_to_evaluate or []
self._evaluated_rewards = evaluated_rewards or []
self._config_constraints = config_constraints
self._metric_constraints = metric_constraints
if metric_constraints:
assert all(x[1] in ["<=", ">="] for x in metric_constraints
), "sign of metric constraints must be <= or >=."
# metric modified by lagrange
metric += self.lagrange
self._cat_hp_cost = cat_hp_cost or {}
if space:
add_cost_to_space(space, init_config, self._cat_hp_cost)
self._ls = self.LocalSearch(
init_config,
metric,
mode,
space,
resource_attr,
min_resource,
max_resource,
reduction_factor,
self.cost_attr,
seed,
)
if global_search_alg is not None:
self._gs = global_search_alg
elif getattr(self, "__name__", None) != "CFO":
if space and self._ls.hierarchical:
from functools import partial
gs_space = partial(define_by_run_func, space=space)
evaluated_rewards = None # not supported by define-by-run
else:
gs_space = space
gs_seed = seed - 10 if (seed - 10) >= 0 else seed - 11 + (1 << 32)
if experimental:
import optuna as ot
sampler = ot.samplers.TPESampler(seed=seed,
multivariate=True,
group=True)
else:
sampler = None
try:
self._gs = GlobalSearch(
space=gs_space,
metric=metric,
mode=mode,
seed=gs_seed,
sampler=sampler,
points_to_evaluate=points_to_evaluate,
evaluated_rewards=evaluated_rewards,
)
except ValueError:
self._gs = GlobalSearch(
space=gs_space,
metric=metric,
mode=mode,
seed=gs_seed,
sampler=sampler,
)
self._gs.space = space
else:
self._gs = None
self._experimental = experimental
if (getattr(self, "__name__", None) == "CFO" and points_to_evaluate
and len(self._points_to_evaluate) > 1):
# use the best config in points_to_evaluate as the start point
self._candidate_start_points = {}
self._started_from_low_cost = not low_cost_partial_config
else:
self._candidate_start_points = None
self._time_budget_s, self._num_samples = time_budget_s, num_samples
if space is not None:
self._init_search()
def set_search_properties(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
config: Optional[Dict] = None,
setting: Optional[Dict] = None,
) -> bool:
metric_changed = mode_changed = False
if metric and self._metric != metric:
metric_changed = True
self._metric = metric
if self._metric_constraints:
# metric modified by lagrange
metric += self.lagrange
# TODO: don't change metric for global search methods that
# can handle constraints already
if mode and self._mode != mode:
mode_changed = True
self._mode = mode
if not self._ls.space:
# the search space can be set only once
if self._gs is not None:
# define-by-run is not supported via set_search_properties
self._gs.set_search_properties(metric, mode, config)
self._gs.space = config
if config:
add_cost_to_space(config, self._ls.init_config,
self._cat_hp_cost)
self._ls.set_search_properties(metric, mode, config)
self._init_search()
else:
if metric_changed or mode_changed:
# reset search when metric or mode changed
self._ls.set_search_properties(metric, mode)
if self._gs is not None:
self._gs = GlobalSearch(
space=self._gs._space,
metric=metric,
mode=mode,
sampler=self._gs._sampler,
)
self._gs.space = self._ls.space
self._init_search()
if setting:
# CFO doesn't need these settings
if "time_budget_s" in setting:
self._time_budget_s = setting[
"time_budget_s"] # budget from now
now = time.time()
self._time_used += now - self._start_time
self._start_time = now
self._set_deadline()
if "metric_target" in setting:
self._metric_target = setting.get("metric_target")
if "num_samples" in setting:
self._num_samples = (setting["num_samples"] +
len(self._result) +
len(self._trial_proposed_by))
return True
def _set_deadline(self):
if self._time_budget_s is not None:
self._deadline = self._time_budget_s + self._start_time
SearchThread.set_eps(self._time_budget_s)
else:
self._deadline = np.inf
def _init_search(self):
"""initialize the search"""
self._start_time = time.time()
self._time_used = 0
self._set_deadline()
self._is_ls_ever_converged = False
self._subspace = {} # the subspace for each trial id
self._metric_target = np.inf * self._ls.metric_op
self._search_thread_pool = {
# id: int -> thread: SearchThread
0: SearchThread(self._ls.mode, self._gs)
}
self._thread_count = 1 # total # threads created
self._init_used = self._ls.init_config is None
self._trial_proposed_by = {} # trial_id: str -> thread_id: int
self._ls_bound_min = normalize(
self._ls.init_config.copy(),
self._ls.space,
self._ls.init_config,
{},
recursive=True,
)
self._ls_bound_max = normalize(
self._ls.init_config.copy(),
self._ls.space,
self._ls.init_config,
{},
recursive=True,
)
self._gs_admissible_min = self._ls_bound_min.copy()
self._gs_admissible_max = self._ls_bound_max.copy()
self._result = {} # config_signature: tuple -> result: Dict
if self._metric_constraints:
self._metric_constraint_satisfied = False
self._metric_constraint_penalty = [
self.penalty for _ in self._metric_constraints
]
else:
self._metric_constraint_satisfied = True
self._metric_constraint_penalty = None
self.best_resource = self._ls.min_resource
def save(self, checkpoint_path: str):
"""save states to a checkpoint path."""
self._time_used += time.time() - self._start_time
self._start_time = time.time()
save_object = self
with open(checkpoint_path, "wb") as outputFile:
pickle.dump(save_object, outputFile)
def restore(self, checkpoint_path: str):
"""restore states from checkpoint."""
with open(checkpoint_path, "rb") as inputFile:
state = pickle.load(inputFile)
self.__dict__ = state.__dict__
self._start_time = time.time()
self._set_deadline()
@property
def metric_target(self):
return self._metric_target
@property
def is_ls_ever_converged(self):
return self._is_ls_ever_converged
def on_trial_complete(self,
trial_id: str,
result: Optional[Dict] = None,
error: bool = False):
"""search thread updater and cleaner."""
metric_constraint_satisfied = True
if result and not error and self._metric_constraints:
# account for metric constraints if any
objective = result[self._metric]
for i, constraint in enumerate(self._metric_constraints):
metric_constraint, sign, threshold = constraint
value = result.get(metric_constraint)
if value:
sign_op = 1 if sign == "<=" else -1
violation = (value - threshold) * sign_op
if violation > 0:
# add penalty term to the metric
objective += (self._metric_constraint_penalty[i] *
violation * self._ls.metric_op)
metric_constraint_satisfied = False
if self._metric_constraint_penalty[i] < self.penalty:
self._metric_constraint_penalty[i] += violation
result[self._metric + self.lagrange] = objective
if metric_constraint_satisfied and not self._metric_constraint_satisfied:
# found a feasible point
self._metric_constraint_penalty = [
1 for _ in self._metric_constraints
]
self._metric_constraint_satisfied |= metric_constraint_satisfied
thread_id = self._trial_proposed_by.get(trial_id)
if thread_id in self._search_thread_pool:
self._search_thread_pool[thread_id].on_trial_complete(
trial_id, result, error)
del self._trial_proposed_by[trial_id]
if result:
config = result.get("config", {})
if not config:
for key, value in result.items():
if key.startswith("config/"):
config[key[7:]] = value
signature = self._ls.config_signature(
config, self._subspace.get(trial_id, {}))
if error: # remove from result cache
del self._result[signature]
else: # add to result cache
self._result[signature] = result
# update target metric if improved
objective = result[self._ls.metric]
if (objective - self._metric_target) * self._ls.metric_op < 0:
self._metric_target = objective
if self._ls.resource:
self._best_resource = config[self._ls.resource_attr]
if thread_id:
if not self._metric_constraint_satisfied:
# no point has been found to satisfy metric constraint
self._expand_admissible_region(
self._ls_bound_min,
self._ls_bound_max,
self._subspace.get(trial_id, self._ls.space),
)
if (self._gs is not None and self._experimental
and (not self._ls.hierarchical)):
self._gs.add_evaluated_point(flatten_dict(config),
objective)
# TODO: recover when supported
# converted = convert_key(config, self._gs.space)
# logger.info(converted)
# self._gs.add_evaluated_point(converted, objective)
elif metric_constraint_satisfied and self._create_condition(
result):
# thread creator
thread_id = self._thread_count
self._started_from_given = (
self._candidate_start_points
and trial_id in self._candidate_start_points)
if self._started_from_given:
del self._candidate_start_points[trial_id]
else:
self._started_from_low_cost = True
self._create_thread(
config, result,
self._subspace.get(trial_id, self._ls.space))
# reset admissible region to ls bounding box
self._gs_admissible_min.update(self._ls_bound_min)
self._gs_admissible_max.update(self._ls_bound_max)
# cleaner
if thread_id and thread_id in self._search_thread_pool:
# local search thread
self._clean(thread_id)
if trial_id in self._subspace and not (
self._candidate_start_points
and trial_id in self._candidate_start_points):
del self._subspace[trial_id]
def _create_thread(self, config, result, space):
self._search_thread_pool[self._thread_count] = SearchThread(
self._ls.mode,
self._ls.create(
config,
result[self._ls.metric],
cost=result.get(self.cost_attr, 1),
space=space,
),
self.cost_attr,
)
self._thread_count += 1
self._update_admissible_region(
unflatten_dict(config),
self._ls_bound_min,
self._ls_bound_max,
space,
self._ls.space,
)
def _update_admissible_region(
self,
config,
admissible_min,
admissible_max,
subspace: Dict = {},
space: Dict = {},
):
# update admissible region
normalized_config = normalize(config, subspace, config, {})
for key in admissible_min:
value = normalized_config[key]
if isinstance(admissible_max[key], list):
domain = space[key]
choice = indexof(domain, value)
self._update_admissible_region(
value,
admissible_min[key][choice],
admissible_max[key][choice],
subspace[key],
domain[choice],
)
if len(admissible_max[key]) > len(domain.categories):
# points + index
normal = (choice + 0.5) / len(domain.categories)
admissible_max[key][-1] = max(normal,
admissible_max[key][-1])
admissible_min[key][-1] = min(normal,
admissible_min[key][-1])
elif isinstance(value, dict):
self._update_admissible_region(
value,
admissible_min[key],
admissible_max[key],
subspace[key],
space[key],
)
else:
if value > admissible_max[key]:
admissible_max[key] = value
elif value < admissible_min[key]:
admissible_min[key] = value
def _create_condition(self, result: Dict) -> bool:
"""create thread condition"""
if len(self._search_thread_pool) < 2:
return True
obj_median = np.median([
thread.obj_best1
for id, thread in self._search_thread_pool.items() if id
])
return result[self._ls.metric] * self._ls.metric_op < obj_median
def _clean(self, thread_id: int):
"""delete thread and increase admissible region if converged,
merge local threads if they are close
"""
assert thread_id
todelete = set()
for id in self._search_thread_pool:
if id and id != thread_id:
if self._inferior(id, thread_id):
todelete.add(id)
for id in self._search_thread_pool:
if id and id != thread_id:
if self._inferior(thread_id, id):
todelete.add(thread_id)
break
create_new = False
if self._search_thread_pool[thread_id].converged:
self._is_ls_ever_converged = True
todelete.add(thread_id)
self._expand_admissible_region(
self._ls_bound_min,
self._ls_bound_max,
self._search_thread_pool[thread_id].space,
)
if self._candidate_start_points:
if not self._started_from_given:
# remove start points whose perf is worse than the converged
obj = self._search_thread_pool[thread_id].obj_best1
worse = [
trial_id for trial_id, r in
self._candidate_start_points.items()
if r and r[self._ls.metric] * self._ls.metric_op >= obj
]
# logger.info(f"remove candidate start points {worse} than {obj}")
for trial_id in worse:
del self._candidate_start_points[trial_id]
if self._candidate_start_points and self._started_from_low_cost:
create_new = True
for id in todelete:
del self._search_thread_pool[id]
if create_new:
self._create_thread_from_best_candidate()
def _create_thread_from_best_candidate(self):
# find the best start point
best_trial_id = None
obj_best = None
for trial_id, r in self._candidate_start_points.items():
if r and (best_trial_id is None
or r[self._ls.metric] * self._ls.metric_op < obj_best):
best_trial_id = trial_id
obj_best = r[self._ls.metric] * self._ls.metric_op
if best_trial_id:
# create a new thread
config = {}
result = self._candidate_start_points[best_trial_id]
for key, value in result.items():
if key.startswith("config/"):
config[key[7:]] = value
self._started_from_given = True
del self._candidate_start_points[best_trial_id]
self._create_thread(
config, result,
self._subspace.get(best_trial_id, self._ls.space))
def _expand_admissible_region(self, lower, upper, space):
"""expand the admissible region for the subspace `space`"""
for key in upper:
ub = upper[key]
if isinstance(ub, list):
choice = space[key]["_choice_"]
self._expand_admissible_region(lower[key][choice],
upper[key][choice], space[key])
elif isinstance(ub, dict):
self._expand_admissible_region(lower[key], ub, space[key])
else:
upper[key] += self._ls.STEPSIZE
lower[key] -= self._ls.STEPSIZE
def _inferior(self, id1: int, id2: int) -> bool:
"""whether thread id1 is inferior to id2"""
t1 = self._search_thread_pool[id1]
t2 = self._search_thread_pool[id2]
if t1.obj_best1 < t2.obj_best2:
return False
elif t1.resource and t1.resource < t2.resource:
return False
elif t2.reach(t1):
return True
return False
def on_trial_result(self, trial_id: str, result: Dict):
"""receive intermediate result."""
if trial_id not in self._trial_proposed_by:
return
thread_id = self._trial_proposed_by[trial_id]
if thread_id not in self._search_thread_pool:
return
if result and self._metric_constraints:
result[self._metric + self.lagrange] = result[self._metric]
self._search_thread_pool[thread_id].on_trial_result(trial_id, result)
def suggest(self, trial_id: str) -> Optional[Dict]:
"""choose thread, suggest a valid config."""
if self._init_used and not self._points_to_evaluate:
choice, backup = self._select_thread()
# if choice < 0: # timeout
# return None
config = self._search_thread_pool[choice].suggest(trial_id)
if not choice and config is not None and self._ls.resource:
config[self._ls.resource_attr] = self.best_resource
elif choice and config is None:
# local search thread finishes
if self._search_thread_pool[choice].converged:
self._expand_admissible_region(
self._ls_bound_min,
self._ls_bound_max,
self._search_thread_pool[choice].space,
)
del self._search_thread_pool[choice]
return None
# preliminary check; not checking config validation
space = self._search_thread_pool[choice].space
skip = self._should_skip(choice, trial_id, config, space)
use_rs = 0
if skip:
if choice:
return None
# use rs when BO fails to suggest a config
config, space = self._ls.complete_config({})
skip = self._should_skip(-1, trial_id, config, space)
if skip:
return None
use_rs = 1
if choice or self._valid(
config,
self._ls.space,
space,
self._gs_admissible_min,
self._gs_admissible_max,
):
# LS or valid or no backup choice
self._trial_proposed_by[trial_id] = choice
self._search_thread_pool[choice].running += use_rs
else: # invalid config proposed by GS
if choice == backup:
# use CFO's init point
init_config = self._ls.init_config
config, space = self._ls.complete_config(
init_config, self._ls_bound_min, self._ls_bound_max)
self._trial_proposed_by[trial_id] = choice
self._search_thread_pool[choice].running += 1
else:
thread = self._search_thread_pool[backup]
config = thread.suggest(trial_id)
space = thread.space
skip = self._should_skip(backup, trial_id, config, space)
if skip:
return None
self._trial_proposed_by[trial_id] = backup
choice = backup
if not choice: # global search
# temporarily relax admissible region for parallel proposals
self._update_admissible_region(
config,
self._gs_admissible_min,
self._gs_admissible_max,
space,
self._ls.space,
)
else:
self._update_admissible_region(
config,
self._ls_bound_min,
self._ls_bound_max,
space,
self._ls.space,
)
self._gs_admissible_min.update(self._ls_bound_min)
self._gs_admissible_max.update(self._ls_bound_max)
signature = self._ls.config_signature(config, space)
self._result[signature] = {}
self._subspace[trial_id] = space
else: # use init config
if self._candidate_start_points is not None and self._points_to_evaluate:
self._candidate_start_points[trial_id] = None
reward = None
if self._points_to_evaluate:
init_config = self._points_to_evaluate.pop(0)
if self._evaluated_rewards:
reward = self._evaluated_rewards.pop(0)
else:
init_config = self._ls.init_config
config, space = self._ls.complete_config(init_config,
self._ls_bound_min,
self._ls_bound_max)
if reward is None:
config_signature = self._ls.config_signature(config, space)
result = self._result.get(config_signature)
if result: # tried before
return None
elif result is None: # not tried before
self._result[config_signature] = {}
else: # running but no result yet
return None
self._init_used = True
self._trial_proposed_by[trial_id] = 0
self._search_thread_pool[0].running += 1
self._subspace[trial_id] = space
if reward is not None:
result = {
self._metric: reward,
self.cost_attr: 1,
"config": config
}
self.on_trial_complete(trial_id, result)
return None
if self._use_incumbent_result_in_evaluation:
if self._trial_proposed_by[trial_id] > 0:
choice_thread = self._search_thread_pool[
self._trial_proposed_by[trial_id]]
config[INCUMBENT_RESULT] = choice_thread.best_result
return config
def _should_skip(self, choice, trial_id, config, space) -> bool:
"""if config is None or config's result is known or constraints are violated
return True; o.w. return False
"""
if config is None:
return True
config_signature = self._ls.config_signature(config, space)
exists = config_signature in self._result
# check constraints
if not exists and self._config_constraints:
for constraint in self._config_constraints:
func, sign, threshold = constraint
value = func(config)
if (sign == "<=" and value > threshold
or sign == ">=" and value < threshold
or sign == ">" and value <= threshold
or sign == "<" and value > threshold):
self._result[config_signature] = {
self._metric: np.inf * self._ls.metric_op,
"time_total_s": 1,
}
exists = True
break
if exists: # suggested before
if choice >= 0: # not fallback to rs
result = self._result.get(config_signature)
if result: # finished
self._search_thread_pool[choice].on_trial_complete(
trial_id, result, error=False)
if choice:
# local search thread
self._clean(choice)
# else: # running
# # tell the thread there is an error
# self._search_thread_pool[choice].on_trial_complete(
# trial_id, {}, error=True)
return True
return False
def _select_thread(self) -> Tuple:
"""thread selector; use can_suggest to check LS availability"""
# update priority
now = time.time()
min_eci = self._deadline - now
if min_eci <= 0:
# return -1, -1
# keep proposing new configs assuming no budget left
min_eci = 0
elif self._num_samples and self._num_samples > 0:
# estimate time left according to num_samples limitation
num_finished = len(self._result)
num_proposed = num_finished + len(self._trial_proposed_by)
num_left = max(self._num_samples - num_proposed, 0)
if num_proposed > 0:
time_used = now - self._start_time + self._time_used
min_eci = min(min_eci, time_used / num_finished * num_left)
# print(f"{min_eci}, {time_used / num_finished * num_left}, {num_finished}, {num_left}")
max_speed = 0
for thread in self._search_thread_pool.values():
if thread.speed > max_speed:
max_speed = thread.speed
for thread in self._search_thread_pool.values():
thread.update_eci(self._metric_target, max_speed)
if thread.eci < min_eci:
min_eci = thread.eci
for thread in self._search_thread_pool.values():
thread.update_priority(min_eci)
top_thread_id = backup_thread_id = 0
priority1 = priority2 = self._search_thread_pool[0].priority
for thread_id, thread in self._search_thread_pool.items():
if thread_id and thread.can_suggest:
priority = thread.priority
if priority > priority1:
priority1 = priority
top_thread_id = thread_id
if priority > priority2 or backup_thread_id == 0:
priority2 = priority
backup_thread_id = thread_id
return top_thread_id, backup_thread_id
def _valid(self, config: Dict, space: Dict, subspace: Dict, lower: Dict,
upper: Dict) -> bool:
"""config validator"""
normalized_config = normalize(config, subspace, config, {})
for key, lb in lower.items():
if key in config:
value = normalized_config[key]
if isinstance(lb, list):
domain = space[key]
index = indexof(domain, value)
nestedspace = subspace[key]
lb = lb[index]
ub = upper[key][index]
elif isinstance(lb, dict):
nestedspace = subspace[key]
domain = space[key]
ub = upper[key]
else:
nestedspace = None
if nestedspace:
valid = self._valid(value, domain, nestedspace, lb, ub)
if not valid:
return False
elif (value + self._ls.STEPSIZE < lower[key]
or value > upper[key] + self._ls.STEPSIZE):
return False
return True