def run(training_function, init_config: dict, config: Optional[dict] = None, cat_hp_cost: Optional[dict] = None, metric: Optional[str] = None, mode: Optional[str] = None, time_budget_s: Union[int, float, datetime.timedelta] = None, prune_attr: Optional[str] = None, min_resource: Optional[float] = None, max_resource: Optional[float] = None, reduction_factor: Optional[float] = None, report_intermediate_result: Optional[bool] = False, search_alg = None, verbose: Optional[int] = 2, local_dir: Optional[str] = None, num_samples: Optional[int] = 1, resources_per_trial: Optional[dict] = None, mem_size = None, use_ray: Optional[bool] = False, ): '''The trigger for HPO. Example: .. code-block:: python import time from flaml import tune def compute_with_config(config): current_time = time.time() metric2minimize = (round(config['x'])-95000)**2 time2eval = time.time() - current_time tune.report(metric2minimize=metric2minimize, time2eval=time2eval) analysis = tune.run( compute_with_config, init_config={}, config={ 'x': tune.qloguniform(lower=1, upper=1000000, q=1), 'y': tune.randint(lower=1, upper=1000000) }, metric='metric2minimize', mode='min', num_samples=-1, time_budget_s=60, use_ray=False) print(analysis.trials[-1].last_result) Args: training_function: A user-defined training function. init_config: A dictionary from a subset of controlled dimensions to the initial low-cost values. e.g., .. code-block:: python {'epochs': 1} If no such dimension, pass an empty dict {}. config: A dictionary to specify the search space. cat_hp_cost: A dictionary from a subset of categorical dimensions to the relative cost of each choice. e.g., .. code-block:: python {'tree_method': [1, 1, 2]} i.e., the relative cost of the three choices of 'tree_method' is 1, 1 and 2 respectively metric: A string of the metric name to optimize for. mode: A string in ['min', 'max'] to specify the objective as minimization or maximization. time_budget_s: A float of the time budget in seconds. prune_attr: A string of the attribute used for pruning. Not necessarily in space. When prune_attr is in space, it is a hyperparameter, e.g., 'n_iters', and the best value is unknown. When prune_attr is not in space, it is a resource dimension, e.g., 'sample_size', and the peak performance is assumed to be at the max_resource. min_resource: A float of the minimal resource to use for the prune_attr; only valid if prune_attr is not in space. max_resource: A float of the maximal resource to use for the prune_attr; only valid if prune_attr is not in space. reduction_factor: A float of the reduction factor used for incremental pruning. report_intermediate_result: A boolean of whether intermediate results are reported. If so, early stopping and pruning can be used. search_alg: An instance of BlendSearch as the search algorithm to be used. The same instance can be used for iterative tuning. e.g., .. code-block:: python from flaml import BlendSearch algo = BlendSearch(metric='val_loss', mode='min', space=search_space, points_to_evaluate=points_to_evaluate) for i in range(10): analysis = tune.run(compute_with_config, init_config=None, search_alg=algo, use_ray=False) print(analysis.trials[-1].last_result) verbose: 0, 1, 2, or 3. Verbosity mode for ray if ray backend is used. 0 = silent, 1 = only status updates, 2 = status and brief trial results, 3 = status and detailed trial results. Defaults to 2. local_dir: A string of the local dir to save ray logs if ray backend is used. num_samples: An integer of the number of configs to try. Defaults to 1. resources_per_trial: A dictionary of the hardware resources to allocate per trial, e.g., `{'mem': 1024**3}`. When not using ray backend, only 'mem' is used as approximate resource constraints (in conjunction with mem_size). mem_size: A function to estimate the memory size for a given config. It is used to skip configs which do not fit in memory. use_ray: A boolean of whether to use ray as the backend ''' global _use_ray global _verbose if not use_ray: _verbose = verbose if verbose > 0: import os os.makedirs(local_dir, exist_ok=True) logger.addHandler(logging.FileHandler(local_dir+'/tune_'+str( datetime.datetime.now())+'.log')) if verbose<=2: logger.setLevel(logging.INFO) else: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.CRITICAL) if search_alg is None: from ..searcher.blendsearch import BlendSearch search_alg = BlendSearch(points_to_evaluate=[init_config], metric=metric, mode=mode, cat_hp_cost=cat_hp_cost, space=config, prune_attr=prune_attr, min_resource=min_resource, max_resource=max_resource, reduction_factor=reduction_factor, resources_per_trial=resources_per_trial, mem_size=mem_size) if time_budget_s: search_alg.set_search_properties(metric, mode, config={ 'time_budget_s':time_budget_s}) if report_intermediate_result: params = {} # scheduler resource_dimension=prune_attr if prune_attr: params['time_attr'] = prune_attr if max_resource: params['max_t'] = max_resource if min_resource: params['grace_period'] = min_resource if reduction_factor: params['reduction_factor'] = reduction_factor try: from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(**params) except: scheduler = None else: scheduler = None if use_ray: try: from ray import tune except: raise ImportError("Failed to import ray tune. " "Please install ray[tune] or set use_ray=False") _use_ray = True return tune.run(training_function, metric=metric, mode=mode, search_alg=search_alg, scheduler=scheduler, time_budget_s=time_budget_s, verbose=verbose, local_dir=local_dir, num_samples=num_samples, resources_per_trial=resources_per_trial ) # simple sequential run without using tune.run() from ray time_start = time.time() _use_ray = False if scheduler: scheduler.set_search_properties(metric=metric, mode=mode) from .trial_runner import SequentialTrialRunner global _runner _runner = SequentialTrialRunner( search_alg=search_alg, scheduler=scheduler, metric=metric, mode=mode, ) num_trials = 0 while time.time()-time_start<time_budget_s and ( num_samples<0 or num_trials<num_samples): trial_to_run = _runner.step() if trial_to_run: num_trials += 1 if verbose: logger.info(f'trial {num_trials} config: {trial_to_run.config}') training_function(trial_to_run.config) _runner.stop_trial(trial_to_run) return ExperimentAnalysis(_runner.get_trials(), metric=metric, mode=mode)
def run( evaluation_function, config: Optional[dict] = None, low_cost_partial_config: Optional[dict] = None, cat_hp_cost: Optional[dict] = None, metric: Optional[str] = None, mode: Optional[str] = None, time_budget_s: Union[int, float] = None, points_to_evaluate: Optional[List[dict]] = None, evaluated_rewards: Optional[List] = None, resource_attr: Optional[str] = None, min_resource: Optional[float] = None, max_resource: Optional[float] = None, reduction_factor: Optional[float] = None, scheduler=None, search_alg=None, verbose: Optional[int] = 2, local_dir: Optional[str] = None, num_samples: Optional[int] = 1, resources_per_trial: Optional[dict] = None, config_constraints: Optional[List[Tuple[Callable[[dict], float], str, float]]] = None, metric_constraints: Optional[List[Tuple[str, str, float]]] = None, max_failure: Optional[int] = 100, use_ray: Optional[bool] = False, use_incumbent_result_in_evaluation: Optional[bool] = None, ): """The trigger for HPO. Example: ```python import time from flaml import tune def compute_with_config(config): current_time = time.time() metric2minimize = (round(config['x'])-95000)**2 time2eval = time.time() - current_time tune.report(metric2minimize=metric2minimize, time2eval=time2eval) analysis = tune.run( compute_with_config, config={ 'x': tune.lograndint(lower=1, upper=1000000), 'y': tune.randint(lower=1, upper=1000000) }, metric='metric2minimize', mode='min', num_samples=-1, time_budget_s=60, use_ray=False) print(analysis.trials[-1].last_result) ``` Args: evaluation_function: A user-defined evaluation function. It takes a configuration as input, outputs a evaluation result (can be a numerical value or a dictionary of string and numerical value pairs) for the input configuration. For machine learning tasks, it usually involves training and scoring a machine learning model, e.g., through validation loss. config: A dictionary to specify the search space. low_cost_partial_config: A dictionary from a subset of controlled dimensions to the initial low-cost values. e.g., ```{'n_estimators': 4, 'max_leaves': 4}``` cat_hp_cost: A dictionary from a subset of categorical dimensions to the relative cost of each choice. e.g., ```{'tree_method': [1, 1, 2]}``` i.e., the relative cost of the three choices of 'tree_method' is 1, 1 and 2 respectively metric: A string of the metric name to optimize for. mode: A string in ['min', 'max'] to specify the objective as minimization or maximization. time_budget_s: int or float | The time budget in seconds. points_to_evaluate: A list of initial hyperparameter configurations to run first. evaluated_rewards (list): If you have previously evaluated the parameters passed in as points_to_evaluate you can avoid re-running those trials by passing in the reward attributes as a list so the optimiser can be told the results without needing to re-compute the trial. Must be the same length as points_to_evaluate. e.g., ```python points_to_evaluate = [ {"b": .99, "cost_related": {"a": 3}}, {"b": .99, "cost_related": {"a": 2}}, ] evaluated_rewards=[3.0, 1.0] ``` means that you know the reward for the two configs in points_to_evaluate are 3.0 and 1.0 respectively and want to inform run(). resource_attr: A string to specify the resource dimension used by the scheduler via "scheduler". min_resource: A float of the minimal resource to use for the resource_attr. max_resource: A float of the maximal resource to use for the resource_attr. reduction_factor: A float of the reduction factor used for incremental pruning. scheduler: A scheduler for executing the experiment. Can be None, 'flaml', 'asha' or a custom instance of the TrialScheduler class. Default is None: in this case when resource_attr is provided, the 'flaml' scheduler will be used, otherwise no scheduler will be used. When set 'flaml', an authentic scheduler implemented in FLAML will be used. It does not require users to report intermediate results in evaluation_function. Find more details about this scheduler in this paper https://arxiv.org/pdf/1911.04706.pdf). When set 'asha', the input for arguments "resource_attr", "min_resource", "max_resource" and "reduction_factor" will be passed to ASHA's "time_attr", "max_t", "grace_period" and "reduction_factor" respectively. You can also provide a self-defined scheduler instance of the TrialScheduler class. When 'asha' or self-defined scheduler is used, you usually need to report intermediate results in the evaluation function. Please find examples using different types of schedulers and how to set up the corresponding evaluation functions in test/tune/test_scheduler.py. TODO: point to notebook examples. search_alg: An instance of BlendSearch as the search algorithm to be used. The same instance can be used for iterative tuning. e.g., ```python from flaml import BlendSearch algo = BlendSearch(metric='val_loss', mode='min', space=search_space, low_cost_partial_config=low_cost_partial_config) for i in range(10): analysis = tune.run(compute_with_config, search_alg=algo, use_ray=False) print(analysis.trials[-1].last_result) ``` verbose: 0, 1, 2, or 3. Verbosity mode for ray if ray backend is used. 0 = silent, 1 = only status updates, 2 = status and brief trial results, 3 = status and detailed trial results. Defaults to 2. local_dir: A string of the local dir to save ray logs if ray backend is used; or a local dir to save the tuning log. num_samples: An integer of the number of configs to try. Defaults to 1. resources_per_trial: A dictionary of the hardware resources to allocate per trial, e.g., `{'cpu': 1}`. It is only valid when using ray backend (by setting 'use_ray = True'). It shall be used when you need to do [parallel tuning](https://microsoft.github.io/FLAML/docs/Use-Cases/Tune-User-Defined-Function#parallel-tuning). config_constraints: A list of config constraints to be satisfied. e.g., ```config_constraints = [(mem_size, '<=', 1024**3)]``` mem_size is a function which produces a float number for the bytes needed for a config. It is used to skip configs which do not fit in memory. metric_constraints: A list of metric constraints to be satisfied. e.g., `['precision', '>=', 0.9]`. The sign can be ">=" or "<=". max_failure: int | the maximal consecutive number of failures to sample a trial before the tuning is terminated. use_ray: A boolean of whether to use ray as the backend. """ global _use_ray global _verbose if not use_ray: _verbose = verbose if verbose > 0: import os if local_dir: os.makedirs(local_dir, exist_ok=True) logger.addHandler( logging.FileHandler( local_dir + "/tune_" + str(datetime.datetime.now()).replace(":", "-") + ".log")) elif not logger.handlers: # Add the console handler. _ch = logging.StreamHandler() logger_formatter = logging.Formatter( "[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s", "%m-%d %H:%M:%S", ) _ch.setFormatter(logger_formatter) logger.addHandler(_ch) if verbose <= 2: logger.setLevel(logging.INFO) else: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.CRITICAL) from ..searcher.blendsearch import BlendSearch, CFO if search_alg is None: flaml_scheduler_resource_attr = ( flaml_scheduler_min_resource ) = flaml_scheduler_max_resource = flaml_scheduler_reduction_factor = None if scheduler in (None, "flaml"): # when scheduler is set 'flaml', we will use a scheduler that is # authentic to the search algorithms in flaml. After setting up # the search algorithm accordingly, we need to set scheduler to # None in case it is later used in the trial runner. flaml_scheduler_resource_attr = resource_attr flaml_scheduler_min_resource = min_resource flaml_scheduler_max_resource = max_resource flaml_scheduler_reduction_factor = reduction_factor scheduler = None try: import optuna SearchAlgorithm = BlendSearch except ImportError: SearchAlgorithm = CFO logger.warning( "Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]" ) search_alg = SearchAlgorithm( metric=metric or DEFAULT_METRIC, mode=mode, space=config, points_to_evaluate=points_to_evaluate, evaluated_rewards=evaluated_rewards, low_cost_partial_config=low_cost_partial_config, cat_hp_cost=cat_hp_cost, time_budget_s=time_budget_s, num_samples=num_samples, resource_attr=flaml_scheduler_resource_attr, min_resource=flaml_scheduler_min_resource, max_resource=flaml_scheduler_max_resource, reduction_factor=flaml_scheduler_reduction_factor, config_constraints=config_constraints, metric_constraints=metric_constraints, use_incumbent_result_in_evaluation= use_incumbent_result_in_evaluation, ) else: if metric is None or mode is None: metric = metric or search_alg.metric mode = mode or search_alg.mode if ray_import: from ray.tune.suggest import ConcurrencyLimiter else: from flaml.searcher.suggestion import ConcurrencyLimiter if (search_alg.__class__.__name__ in [ "BlendSearch", "CFO", "CFOCat", ] and use_incumbent_result_in_evaluation is not None): search_alg.use_incumbent_result_in_evaluation = ( use_incumbent_result_in_evaluation) searcher = (search_alg.searcher if isinstance( search_alg, ConcurrencyLimiter) else search_alg) if isinstance(searcher, BlendSearch): setting = {} if time_budget_s: setting["time_budget_s"] = time_budget_s if num_samples > 0: setting["num_samples"] = num_samples searcher.set_search_properties(metric, mode, config, setting) else: searcher.set_search_properties(metric, mode, config) if scheduler == "asha": params = {} # scheduler resource_dimension=resource_attr if resource_attr: params["time_attr"] = resource_attr if max_resource: params["max_t"] = max_resource if min_resource: params["grace_period"] = min_resource if reduction_factor: params["reduction_factor"] = reduction_factor if ray_import: from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(**params) if use_ray: try: from ray import tune except ImportError: raise ImportError("Failed to import ray tune. " "Please install ray[tune] or set use_ray=False") _use_ray = True return tune.run( evaluation_function, metric=metric, mode=mode, search_alg=search_alg, scheduler=scheduler, time_budget_s=time_budget_s, verbose=verbose, local_dir=local_dir, num_samples=num_samples, resources_per_trial=resources_per_trial, ) # simple sequential run without using tune.run() from ray time_start = time.time() _use_ray = False if scheduler: scheduler.set_search_properties(metric=metric, mode=mode) from .trial_runner import SequentialTrialRunner global _runner _runner = SequentialTrialRunner( search_alg=search_alg, scheduler=scheduler, metric=metric, mode=mode, ) num_trials = 0 if time_budget_s is None: time_budget_s = np.inf fail = 0 ub = (len(evaluated_rewards) if evaluated_rewards else 0) + max_failure while (time.time() - time_start < time_budget_s and (num_samples < 0 or num_trials < num_samples) and fail < ub): trial_to_run = _runner.step() if trial_to_run: num_trials += 1 if verbose: logger.info( f"trial {num_trials} config: {trial_to_run.config}") result = evaluation_function(trial_to_run.config) if result is not None: if isinstance(result, dict): report(**result) else: report(_metric=result) _runner.stop_trial(trial_to_run) fail = 0 else: fail += 1 # break with ub consecutive failures if fail == ub: logger.warning( f"fail to sample a trial for {max_failure} times in a row, stopping." ) if verbose > 0: logger.handlers.clear() return ExperimentAnalysis(_runner.get_trials(), metric=metric, mode=mode)