def check_format_param_grid(param_grid, control, param_grid_name='param_grid_0'): '''Run validations on a param_grid''' if not isinstance(control, dict): raise ElmConfigError("Expected 'control' as dict") early_stop = control.get('early_stop') or None if early_stop: if not isinstance(early_stop, dict) or not any( k in early_stop for k in ('abs_change', 'percent_change', 'threshold')): raise ElmConfigError( 'Expected "early_stop" to be a dict with a key in ("abs_change", "percent_change", "threshold")' ) if not isinstance(control, dict): raise ElmConfigError( 'Expected param_grids: {} - "control" to be a dict'.format( control)) for required_key, typ in REQUIRED_CONTROL_KEYS_TYPES.items(): item = control.get(required_key) or None if not isinstance(item, typ): raise ElmConfigError('Expected params_grids:{} ' '"control" to have key {} with ' 'type {}'.format(param_grid_name, required_key, typ)) return _to_param_meta(param_grid, control)
def eval_stop_wrapper(evo_params, original_fitness): '''Handle sections of config's {param_grids: {pg_name: control: {}} Examples of configs handled by this function: * :early_stop: {abs_change: [10], agg: all}, * :early_stop: {percent_change: [10], agg: all} * :early_stop: {threshold: [10], agg: any} Parameters: :evo_params: EA parameters Returns: :decorator: Evaluates stop on each EA iteration ''' early_stop = evo_params['early_stop'] if not early_stop: # If not early_stop, this is a do-nothing function value = [ -1, ] * len(evo_params['score_weights']) func = _no_stop elif 'percent_change' in early_stop: typ = 'percent_change' value = early_stop['percent_change'] _check_number(typ, value) func = _percent_change_stop elif 'threshold' in early_stop: typ = 'threshold' value = early_stop['threshold'] _check_number(typ, value, positive_definite=False) func = _threshold_stop elif 'abs_change' in early_stop: typ = 'abs_change' value = early_stop['abs_change'] _check_number(typ, value) func = _abs_change_stop else: raise ValueError( 'early_stop:{} does not have a ' 'key in ("abs_change", "percent_change", "threshold")'.format( early_stop)) if not isinstance(value, Sequence): raise ElmConfigError( 'Expected early_stop:{} to be a Sequence'.format(value)) if not len(evo_params['score_weights']) == len(value) == len( original_fitness): raise ElmConfigError('Expected score_weights {}, early_stop: {} ({}) ' 'and fitness {} to all have same ' 'len()'.format(evo_params['score_weights'], typ, value, original_fitness)) if not early_stop: agg = all else: early_stop.get('agg', all) eval_stop = partial(func, agg, evo_params['score_weights'], value, original_fitness) return eval_stop
def ea_setup(config=None, param_grid=None, param_grid_name=None, score_weights=None): '''ea_setup parses a yaml config or param_grid dictionary to create an EvoParams instance with fields needed for algorithm Parameters: :config: elm.config.ConfigParser instance or None :param_grid: param_grid dictionary if not giving config :param_grid_name: name of param_grid, if not giving config :score_weights: list of weights, -1 or 1, for each objective, ignored if config is not None Returns: * if config was given dict of `EvoParams` for each step in config's "run" list (integers as key, EvoParams instances as values) * elif config was None `EvoParams` instance ''' if config and not param_grid: idx_to_param_grid = _get_evolve_meta(config) items = tuple(idx_to_param_grid.items()) elif param_grid and not config: items = ((0, check_format_param_grid(param_grid, param_grid_name)),) else: raise ValueError('Expected config or param_grid but not both') if not items: return {} evo_params_dict = {} for (idx, deap_params) in items: if config: param_grid_name = config.run[idx]['train'] kwargs = copy.deepcopy(deap_params['control']) toolbox = base.Toolbox() if config and score_weights is None: train = config.train[param_grid_name] if 'model_scoring' in train: ms = config.model_scoring[train['model_scoring']] if 'score_weights' in ms: score_weights = ms['score_weights'] if score_weights is None: raise ElmConfigError('Cannot continue EA ' 'when score_weights is None (train section of config: ' '{})'.format(step_name)) creator.create('FitnessMulti', base.Fitness, weights=tuple(score_weights)) creator.create('Individual', list, fitness=creator.FitnessMulti) toolbox.register('indices', _random_choice, deap_params['choices']) toolbox.register('individual', tools.initIterate, creator.Individual, toolbox.indices) toolbox.register('population', tools.initRepeat, list, toolbox.individual) early_stop = deap_params['control'].get('early_stop') or None evo_params = EvoParams( toolbox=_configure_toolbox(toolbox, deap_params, config, **kwargs), deap_params=deap_params, param_grid_name=param_grid_name, history_file='{}.csv'.format(param_grid_name), score_weights=score_weights, early_stop=early_stop, ) if not config: return evo_params evo_params_dict[idx] = evo_params return evo_params_dict
def _check_number(typ, value, positive_definite=True): '''Check all members of Sequence value are ints''' if not all( isinstance(val, numbers.Number) and (val > 0 if positive_definite else True) for val in value): raise ElmConfigError( 'With early_stop: {} expected a Sequence of positive ints (indices) but found {}' .format(typ, value))
def get_param_grid(config, step): '''Get the metadata for one config step that has a param_grid or return None''' param_grid_name = step.get('param_grid') or None if not param_grid_name: return None train_step_name = step.get('train') if train_step_name: train_config = config.train[train_step_name] else: raise ElmConfigError('Expected param_grid to be used with a "train" ' 'step of a pipeline, but found param_grid ' 'was used in step {}'.format(step)) param_grid = config.param_grids[param_grid_name] return check_format_param_grid(param_grid, param_grid_name)
def main(args=None, sys_argv=None, return_0_if_ok=True): raise ElmMainDeprecation('The console entry point elm-main for running ' 'yaml configs is temporarily deprecated during ' 'refactoring of elm') started = datetime.datetime.now() args = cli(args=args, sys_argv=sys_argv) if args.config_dir is not None and args.config is not None: raise ElmConfigError('Expected --config-dir or --config, not both args') elif args.config_dir: ret = run_many_configs(args, started=started, return_0_if_ok=return_0_if_ok) logger.info('Elapsed time {}'.format((datetime.datetime.now() - started).total_seconds())) ret = run_one_config(args=args, sys_argv=sys_argv, return_0_if_ok=return_0_if_ok, started=started) return ret
def main(args=None, sys_argv=None, return_0_if_ok=True): started = datetime.datetime.now() args = cli(args=args, sys_argv=sys_argv) if args.config_dir is not None and args.config is not None: raise ElmConfigError( 'Expected --config-dir or --config, not both args') elif args.config_dir: ret = run_many_configs(args, started=started, return_0_if_ok=return_0_if_ok) logger.info('Elapsed time {}'.format( (datetime.datetime.now() - started).total_seconds())) ret = run_one_config(args=args, sys_argv=sys_argv, return_0_if_ok=return_0_if_ok, started=started) return ret