def check_format_param_grid(param_grid,
control,
param_grid_name='param_grid_0'):
'''Run validations on a param_grid'''
if not isinstance(control, dict):
raise ElmConfigError("Expected 'control' as dict")
early_stop = control.get('early_stop') or None
if early_stop:
if not isinstance(early_stop, dict) or not any(
k in early_stop
for k in ('abs_change', 'percent_change', 'threshold')):
raise ElmConfigError(
'Expected "early_stop" to be a dict with a key in ("abs_change", "percent_change", "threshold")'
)
if not isinstance(control, dict):
raise ElmConfigError(
'Expected param_grids: {} - "control" to be a dict'.format(
control))
for required_key, typ in REQUIRED_CONTROL_KEYS_TYPES.items():
item = control.get(required_key) or None
if not isinstance(item, typ):
raise ElmConfigError('Expected params_grids:{} '
'"control" to have key {} with '
'type {}'.format(param_grid_name,
required_key, typ))
return _to_param_meta(param_grid, control)
def eval_stop_wrapper(evo_params, original_fitness):
'''Handle sections of config's {param_grids: {pg_name: control: {}}
Examples of configs handled by this function:
* :early_stop: {abs_change: [10], agg: all},
* :early_stop: {percent_change: [10], agg: all}
* :early_stop: {threshold: [10], agg: any}
Parameters:
:evo_params: EA parameters
Returns:
:decorator: Evaluates stop on each EA iteration
'''
early_stop = evo_params['early_stop']
if not early_stop:
# If not early_stop, this is a do-nothing function
value = [
-1,
] * len(evo_params['score_weights'])
func = _no_stop
elif 'percent_change' in early_stop:
typ = 'percent_change'
value = early_stop['percent_change']
_check_number(typ, value)
func = _percent_change_stop
elif 'threshold' in early_stop:
typ = 'threshold'
value = early_stop['threshold']
_check_number(typ, value, positive_definite=False)
func = _threshold_stop
elif 'abs_change' in early_stop:
typ = 'abs_change'
value = early_stop['abs_change']
_check_number(typ, value)
func = _abs_change_stop
else:
raise ValueError(
'early_stop:{} does not have a '
'key in ("abs_change", "percent_change", "threshold")'.format(
early_stop))
if not isinstance(value, Sequence):
raise ElmConfigError(
'Expected early_stop:{} to be a Sequence'.format(value))
if not len(evo_params['score_weights']) == len(value) == len(
original_fitness):
raise ElmConfigError('Expected score_weights {}, early_stop: {} ({}) '
'and fitness {} to all have same '
'len()'.format(evo_params['score_weights'], typ,
value, original_fitness))
if not early_stop:
agg = all
else:
early_stop.get('agg', all)
eval_stop = partial(func, agg, evo_params['score_weights'], value,
original_fitness)
return eval_stop
def ea_setup(config=None, param_grid=None, param_grid_name=None,
score_weights=None):
'''ea_setup parses a yaml config or param_grid dictionary to create an EvoParams instance with fields needed for algorithm
Parameters:
:config: elm.config.ConfigParser instance or None
:param_grid: param_grid dictionary if not giving config
:param_grid_name: name of param_grid, if not giving config
:score_weights: list of weights, -1 or 1, for each objective, ignored if config is not None
Returns:
* if config was given dict of `EvoParams` for each step in config's "run" list (integers as key, EvoParams instances as values)
* elif config was None `EvoParams` instance
'''
if config and not param_grid:
idx_to_param_grid = _get_evolve_meta(config)
items = tuple(idx_to_param_grid.items())
elif param_grid and not config:
items = ((0, check_format_param_grid(param_grid, param_grid_name)),)
else:
raise ValueError('Expected config or param_grid but not both')
if not items:
return {}
evo_params_dict = {}
for (idx, deap_params) in items:
if config:
param_grid_name = config.run[idx]['train']
kwargs = copy.deepcopy(deap_params['control'])
toolbox = base.Toolbox()
if config and score_weights is None:
train = config.train[param_grid_name]
if 'model_scoring' in train:
ms = config.model_scoring[train['model_scoring']]
if 'score_weights' in ms:
score_weights = ms['score_weights']
if score_weights is None:
raise ElmConfigError('Cannot continue EA '
'when score_weights is None (train section of config: '
'{})'.format(step_name))
creator.create('FitnessMulti', base.Fitness, weights=tuple(score_weights))
creator.create('Individual', list, fitness=creator.FitnessMulti)
toolbox.register('indices', _random_choice, deap_params['choices'])
toolbox.register('individual', tools.initIterate, creator.Individual,
toolbox.indices)
toolbox.register('population', tools.initRepeat, list, toolbox.individual)
early_stop = deap_params['control'].get('early_stop') or None
evo_params = EvoParams(
toolbox=_configure_toolbox(toolbox, deap_params, config, **kwargs),
deap_params=deap_params,
param_grid_name=param_grid_name,
history_file='{}.csv'.format(param_grid_name),
score_weights=score_weights,
early_stop=early_stop,
)
if not config:
return evo_params
evo_params_dict[idx] = evo_params
return evo_params_dict
def _check_number(typ, value, positive_definite=True):
'''Check all members of Sequence value are ints'''
if not all(
isinstance(val, numbers.Number) and
(val > 0 if positive_definite else True) for val in value):
raise ElmConfigError(
'With early_stop: {} expected a Sequence of positive ints (indices) but found {}'
.format(typ, value))
def get_param_grid(config, step):
'''Get the metadata for one config step that has a param_grid or return None'''
param_grid_name = step.get('param_grid') or None
if not param_grid_name:
return None
train_step_name = step.get('train')
if train_step_name:
train_config = config.train[train_step_name]
else:
raise ElmConfigError('Expected param_grid to be used with a "train" '
'step of a pipeline, but found param_grid '
'was used in step {}'.format(step))
param_grid = config.param_grids[param_grid_name]
return check_format_param_grid(param_grid, param_grid_name)
def main(args=None, sys_argv=None, return_0_if_ok=True):
raise ElmMainDeprecation('The console entry point elm-main for running '
'yaml configs is temporarily deprecated during '
'refactoring of elm')
started = datetime.datetime.now()
args = cli(args=args, sys_argv=sys_argv)
if args.config_dir is not None and args.config is not None:
raise ElmConfigError('Expected --config-dir or --config, not both args')
elif args.config_dir:
ret = run_many_configs(args, started=started, return_0_if_ok=return_0_if_ok)
logger.info('Elapsed time {}'.format((datetime.datetime.now() - started).total_seconds()))
ret = run_one_config(args=args, sys_argv=sys_argv,
return_0_if_ok=return_0_if_ok,
started=started)
return ret
def main(args=None, sys_argv=None, return_0_if_ok=True):
started = datetime.datetime.now()
args = cli(args=args, sys_argv=sys_argv)
if args.config_dir is not None and args.config is not None:
raise ElmConfigError(
'Expected --config-dir or --config, not both args')
elif args.config_dir:
ret = run_many_configs(args,
started=started,
return_0_if_ok=return_0_if_ok)
logger.info('Elapsed time {}'.format(
(datetime.datetime.now() - started).total_seconds()))
ret = run_one_config(args=args,
sys_argv=sys_argv,
return_0_if_ok=return_0_if_ok,
started=started)
return ret