def _setup(config=None):
'''Return the config above and the param_grid'''
from elm.sample_util.sample_pipeline import make_pipeline_steps
from elm.pipeline import Pipeline
if not config:
config = ConfigParser(config=yaml.load(CONFIG_STR))
sample_steps = make_pipeline_steps(config, config.run[0]['pipeline'])
estimator = [('kmeans', MiniBatchKMeans(**config.train['kmeans']['model_init_kwargs']))]
pipe = Pipeline(sample_steps + estimator)
idx_to_param_grid = ea_setup(config)
return config, pipe, idx_to_param_grid
def config_to_pipeline(config, client=None):
'''
Run the elm config's train and predict "run"
actions with dask client and config's updates
based on args passed to elm-main, such as --train-only
or --predict-only, or edits to ensemble settings, such as
--ngen 4
Parameters:
:config: elm.config.ConfigParser instance
:client: dask client or None
'''
from elm.sample_util.sample_pipeline import make_pipeline_steps
_makedirs(config)
idx_to_evo_params = ea_setup(config)
for idx, step in enumerate(config.run):
pipeline = step['pipeline']
if 'train' in step:
train = config.train[step['train']]
pipe_steps = make_pipeline_steps(config, pipeline)
cls = import_callable(train['model_init_class'])
estimator = cls(**(train.get('model_init_kwargs') or {}))
pipe_steps.append((step['train'], estimator))
ensemble_kwargs = train.get('ensemble')
if isinstance(ensemble_kwargs, str):
ensemble_kwargs = config.ensembles[ensemble_kwargs]
ensemble_kwargs['client'] = client
data_source = step['data_source']
if not isinstance(data_source, dict):
data_source = config.data_sources[data_source]
data_source['sampler'] = import_callable(data_source['sampler'])
data_source['load_meta'] = load_meta
data_source['load_array'] = load_array
if callable(data_source.get('args_list')):
kw = {k: v for k, v in data_source.items() if k != 'args_list'}
data_source['args_list'] = tuple(data_source['args_list'](**kw))
if 'train' in step and not getattr(config, 'PREDICT_ONLY', False):
s = train.get('model_scoring')
if s:
scoring = config.model_scoring[s]
scoring_kwargs = {
k: v
for k, v in scoring.items() if k != 'scoring'
}
scoring = import_callable(scoring['scoring'])
else:
scoring = None
scoring_kwargs = {}
if 'method_kwargs' in train:
method_kwargs = train['method_kwargs']
else:
method_kwargs = {}
if 'classes' in train:
method_kwargs['classes'] = train['classes']
ensemble_kwargs['method_kwargs'] = method_kwargs
pipe = Pipeline(pipe_steps,
scoring=scoring,
scoring_kwargs=scoring_kwargs)
evo_params = idx_to_evo_params.get(idx, None)
if evo_params:
kw = dict(evo_params=evo_params)
kw.update(data_source)
kw.update(ensemble_kwargs)
pipe.fit_ea(**kw)
else:
kw = {}
kw.update(data_source)
kw.update(ensemble_kwargs)
pipe.fit_ensemble(**kw)
serialize_pipe(pipe, config.ELM_TRAIN_PATH, step['train'])
elif 'predict' in step and not getattr(config, 'TRAIN_ONLY', False):
pipe = load_pipe_from_tag(config.ELM_TRAIN_PATH, step['predict'])
else:
logger.info(
'Do nothing for {} (has no "train" or "predict" key)'.format(
step))
if 'predict' in step:
# serialize is called with (prediction, sample, tag)
serialize = partial(serialize_prediction, config)
pipe.predict_many(serialize=serialize, **data_source)
'cxpb': 0.3,
'eta': 20,
'ngen': 2,
'mu': 4,
'k': 4,
'early_stop': {
'abs_change': [10],
'agg': 'all'
},
# alternatively early_stop: {percent_change: [10], agg: all}
# alternatively early_stop: {threshold: [10], agg: any}
}
}
evo_params = ea_setup(param_grid=param_grid,
param_grid_name='param_grid_example',
score_weights=[-1]) # minimization
def main():
with client_context() as client:
fitted = pipeline.fit_ea(
evo_params=evo_params,
client=client,
saved_ensemble_size=param_grid['control']['mu'],
**data_source)
preds = pipeline.predict_many(client=client, **data_source)
return fitted, preds
if __name__ == '__main__':