def cross_validate(data=None,
folds=5,
repeat=1,
metrics=None,
reporters=None,
model_def=None,
**kwargs):
"""Shortcut to cross-validate a single configuration.
ModelDefinition variables are passed in as keyword args, along
with the cross-validation parameters.
"""
md_kwargs = {}
if model_def is None:
for arg in ModelDefinition.params:
if arg in kwargs:
md_kwargs[arg] = kwargs.pop(arg)
model_def = ModelDefinition(**md_kwargs)
if metrics is None:
metrics = []
if reporters is None:
reporters = []
metrics = [MetricReporter(metric) for metric in metrics]
results = modeling.cross_validate(model_def,
data,
folds,
repeat=repeat,
**kwargs)
for r in reporters + metrics:
r.process_results(results)
return CVResult(results, reporters, metrics)
def cross_validate(data=None, folds=None, repeat=1, **kwargs):
"""Shortcut to cross-validate a single configuration.
ModelDefinition variables are passed in as keyword args, along
with the cross-validation parameters.
"""
md_kwargs = {}
for arg in ModelDefinition.params:
if arg in kwargs:
md_kwargs[arg] = kwargs.pop(arg)
model_def = ModelDefinition(**md_kwargs)
return modeling.cross_validate(model_def, data, folds, repeat=repeat, **kwargs)
def test_cross_validate_with_alternative_predictions(self):
features = [F(10), F("a")]
target = Map("b", np.log)
estimator = DummyEstimator()
model_def = ModelDefinition(
features=features,
estimator=estimator,
target=target,
evaluation_transformation=Map("__predictions", np.exp),
evaluation_target=F("b"),
)
results = cross_validate(model_def, self.data, folds=3)
self.assertEqual(len(results), 3)
# assert we've transformed predictions correctly
yt = results[0].y_test
assert_almost_equal(yt.values, self.data["b"].reindex(yt.index).values)
assert_almost_equal(estimator.fity, np.log(self.data["b"].reindex(results[-1].y_train.index)).values)
def cross_validate(data=None, folds=5, repeat=1, metrics=None,
reporters=None, model_def=None, **kwargs):
"""Shortcut to cross-validate a single configuration.
ModelDefinition variables are passed in as keyword args, along
with the cross-validation parameters.
"""
md_kwargs = {}
if model_def is None:
for arg in ModelDefinition.params:
if arg in kwargs:
md_kwargs[arg] = kwargs.pop(arg)
model_def = ModelDefinition(**md_kwargs)
if metrics is None:
metrics = []
if reporters is None:
reporters = []
metrics = [MetricReporter(metric) for metric in metrics]
results = modeling.cross_validate(model_def, data, folds, repeat=repeat, **kwargs)
for r in reporters + metrics:
r.process_results(results)
return CVResult(results, reporters, metrics)
def test_cross_validate(self):
model_def = self.make_model_def_basic()
results, reporters = cross_validate(model_def, self.data, folds=3)
self.assertEqual(len(results), 3)