def test_make_ts_scorer(greater_is_better, needs_proba, needs_threshold, expected_error):
if expected_error is None:
scorer = make_ts_scorer(
mean_absolute_error,
greater_is_better=greater_is_better,
needs_proba=needs_proba,
needs_threshold=needs_threshold,
)
isinstance(scorer, _TSPredictScorer)
else:
with pytest.raises(expected_error):
_ = make_ts_scorer(
mean_absolute_error,
greater_is_better=greater_is_better,
needs_proba=needs_proba,
needs_threshold=needs_threshold,
)
def grid_search(request):
from sklearn.dummy import DummyRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from hcrystalball.feature_extraction import HolidayTransformer
from hcrystalball.feature_extraction import SeasonalityTransformer
from hcrystalball.metrics import make_ts_scorer
from hcrystalball.model_selection import FinerTimeSplit
from hcrystalball.wrappers import get_sklearn_wrapper
scoring = make_ts_scorer(mean_absolute_error, greater_is_better=False)
bad_dummy = get_sklearn_wrapper(DummyRegressor,
strategy="constant",
constant=42,
name="bad_dummy",
lags=2)
good_dummy = get_sklearn_wrapper(DummyRegressor,
strategy="mean",
name="good_dummy",
lags=2)
parameters = [
{
"model": [good_dummy]
},
{
"model": [bad_dummy],
"model__strategy": ["constant"],
"model__constant": [42],
},
]
holiday_model = Pipeline([
("holiday", HolidayTransformer(country_code_column="Holidays_code")),
("seasonality", SeasonalityTransformer(week_day=True, freq="D")),
("model", good_dummy),
])
cv = FinerTimeSplit(n_splits=2, horizon=5)
grid_search = GridSearchCV(holiday_model,
parameters,
cv=cv,
scoring=scoring)
return grid_search
def test_save_model_repr_and_hash(X_y_linear_trend, model4persistence):
X, y = X_y_linear_trend
test_scorer = make_ts_scorer(
mean_absolute_error,
greater_is_better=False,
needs_proba=False,
needs_threshold=False,
)
estimators, expected_model_ids, _ = model4persistence
for iestimator in estimators:
iestimator.fit(X, y)
_ = test_scorer(iestimator, X, y)
assert test_scorer.estimator_ids == expected_model_ids
needs_threshold=False,
)
estimators, expected_model_ids, _ = model4persistence
for iestimator in estimators:
iestimator.fit(X, y)
_ = test_scorer(iestimator, X, y)
assert test_scorer.estimator_ids == expected_model_ids
@pytest.mark.parametrize(
"function, expected_error",
[
("neg_mean_absolute_error", None),
(make_ts_scorer(mean_absolute_error), None),
(mean_absolute_error, ValueError),
],
)
def test_get_scorer(function, expected_error):
if expected_error is None:
result = get_scorer(function)
assert all([
hasattr(result, "_cv_data"),
hasattr(result, "_estimator_ids"),
hasattr(result, "_sign"),
hasattr(result, "_score_func"),
])
else:
with pytest.raises(expected_error):