Esempio n. 1
0
def test_tweedie_convergence(max_depth, split_criterion):
    np.random.seed(33)
    bootstrap = None
    max_features = 1.0
    n_estimators = 1
    min_impurity_decrease = 1e-5
    n_datapoints = 1000
    tweedie = {
        "poisson": {
            "power": 1,
            "gen": np.random.poisson,
            "args": [0.01]
        },
        "gamma": {
            "power": 2,
            "gen": np.random.gamma,
            "args": [2.0]
        },
        "inverse_gaussian": {
            "power": 3,
            "gen": np.random.wald,
            "args": [0.1, 2.0]
        }
    }
    # generating random dataset with tweedie distribution
    X = np.random.random((n_datapoints, 4)).astype(np.float32)
    y = tweedie[split_criterion]["gen"](*tweedie[split_criterion]["args"],
                                        size=n_datapoints).astype(np.float32)

    tweedie_preds = curfr(split_criterion=split_criterion,
                          max_depth=max_depth,
                          n_estimators=n_estimators,
                          bootstrap=bootstrap,
                          max_features=max_features,
                          min_impurity_decrease=min_impurity_decrease).fit(
                              X, y).predict(X)
    mse_preds = curfr(split_criterion=2,
                      max_depth=max_depth,
                      n_estimators=n_estimators,
                      bootstrap=bootstrap,
                      max_features=max_features,
                      min_impurity_decrease=min_impurity_decrease).fit(
                          X, y).predict(X)
    # y should not be non-positive for mean_poisson_deviance
    mask = mse_preds > 0
    mse_tweedie_deviance = mean_tweedie_deviance(
        y[mask], mse_preds[mask], power=tweedie[split_criterion]["power"])
    tweedie_tweedie_deviance = mean_tweedie_deviance(
        y[mask], tweedie_preds[mask], power=tweedie[split_criterion]["power"])

    # model trained on tweedie data with
    # tweedie criterion must perform better on tweedie loss
    assert mse_tweedie_deviance >= tweedie_tweedie_deviance
Esempio n. 2
0
def test_tweedie_deviance_continuity():
    n_samples = 100

    y_true = np.random.RandomState(0).rand(n_samples) + 0.1
    y_pred = np.random.RandomState(1).rand(n_samples) + 0.1

    assert_allclose(
        mean_tweedie_deviance(y_true, y_pred, power=0 - 1e-10),
        mean_tweedie_deviance(y_true, y_pred, power=0),
    )

    # Ws we get closer to the limit, with 1e-12 difference the absolute
    # tolerance to pass the below check increases. There are likely
    # numerical precision issues on the edges of different definition
    # regions.
    assert_allclose(
        mean_tweedie_deviance(y_true, y_pred, power=1 + 1e-10),
        mean_tweedie_deviance(y_true, y_pred, power=1),
        atol=1e-6,
    )

    assert_allclose(
        mean_tweedie_deviance(y_true, y_pred, power=2 - 1e-10),
        mean_tweedie_deviance(y_true, y_pred, power=2),
        atol=1e-6,
    )

    assert_allclose(
        mean_tweedie_deviance(y_true, y_pred, power=2 + 1e-10),
        mean_tweedie_deviance(y_true, y_pred, power=2),
        atol=1e-6,
    )
Esempio n. 3
0
    def evaluate_forecast(self):
        n = min(len(self.validation_data), len(self.forecasts))
        y_forecast = self.forecasts[:n]
        y_actual = self.validation_data.tail(n)["close"]

        mean_abs_err = learn.mean_absolute_error(y_actual, y_forecast)
        mean_sq_err = learn.mean_squared_error(y_actual, y_forecast)
        mean_sq_lg_err = learn.mean_squared_log_error(y_actual, y_forecast)
        mean_abs_percent_err = learn.mean_absolute_percentage_error(
            y_actual, y_forecast)
        median_abs_err = learn.median_absolute_error(y_actual, y_forecast)
        mean_gamma_dev = learn.mean_gamma_deviance(y_actual, y_forecast)
        mean_poisson_dev = learn.mean_poisson_deviance(y_actual, y_forecast)
        mean_tweedie_dev = learn.mean_tweedie_deviance(y_actual, y_forecast)
        explained_variance = learn.explained_variance_score(
            y_actual, y_forecast)
        max_residual = learn.max_error(y_actual, y_forecast)
        coeff_determination = learn.r2_score(y_actual, y_forecast)

        metrics = {
            "Mean Squared Error (MSE)": mean_sq_err,
            "Mean Absolute Error (MAE)": mean_abs_err,
            "Mean Squared Logarithmic Error (MSLE)": mean_sq_lg_err,
            "Mean Absolute Percentage Error (MAPE)": mean_abs_percent_err,
            "Median Absolute Error (MedAE)": median_abs_err,
            "Mean Gamma Deviance": mean_gamma_dev,
            "Mean Poisson Deviance": mean_poisson_dev,
            "Mean Tweedie Deviance Error": mean_tweedie_dev,
            "Explained Variance Regression Score": explained_variance,
            "Max Residual Error": max_residual,
            "Coefficient of Determination": coeff_determination
        }
        self.metrics = metrics
    def mtd(self) -> float:
        """
        Mean tweedie deviance error metric for regression problems

        :return: float
            Mean-Tweedie-Deviance-Error Score
        """
        return mean_tweedie_deviance(y_true=self.obs,
                                     y_pred=self.pred,
                                     sample_weight=None)
Esempio n. 5
0
    def validate(self, X_test, y_test):
        if not self._is_fitted:
            raise Exception("Model is not fitted.")

        scores = {}
        scores_exp = {}

        y_pred = self.model.predict(X_test)

        scores["r2"] = r2_score(y_test, y_pred)
        scores["mse"] = mean_tweedie_deviance(y_test, y_pred, power=0)
        scores["poisson_deviance"] = mean_tweedie_deviance(y_test,
                                                           y_pred,
                                                           power=1)
        scores["gamma_deviance"] = mean_tweedie_deviance(y_test,
                                                         y_pred,
                                                         power=2)
        self.test_scores = scores

        scores_exp["r2"] = r2_score(np.exp(y_test), np.exp(y_pred))
        scores_exp["mse"] = mean_tweedie_deviance(np.exp(y_test),
                                                  np.exp(y_pred),
                                                  power=0)
        scores_exp["poisson_deviance"] = mean_tweedie_deviance(np.exp(y_test),
                                                               np.exp(y_pred),
                                                               power=1)
        scores_exp["gamma_deviance"] = mean_tweedie_deviance(np.exp(y_test),
                                                             np.exp(y_pred),
                                                             power=2)
        self.test_scores_exp = scores_exp

        return self.test_scores, self.test_scores_exp
Esempio n. 6
0
def test_regression_metrics(n_samples=50):
    y_true = np.arange(n_samples)
    y_pred = y_true + 1

    assert_almost_equal(mean_squared_error(y_true, y_pred), 1.)
    assert_almost_equal(mean_squared_log_error(y_true, y_pred),
                        mean_squared_error(np.log(1 + y_true),
                                           np.log(1 + y_pred)))
    assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.)
    assert_almost_equal(median_absolute_error(y_true, y_pred), 1.)
    assert_almost_equal(max_error(y_true, y_pred), 1.)
    assert_almost_equal(r2_score(y_true, y_pred),  0.995, 2)
    assert_almost_equal(explained_variance_score(y_true, y_pred), 1.)
    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=0),
                        mean_squared_error(y_true, y_pred))

    # Tweedie deviance needs positive y_pred, except for p=0,
    # p>=2 needs positive y_true
    # results evaluated by sympy
    y_true = np.arange(1, 1 + n_samples)
    y_pred = 2 * y_true
    n = n_samples
    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=-1),
                        5/12 * n * (n**2 + 2 * n + 1))
    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=1),
                        (n + 1) * (1 - np.log(2)))
    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=2),
                        2 * np.log(2) - 1)
    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=3/2),
                        ((6 * np.sqrt(2) - 8) / n) * np.sqrt(y_true).sum())
    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=3),
                        np.sum(1 / y_true) / (4 * n))
def test_regression_metrics(n_samples=50):
    y_true = np.arange(n_samples)
    y_pred = y_true + 1
    y_pred_2 = y_true - 1

    assert_almost_equal(mean_squared_error(y_true, y_pred), 1.0)
    assert_almost_equal(
        mean_squared_log_error(y_true, y_pred),
        mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred)),
    )
    assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.0)
    assert_almost_equal(mean_pinball_loss(y_true, y_pred), 0.5)
    assert_almost_equal(mean_pinball_loss(y_true, y_pred_2), 0.5)
    assert_almost_equal(mean_pinball_loss(y_true, y_pred, alpha=0.4), 0.6)
    assert_almost_equal(mean_pinball_loss(y_true, y_pred_2, alpha=0.4), 0.4)
    assert_almost_equal(median_absolute_error(y_true, y_pred), 1.0)
    mape = mean_absolute_percentage_error(y_true, y_pred)
    assert np.isfinite(mape)
    assert mape > 1e6
    assert_almost_equal(max_error(y_true, y_pred), 1.0)
    assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2)
    assert_almost_equal(explained_variance_score(y_true, y_pred), 1.0)
    assert_almost_equal(
        mean_tweedie_deviance(y_true, y_pred, power=0),
        mean_squared_error(y_true, y_pred),
    )
    assert_almost_equal(d2_tweedie_score(y_true, y_pred, power=0),
                        r2_score(y_true, y_pred))

    # Tweedie deviance needs positive y_pred, except for p=0,
    # p>=2 needs positive y_true
    # results evaluated by sympy
    y_true = np.arange(1, 1 + n_samples)
    y_pred = 2 * y_true
    n = n_samples
    assert_almost_equal(
        mean_tweedie_deviance(y_true, y_pred, power=-1),
        5 / 12 * n * (n**2 + 2 * n + 1),
    )
    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=1),
                        (n + 1) * (1 - np.log(2)))
    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=2),
                        2 * np.log(2) - 1)
    assert_almost_equal(
        mean_tweedie_deviance(y_true, y_pred, power=3 / 2),
        ((6 * np.sqrt(2) - 8) / n) * np.sqrt(y_true).sum(),
    )
    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=3),
                        np.sum(1 / y_true) / (4 * n))

    dev_mean = 2 * np.mean(xlogy(y_true, 2 * y_true / (n + 1)))
    assert_almost_equal(
        d2_tweedie_score(y_true, y_pred, power=1),
        1 - (n + 1) * (1 - np.log(2)) / dev_mean,
    )

    dev_mean = 2 * np.log((n + 1) / 2) - 2 / n * np.log(factorial(n))
    assert_almost_equal(d2_tweedie_score(y_true, y_pred, power=2),
                        1 - (2 * np.log(2) - 1) / dev_mean)
Esempio n. 8
0
def mean_tweedie_deviance(y_true: np.ndarray,
                          y_pred: np.ndarray,
                          *,
                          sample_weight: Optional[np.ndarray] = None,
                          power: float = 0) -> float:
    """Mean Tweedie deviance regression loss.

    Read more in the :ref:`User Guide <mean_tweedie_deviance>`.

    Parameters
    ----------
    y_true : array-like of shape (n_samples,)
        Ground truth (correct) target values.
    y_pred : array-like of shape (n_samples,)
        Estimated target values.
    sample_weight : array-like of shape (n_samples,), default=None
        Sample weights.
    power : float, default=0
        Tweedie power parameter. Either power <= 0 or power >= 1.
        The higher `p` the less weight is given to extreme
        deviations between true and predicted targets.
        - power < 0: Extreme stable distribution. Requires: y_pred > 0.
        - power = 0 : Normal distribution, output corresponds to
          mean_squared_error. y_true and y_pred can be any real numbers.
        - power = 1 : Poisson distribution. Requires: y_true >= 0 and
          y_pred > 0.
        - 1 < p < 2 : Compound Poisson distribution. Requires: y_true >= 0
          and y_pred > 0.
        - power = 2 : Gamma distribution. Requires: y_true > 0 and y_pred > 0.
        - power = 3 : Inverse Gaussian distribution. Requires: y_true > 0
          and y_pred > 0.
        - otherwise : Positive stable distribution. Requires: y_true > 0
          and y_pred > 0.

    Returns
    -------
    loss : float
        A non-negative floating point value (the best value is 0.0).
    """
    y_type, y_true, y_pred, sample_weight, multioutput = _check_reg_targets(
        y_true, y_pred, sample_weight)
    if sample_weight is not None:
        check_consistent_length(y_true, y_pred, sample_weight)
    else:
        check_consistent_length(y_true, y_pred)
    return sklearn_metrics.mean_tweedie_deviance(y_true=y_true,
                                                 y_pred=y_pred,
                                                 sample_weight=sample_weight,
                                                 power=power)
Esempio n. 9
0
def set_metrics(y_pred, y_true, dict):
    try:
        dict["max_error"] = mets.max_error(y_true, y_pred)
    except:
        pass
    try:
        dict["explained_variance_score"] = mets.explained_variance_score(y_true, y_pred)
    except:
        pass
    try:
        dict["mean_absolute_error"] = mets.mean_absolute_error(y_true, y_pred)
    except:
        pass
    try:
        dict["mean_squared_error"] = mets.mean_squared_error(y_true, y_pred)
    except:
        pass
    try:
        dict["mean_squared_log_error"] = mets.mean_squared_log_error(y_true, y_pred)
    except:
        pass
    try:
        dict["median_absolute_error"] = mets.median_absolute_error(y_true, y_pred)
    except:
        pass
    try:
        dict["r2_score"] = mets.r2_score(y_true, y_pred)
    except:
        pass
    try:
        dict["mean_poisson_deviance"] = mets.mean_poisson_deviance(y_true, y_pred)
    except:
        pass
    try:
        dict["mean_gamma_deviance"] = mets.mean_gamma_deviance(y_true, y_pred)
    except:
        pass
    try:
        dict["mean_tweedie_deviance"] =  mets.mean_tweedie_deviance(y_true, y_pred)
    except:
        pass
    return dict
Esempio n. 10
0
def test_regression_metrics_at_limits():
    # Single-sample case
    # Note: for r2 and d2_tweedie see also test_regression_single_sample
    assert_almost_equal(mean_squared_error([0.0], [0.0]), 0.0)
    assert_almost_equal(mean_squared_error([0.0], [0.0], squared=False), 0.0)
    assert_almost_equal(mean_squared_log_error([0.0], [0.0]), 0.0)
    assert_almost_equal(mean_absolute_error([0.0], [0.0]), 0.0)
    assert_almost_equal(mean_pinball_loss([0.0], [0.0]), 0.0)
    assert_almost_equal(mean_absolute_percentage_error([0.0], [0.0]), 0.0)
    assert_almost_equal(median_absolute_error([0.0], [0.0]), 0.0)
    assert_almost_equal(max_error([0.0], [0.0]), 0.0)
    assert_almost_equal(explained_variance_score([0.0], [0.0]), 1.0)

    # Perfect cases
    assert_almost_equal(r2_score([0.0, 1], [0.0, 1]), 1.0)
    assert_almost_equal(d2_pinball_score([0.0, 1], [0.0, 1]), 1.0)

    # Non-finite cases
    # R² and explained variance have a fix by default for non-finite cases
    for s in (r2_score, explained_variance_score):
        assert_almost_equal(s([0, 0], [1, -1]), 0.0)
        assert_almost_equal(s([0, 0], [1, -1], force_finite=False), -np.inf)
        assert_almost_equal(s([1, 1], [1, 1]), 1.0)
        assert_almost_equal(s([1, 1], [1, 1], force_finite=False), np.nan)
    msg = ("Mean Squared Logarithmic Error cannot be used when targets "
           "contain negative values.")
    with pytest.raises(ValueError, match=msg):
        mean_squared_log_error([-1.0], [-1.0])
    msg = ("Mean Squared Logarithmic Error cannot be used when targets "
           "contain negative values.")
    with pytest.raises(ValueError, match=msg):
        mean_squared_log_error([1.0, 2.0, 3.0], [1.0, -2.0, 3.0])
    msg = ("Mean Squared Logarithmic Error cannot be used when targets "
           "contain negative values.")
    with pytest.raises(ValueError, match=msg):
        mean_squared_log_error([1.0, -2.0, 3.0], [1.0, 2.0, 3.0])

    # Tweedie deviance error
    power = -1.2
    assert_allclose(mean_tweedie_deviance([0], [1.0], power=power),
                    2 / (2 - power),
                    rtol=1e-3)
    msg = "can only be used on strictly positive y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.0], [0.0], power=power)
    with pytest.raises(ValueError, match=msg):
        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)

    assert_almost_equal(mean_tweedie_deviance([0.0], [0.0], power=0), 0.0, 2)

    power = 1.0
    msg = "only be used on non-negative y and strictly positive y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.0], [0.0], power=power)
    with pytest.raises(ValueError, match=msg):
        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)

    power = 1.5
    assert_allclose(mean_tweedie_deviance([0.0], [1.0], power=power),
                    2 / (2 - power))
    msg = "only be used on non-negative y and strictly positive y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.0], [0.0], power=power)
    with pytest.raises(ValueError, match=msg):
        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)

    power = 2.0
    assert_allclose(mean_tweedie_deviance([1.0], [1.0], power=power),
                    0.00,
                    atol=1e-8)
    msg = "can only be used on strictly positive y and y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.0], [0.0], power=power)
    with pytest.raises(ValueError, match=msg):
        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)

    power = 3.0
    assert_allclose(mean_tweedie_deviance([1.0], [1.0], power=power),
                    0.00,
                    atol=1e-8)
    msg = "can only be used on strictly positive y and y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.0], [0.0], power=power)
    with pytest.raises(ValueError, match=msg):
        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)

    power = 0.5
    with pytest.raises(ValueError,
                       match="is only defined for power<=0 and power>=1"):
        mean_tweedie_deviance([0.0], [0.0], power=power)
    with pytest.raises(ValueError,
                       match="is only defined for power<=0 and power>=1"):
        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
    def score(self,
              actual: np.array,
              predicted: np.array,
              sample_weight: typing.Optional[np.array] = None,
              labels: typing.Optional[np.array] = None) -> float:
        """

        :param actual:          Ground truth (correct) target values.
        :param predicted:       Estimated target values
        :param sample_weight:   weights
        :param labels:          not used

        power:                  default=1.5 sent to function via toml dictionary
        Tweedie power parameter. Either power <= 0 or power >= 1.
        To non-default power parameter use recipe_dict add via toml config DAI option. Example:
        recipe_dict = "{'power':2.0}"
        Multiple parameters example (first param is for demo only):
        validate_meta_learner=false\nrecipe_dict = "{'power':2.0}"

        The higher p the less weight is given to extreme deviations between true and predicted targets.
        power < 0: Extreme stable distribution. Requires: y_pred > 0.
        power = 0 : Normal distribution, output corresponds to mean_squared_error. y_true and y_pred can be any real numbers.
        power = 1 : Poisson distribution. Requires: y_true >= 0 and y_pred > 0.
        1 < p < 2 : Compound Poisson distribution. Requires: y_true >= 0 and y_pred > 0.
        power = 2 : Gamma distribution. Requires: y_true > 0 and y_pred > 0.
        power = 3 : Inverse Gaussian distribution. Requires: y_true > 0 and y_pred > 0.
        otherwise : Positive stable distribution. Requires: y_true > 0 and y_pred > 0.

        :return: score
        """

        try:
            """Initialize logger to print additional info in case of invalid inputs(exception is raised) and to enable debug prints"""
            logger = self.logger
            from h2oaicore.systemutils import loggerinfo
            # loggerinfo(logger, "Start TW Deviance Scorer.......")
            # loggerinfo(logger, 'Actual:%s' % str(actual))
            # loggerinfo(logger, 'Predicted:%s' % str(predicted))
            # loggerinfo(logger, 'Sample W:%s' % str(sample_weight))

            from sklearn.metrics import mean_tweedie_deviance
            if config.recipe_dict is not None:
                power = config.recipe_dict.get('power', 1.5)
            else:
                power = 1.5
            # loggerinfo(logger, 'Power:%s' % str(power))

            if sample_weight is not None:
                '''Check if any element of the sample_weight array is nan'''
                if np.isnan(np.sum(sample_weight)):
                    loggerinfo(logger, 'Sample Weight:%s' % str(sample_weight))
                    loggerinfo(
                        logger, 'Sample Weight Nan values index:%s' %
                        str(np.argwhere(np.isnan(sample_weight))))
                    raise RuntimeError(
                        'Error during Tweedie Deviance score calculation. Invalid sample weight values. Expecting only non-nan values'
                    )

            if 0 < power < 1:
                loggerinfo(logger, 'Power:%s' % str(power))
                loggerinfo(
                    logger,
                    """Invalid power value. Power should be one of the following: \n
                            power < 0: Extreme stable distribution. Requires: y_pred > 0.
                            power = 0 : Normal distribution, output corresponds to mean_squared_error. y_true and y_pred can be any real numbers.
                            power = 1 : Poisson distribution.          Requires: y_true >= 0 and y_pred > 0.
                            1 < p < 2 : Compound Poisson distribution. Requires: y_true >= 0 and y_pred > 0.
                            power = 2 : Gamma distribution.            Requires: y_true > 0 and y_pred > 0.
                            power = 3 : Inverse Gaussian distribution. Requires: y_true > 0 and y_pred > 0.
                            otherwise : Positive stable distribution.  Requires: y_true > 0 and y_pred > 0."""
                )
                raise RuntimeError(
                    'Error during Tweedie Deviance score calculation. Invalid power value.'
                )

            actual = actual.astype('float64')
            predicted = predicted.astype('float64')
            '''Safety mechanizm in case predictions or actuals are zero'''
            epsilon = 1E-8
            actual += epsilon
            predicted += epsilon

            if power == 0:
                '''No need to validate sign of actual or predicted'''
                pass
            elif power < 0:
                if (predicted <= 0).any():
                    loggerinfo(logger, 'Predicted:%s' % str(predicted))
                    loggerinfo(
                        logger, 'Invalid Predicted:%s' %
                        str(predicted[predicted <= 0]))
                    raise RuntimeError(
                        'Power <0. Error during Tweedie Deviance score calculation. Invalid predicted values. Expecting only positive values'
                    )
            elif 1 <= power < 2:
                if (actual < 0).any():
                    loggerinfo(logger, 'Actual:%s' % str(actual))
                    loggerinfo(
                        logger,
                        'Non-positive Actuals:%s' % str(actual[actual < 0]))
                    raise RuntimeError(
                        '1 <= power < 2. Error during Tweedie Deviance score calculation. Invalid actuals values. Expecting zero or positive values'
                    )
                if (predicted <= 0).any() or np.isnan(np.sum(predicted)):
                    loggerinfo(logger, 'Predicted:%s' % str(predicted))
                    loggerinfo(
                        logger, 'Invalid Predicted:%s' %
                        str(predicted[predicted <= 0]))
                    raise RuntimeError(
                        '1 <= power < 2. Error during Tweedie Deviance score calculation. Invalid predicted values. Expecting only positive values'
                    )
            elif power >= 2:
                if (actual <= 0).any():
                    loggerinfo(logger, 'Actual:%s' % str(actual))
                    loggerinfo(
                        logger,
                        'Non-positive Actuals:%s' % str(actual[actual <= 0]))
                    raise RuntimeError(
                        'power >= 2. Error during Tweedie Deviance score calculation. Invalid actuals values. Expecting zero or positive values'
                    )
                if (predicted <= 0).any() or np.isnan(np.sum(predicted)):
                    loggerinfo(logger, 'Predicted:%s' % str(predicted))
                    loggerinfo(
                        logger, 'Invalid Predicted:%s' %
                        str(predicted[predicted <= 0]))
                    raise RuntimeError(
                        'power >= 2. Error during Tweedie Deviance score calculation. Invalid predicted values. Expecting only positive values'
                    )
            '''Check if any element of the arrays is nan'''
            if np.isnan(np.sum(actual)):
                loggerinfo(logger, 'Actual:%s' % str(actual))
                loggerinfo(
                    logger,
                    'Nan values index:%s' % str(np.argwhere(np.isnan(actual))))
                raise RuntimeError(
                    'Error during Tweedie Deviance score calculation. Invalid actuals values. Expecting only non-nan values'
                )
            if np.isnan(np.sum(predicted)):
                loggerinfo(logger, 'Predicted:%s' % str(predicted))
                loggerinfo(
                    logger, 'Nan values index:%s' %
                    str(np.argwhere(np.isnan(predicted))))
                raise RuntimeError(
                    'Error during Tweedie Deviance score calculation. Invalid predicted values. Expecting only non-nan values'
                )

            score = mean_tweedie_deviance(actual,
                                          predicted,
                                          sample_weight=sample_weight,
                                          power=power)
            '''Validate that score is non-negative and is not infinity or Nan'''
            if score >= 0 and score < float("inf"):
                pass
            else:
                loggerinfo(logger, 'Invalid calculated score:%s' % str(score))
                raise RuntimeError(
                    'Error during Tweedie Deviance score calculation. Invalid calculated score:%s. \
                     Score should be non-negative and less than infinity. Nan is not valid'
                    % str(score))
        except Exception as e:
            '''Print error message into DAI log file'''
            loggerinfo(
                logger,
                'Error during Tweedie Deviance score calculation. Exception raised: %s'
                % str(e))
            raise
        return score
Esempio n. 12
0
def test_regression_metrics_at_limits():
    assert_almost_equal(mean_squared_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(mean_squared_log_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(mean_absolute_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(median_absolute_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(max_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(explained_variance_score([0.], [0.]), 1.00, 2)
    assert_almost_equal(r2_score([0., 1], [0., 1]), 1.00, 2)
    assert_raises_regex(
        ValueError, "Mean Squared Logarithmic Error cannot be "
        "used when targets contain negative values.", mean_squared_log_error,
        [-1.], [-1.])
    assert_raises_regex(
        ValueError, "Mean Squared Logarithmic Error cannot be "
        "used when targets contain negative values.", mean_squared_log_error,
        [1., 2., 3.], [1., -2., 3.])
    assert_raises_regex(
        ValueError, "Mean Squared Logarithmic Error cannot be "
        "used when targets contain negative values.", mean_squared_log_error,
        [1., -2., 3.], [1., 2., 3.])

    # Tweedie deviance error
    p = -1.2
    assert_allclose(mean_tweedie_deviance([0], [1.], p=p),
                    2. / (2. - p),
                    rtol=1e-3)
    with pytest.raises(ValueError,
                       match="can only be used on strictly positive y_pred."):
        mean_tweedie_deviance([0.], [0.], p=p)
    assert_almost_equal(mean_tweedie_deviance([0.], [0.], p=0), 0.00, 2)

    msg = "only be used on non-negative y_true and strictly positive y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.], [0.], p=1.0)

    p = 1.5
    assert_allclose(mean_tweedie_deviance([0.], [1.], p=p), 2. / (2. - p))
    msg = "only be used on non-negative y_true and strictly positive y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.], [0.], p=p)
    p = 2.
    assert_allclose(mean_tweedie_deviance([1.], [1.], p=p), 0.00, atol=1e-8)
    msg = "can only be used on strictly positive y_true and y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.], [0.], p=p)
    p = 3.
    assert_allclose(mean_tweedie_deviance([1.], [1.], p=p), 0.00, atol=1e-8)

    msg = "can only be used on strictly positive y_true and y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.], [0.], p=p)

    with pytest.raises(ValueError,
                       match="deviance is only defined for p<=0 and p>=1."):
        mean_tweedie_deviance([0.], [0.], p=0.5)
Esempio n. 13
0
    def cross_validate(self, X, y, k=None):
        if not self._is_fitted:
            raise Exception("Model is not fitted.")

        if self.gs_model is None:
            raise Exception("Cross validation requires a grid search model.")

        if k == None:
            k = int(X.shape[0] / 2) if X.shape[0] < 3 else 3

        print(f"Cross validating on '{k}' folds")

        scores = {}
        scores["r2"] = []
        scores["mse"] = []
        scores["poisson_deviance"] = []
        scores["gamma_deviance"] = []

        scores_exp = {}
        scores_exp["r2"] = []
        scores_exp["mse"] = []
        scores_exp["poisson_deviance"] = []
        scores_exp["gamma_deviance"] = []

        kf = KFold(k)

        for train_index, test_index in kf.split(X):

            clf_model = RandomForestQuantileRegressor(
                **self.gs_model.best_params_)
            clf_model.fit(X[train_index, :], y[train_index])

            y_pred = clf_model.predict(X[test_index, :])
            y_test = y[test_index]

            if test_index.__len__() > 1:
                r2 = r2_score(y_test, y_pred)
                scores["r2"].append(r2)

                r2_exp = r2_score(np.exp(y_test), np.exp(y_pred))
                scores_exp["r2"].append(r2_exp)

            mse = mean_tweedie_deviance(y_test, y_pred, power=0)
            poisson_deviance = mean_tweedie_deviance(y_test, y_pred, power=1)
            gamma_deviance = mean_tweedie_deviance(y_test, y_pred, power=2)

            scores["mse"].append(mse)
            scores["poisson_deviance"].append(poisson_deviance)
            scores["gamma_deviance"].append(gamma_deviance)

            mse_exp = mean_tweedie_deviance(np.exp(y_test),
                                            np.exp(y_pred),
                                            power=0)
            poisson_deviance_exp = mean_tweedie_deviance(np.exp(y_test),
                                                         np.exp(y_pred),
                                                         power=1)
            gamma_deviance_exp = mean_tweedie_deviance(np.exp(y_test),
                                                       np.exp(y_pred),
                                                       power=2)

            scores_exp["mse"].append(mse_exp)
            scores_exp["poisson_deviance"].append(poisson_deviance_exp)
            scores_exp["gamma_deviance"].append(gamma_deviance_exp)

        final_scores = {}
        final_scores["r2_mean"] = np.mean(
            [0 if s < 0 else s for s in scores["r2"]])
        final_scores["r2_std"] = np.std(
            [0 if s < 0 else s for s in scores["r2"]])
        final_scores["mse_mean"] = np.mean(scores["mse"])
        final_scores["mse_std"] = np.std(scores["mse"])
        final_scores["poisson_deviance_mean"] = np.mean(
            scores["poisson_deviance"])
        final_scores["poisson_deviance_std"] = np.std(
            scores["poisson_deviance"])
        final_scores["gamma_deviance_mean"] = np.mean(scores["gamma_deviance"])
        final_scores["gamma_deviance_std"] = np.std(scores["gamma_deviance"])
        self.cross_validation_scores = final_scores

        final_scores_exp = {}
        final_scores_exp["r2_mean"] = np.mean(
            [0 if s < 0 else s for s in scores_exp["r2"]])
        final_scores_exp["r2_std"] = np.std(
            [0 if s < 0 else s for s in scores_exp["r2"]])
        final_scores_exp["mse_mean"] = np.mean(scores_exp["mse"])
        final_scores_exp["mse_std"] = np.std(scores_exp["mse"])
        final_scores_exp["poisson_deviance_mean"] = np.mean(
            scores_exp["poisson_deviance"])
        final_scores_exp["poisson_deviance_std"] = np.std(
            scores_exp["poisson_deviance"])
        final_scores_exp["gamma_deviance_mean"] = np.mean(
            scores_exp["gamma_deviance"])
        final_scores_exp["gamma_deviance_std"] = np.std(
            scores_exp["gamma_deviance"])
        self.cross_validation_scores_exp = final_scores_exp

        return self.cross_validation_scores, self.cross_validation_scores_exp
def tweedie_eval(y_pred, y_true):
    y_true = y_true.get_label()
    y_true = rollup(y_true.reshape(NUM_ITEMS, -1)).flatten()
    y_pred = rollup(y_pred.reshape(NUM_ITEMS, -1)).flatten()
    loss = mean_tweedie_deviance(y_true, y_pred, power=1.5)
    return "tweedie_eval", loss, False       
Esempio n. 15
0
def _sk_deviance(preds: Tensor, targets: Tensor, power: float):
    sk_preds = preds.view(-1).numpy()
    sk_target = targets.view(-1).numpy()
    return mean_tweedie_deviance(sk_target, sk_preds, power=power)
Esempio n. 16
0
def test_regression_metrics_at_limits():
    assert_almost_equal(mean_squared_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(mean_squared_error([0.], [0.], squared=False), 0.00, 2)
    assert_almost_equal(mean_squared_log_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(mean_absolute_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(median_absolute_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(max_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(explained_variance_score([0.], [0.]), 1.00, 2)
    assert_almost_equal(r2_score([0., 1], [0., 1]), 1.00, 2)
    err_msg = ("Mean Squared Logarithmic Error cannot be used when targets "
               "contain negative values.")
    with pytest.raises(ValueError, match=err_msg):
        mean_squared_log_error([-1.], [-1.])
    err_msg = ("Mean Squared Logarithmic Error cannot be used when targets "
               "contain negative values.")
    with pytest.raises(ValueError, match=err_msg):
        mean_squared_log_error([1., 2., 3.], [1., -2., 3.])
    err_msg = ("Mean Squared Logarithmic Error cannot be used when targets "
               "contain negative values.")
    with pytest.raises(ValueError, match=err_msg):
        mean_squared_log_error([1., -2., 3.], [1., 2., 3.])

    # Tweedie deviance error
    power = -1.2
    assert_allclose(mean_tweedie_deviance([0], [1.], power=power),
                    2 / (2 - power), rtol=1e-3)
    with pytest.raises(ValueError,
                       match="can only be used on strictly positive y_pred."):
        mean_tweedie_deviance([0.], [0.], power=power)
    assert_almost_equal(mean_tweedie_deviance([0.], [0.], power=0), 0.00, 2)

    msg = "only be used on non-negative y_true and strictly positive y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.], [0.], power=1.0)

    power = 1.5
    assert_allclose(mean_tweedie_deviance([0.], [1.], power=power),
                    2 / (2 - power))
    msg = "only be used on non-negative y_true and strictly positive y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.], [0.], power=power)
    power = 2.
    assert_allclose(mean_tweedie_deviance([1.], [1.], power=power), 0.00,
                    atol=1e-8)
    msg = "can only be used on strictly positive y_true and y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.], [0.], power=power)
    power = 3.
    assert_allclose(mean_tweedie_deviance([1.], [1.], power=power),
                    0.00, atol=1e-8)

    msg = "can only be used on strictly positive y_true and y_pred."
    with pytest.raises(ValueError, match=msg):
        mean_tweedie_deviance([0.], [0.], power=power)

    with pytest.raises(ValueError,
                       match="is only defined for power<=0 and power>=1"):
        mean_tweedie_deviance([0.], [0.], power=0.5)
def _mean_tweedie_deviance(y_true, y_pred):
    from sklearn.metrics import mean_tweedie_deviance
    return mean_tweedie_deviance(y_true, y_pred)