def test_fit_ml_model_with_evaluation_with_user_provided_bounds():
"""Tests fit_ml_model_with_evaluation
with min_admissible_value and max_admissible_value"""
data = generate_test_data_for_fitting()
df = data["df"]
model_formula_str = data["model_formula_str"]
y_test = data["y_test"]
df_test = data["df_test"]
y_col = "y"
trained_model = fit_ml_model_with_evaluation(
df=df,
model_formula_str=model_formula_str,
min_admissible_value=-7,
max_admissible_value=20.00)
y_test_pred = predict_ml(fut_df=df_test,
trained_model=trained_model)[y_col]
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert err[enum.get_metric_name()] < 3.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert err[enum.get_metric_name()] < 3.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
# testing actual values on a smaller set
y_test_pred = predict_ml(fut_df=df_test[:10],
trained_model=trained_model)[y_col]
expected_values = [
8.36, 11.19, 1.85, 15.57, 16.84, 14.44, 20.00, 9.02, 1.81, -7.00
]
assert list(y_test_pred.round(2)) == expected_values
def test_fit_ml_model_with_evaluation_skip_test():
"""Tests fit_ml_model_with_evaluation, on linear model,
skipping test set"""
data = generate_test_data_for_fitting()
df = data["df"]
model_formula_str = data["model_formula_str"]
y_test = data["y_test"]
df_test = data["df_test"]
y_col = "y"
trained_model = fit_ml_model_with_evaluation(
df=df, model_formula_str=model_formula_str, training_fraction=1.0)
assert len(trained_model["y_test"]) == 0
assert trained_model["y_test_pred"] is None
assert trained_model["test_evaluation"] is None
assert trained_model["plt_compare_test"] is None
arr1 = predict_ml(fut_df=df, trained_model=trained_model)[y_col].tolist()
arr2 = trained_model["y_train_pred"]
assert np.array_equal(arr1, arr2)
y_test_pred = predict_ml(fut_df=df_test,
trained_model=trained_model)[y_col]
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert err[enum.get_metric_name()] < 3.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert err[enum.get_metric_name()] < 3.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
def test_fit_ml_model_with_evaluation_with_test_set():
"""Tests fit_ml_model_with_evaluation, with test set"""
data = generate_test_data_for_fitting()
df = data["df"]
model_formula_str = data["model_formula_str"]
y_test = data["y_test"]
df_test = data["df_test"]
y_col = "y"
trained_model = fit_ml_model_with_evaluation(
df=df,
model_formula_str=model_formula_str,
fit_algorithm="sgd",
fit_algorithm_params={"alpha": 0.1})
y_test_pred = predict_ml(fut_df=df_test,
trained_model=trained_model)[y_col]
assert trained_model["ml_model"].alpha == 0.1
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert round(err[enum.get_metric_name()]) == 6.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert round(err[enum.get_metric_name()]) == 7.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
# testing actual values for a smaller set
y_test_pred = predict_ml(fut_df=df_test[:10],
trained_model=trained_model)[y_col]
expected_values = [9.0, 9.0, 7.0, 10.0, 10.0, 10.0, 11.0, 9.0, 8.0, 6.0]
assert list(y_test_pred.round()) == expected_values
def test_fit_ml_model_with_evaluation_constant_column_sgd():
"""Tests fit_ml_model_with_evaluation using sgd with
no penalty when some regressors are constant
With limited data, the models converge to slightly different predictions
than the linear model"""
res = generate_test_data_for_fitting(n=80)
df = res["df"]
y_test = res["y_test"]
df_test = res["df_test"]
y_col = "y"
# add constant columns
new_cols = []
for i in range(300):
col = f"cst{i}"
df[col] = 0
df_test[col] = 2
new_cols.append(col)
df["cst_event"] = "string"
df_test["cst_event"] = "string"
new_cols.append("cst_event")
model_formula_str = "+".join([res["model_formula_str"]] + new_cols)
fit_algorithm = "sgd"
trained_model = fit_ml_model_with_evaluation(
df=df,
model_formula_str=model_formula_str,
fit_algorithm=fit_algorithm,
fit_algorithm_params={
"tol": 1e-5,
"penalty": "none"
})
y_test_pred = predict_ml(fut_df=df_test,
trained_model=trained_model)[y_col]
expected_values = [-8.0, 2.0, 3.0, -2.0, 38.0, 0.0, 0.0]
assert list(pd.Series(
trained_model["ml_model"].coef_)[:7].round()) == expected_values
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert err[enum.get_metric_name()] < 3.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert err[enum.get_metric_name()] < 3.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
# testing actual values for a smaller set
y_test_pred = predict_ml(fut_df=df_test[:10],
trained_model=trained_model)[y_col]
expected_values = [
17.0, 18.0, 16.0, 13.0, 9.0, 2.0, 20.0, 12.0, 12.0, 13.0
]
assert list(y_test_pred.round()) == expected_values
def test_fit_ml_model_with_evaluation_constant_column():
"""Tests ``fit_ml_model_with_evaluation``
when some regressors are constant"""
res = generate_test_data_for_fitting(n=80)
df = res["df"]
y_test = res["y_test"]
df_test = res["df_test"]
y_col = "y"
# add constant columns
new_cols = []
for i in range(300):
col = f"cst{i}"
df[col] = 0
df_test[col] = 2
new_cols.append(col)
df["cst_event"] = "string"
df_test["cst_event"] = "string"
new_cols.append("cst_event")
model_formula_str = "+".join([res["model_formula_str"]] + new_cols)
fit_algorithm = "linear"
trained_model = fit_ml_model_with_evaluation(
df=df,
model_formula_str=model_formula_str,
fit_algorithm=fit_algorithm,
normalize_method="min_max")
y_test_pred = predict_ml(fut_df=df_test,
trained_model=trained_model)[y_col]
# intercept, x1, x2, x3, x4, [constant columns]
expected_values = [-23.0, 1.0, 4.0, 0.0, 44.0, 0.0, 0.0]
assert list(pd.Series(
trained_model["ml_model"].coef_)[:7].round()) == expected_values
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert err[enum.get_metric_name()] < 3.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert err[enum.get_metric_name()] < 3.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
# testing actual values for a smaller set
y_test_pred = predict_ml(fut_df=df_test[:10],
trained_model=trained_model)[y_col]
expected_values = [
18.0, 19.0, 16.0, 13.0, 9.0, 1.0, 23.0, 14.0, 12.0, 14.0
]
assert list(y_test_pred.round()) == expected_values
def test_fit_ml_model_with_evaluation_with_weights():
"""Tests fit_ml_model_with_evaluation, with test set"""
data = generate_test_data_for_fitting()
df = data["df"]
model_formula_str = data["model_formula_str"]
y_test = data["y_test"]
df_test = data["df_test"]
y_col = "y"
df["weights"] = range(1, len(df) + 1)
trained_model = fit_ml_model_with_evaluation(
df=df,
model_formula_str=model_formula_str,
fit_algorithm="ridge",
regression_weight_col="weights")
assert trained_model["regression_weight_col"] == "weights"
y_test_pred = predict_ml(fut_df=df_test,
trained_model=trained_model)[y_col]
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert round(err[enum.get_metric_name()]) == 2.0
# Checks for raising exception if weights have negative values
df["weights"] = -df["weights"]
with pytest.raises(ValueError, match="Weights can not be negative."):
fit_ml_model_with_evaluation(df=df,
model_formula_str=model_formula_str,
fit_algorithm="ridge",
regression_weight_col="weights")
def test_fit_ml_model_with_evaluation_sgd():
"""Tests fit_ml_model_with_evaluation, on sgd model"""
res = generate_test_data_for_fitting()
df = res["df"]
model_formula_str = res["model_formula_str"]
y_test = res["y_test"]
df_test = res["df_test"]
y_col = "y"
trained_model = fit_ml_model_with_evaluation(
df=df,
model_formula_str=model_formula_str,
fit_algorithm="sgd",
fit_algorithm_params={"penalty": "none"})
y_test_pred = predict_ml(fut_df=df_test,
trained_model=trained_model)[y_col]
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert err[enum.get_metric_name()] < 3.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert err[enum.get_metric_name()] < 3.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
trained_model = fit_ml_model_with_evaluation(
df=df,
model_formula_str=model_formula_str,
fit_algorithm="sgd",
fit_algorithm_params={
"penalty": "elasticnet",
"alpha": 0.01,
"l1_ratio": 0.2
})
y_test_pred = predict_ml(fut_df=df_test,
trained_model=trained_model)[y_col]
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert round(err[enum.get_metric_name()]) == 3.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert round(err[enum.get_metric_name()]) == 3.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
def test_fit_ml_model_with_evaluation_random_forest():
"""Tests fit_ml_model_with_evaluation, on random forest model"""
data = generate_test_data_for_fitting()
df = data["df"]
model_formula_str = data["model_formula_str"]
y_test = data["y_test"]
df_test = data["df_test"]
y_col = "y"
trained_model = fit_ml_model_with_evaluation(
df=df, model_formula_str=model_formula_str, fit_algorithm="rf")
y_test_pred = predict_ml(fut_df=df_test,
trained_model=trained_model)[y_col]
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert err[enum.get_metric_name()] < 4.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert err[enum.get_metric_name()] < 4.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
def test_fit_ml_model_with_evaluation_with_uncertainty():
"""Tests fit_ml_model_with_evaluation with uncertainty intervals"""
df = gen_sliced_df(sample_size_dict={
"a": 200,
"b": 340,
"c": 300,
"d": 8,
"e": 800
},
seed_dict={
"a": 301,
"b": 167,
"c": 593,
"d": 893,
"e": 191,
"z": 397
},
err_magnitude_coef=8.0)
df = df[["x", "z_categ", "y_hat"]]
df.rename(columns={"y_hat": "y"}, inplace=True)
model_formula_str = "y~x+z_categ"
y_col = "y"
# test_df
fut_df = df.copy()
# we change the name of the column of true values in fut_df
# to be able to keep track of true values later
fut_df.rename(columns={"y": "y_true"}, inplace=True)
y_test = fut_df["y_true"]
# create a small dataframe for testing values only
small_sample_index = [1, 500, 750, 1000]
trained_model = fit_ml_model_with_evaluation(
df=df,
model_formula_str=model_formula_str,
uncertainty_dict={
"uncertainty_method": "simple_conditional_residuals",
"params": {
"quantiles": [0.025, 0.975],
"quantile_estimation_method": "normal_fit",
"sample_size_thresh": 10,
"small_sample_size_method": "std_quantiles",
"small_sample_size_quantile": 0.8
}
})
y_test_pred = predict_ml(fut_df=fut_df, trained_model=trained_model)[y_col]
y_test_pred_small = y_test_pred[small_sample_index]
# testing predictions
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert err[enum.get_metric_name()] < 10.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert err[enum.get_metric_name()] < 10.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
# testing actual values for a smaller set
assert list(
y_test_pred_small.round(1)) == [99.7, 201.5, 303.5,
7.3], ("predictions are not correct")
# testing uncertainty
# assign the predicted y to the response in fut_df
fut_df["y"] = y_test_pred
new_df_with_uncertainty = predict_ci(fut_df,
trained_model["uncertainty_model"])
assert list(new_df_with_uncertainty.columns) == [
"y_quantile_summary", ERR_STD_COL
], ("column names are not as expected")
fut_df["y_quantile_summary"] = new_df_with_uncertainty[
"y_quantile_summary"]
# calculate coverage of the CI
fut_df["inside_95_ci"] = fut_df.apply(lambda row: (
(row["y_true"] <= row["y_quantile_summary"][1]) and
(row["y_true"] >= row["y_quantile_summary"][0])),
axis=1)
ci_coverage = 100.0 * fut_df["inside_95_ci"].mean()
assert ci_coverage > 94.0 and ci_coverage < 96.0, (
"95 percent CI coverage is not between 94 and 96")
# testing uncertainty_method not being implemented but passed
with pytest.raises(
Exception,
match="uncertainty method: non_existing_method is not implemented"
):
fit_ml_model_with_evaluation(df=df,
model_formula_str=model_formula_str,
uncertainty_dict={
"uncertainty_method":
"non_existing_method",
"params": {
"quantiles": [0.025, 0.975],
"quantile_estimation_method":
"normal_fit",
"sample_size_thresh": 10,
"small_sample_size_method":
"std_quantiles",
"small_sample_size_quantile": 0.8
}
})
def test_fit_ml_model():
"""Tests fit_ml_model"""
data = generate_test_data_for_fitting()
df = data["df"]
model_formula_str = data["model_formula_str"]
y_test = data["y_test"]
df_test = data["df_test"]
y_col = "y"
trained_model = fit_ml_model(df=df,
model_formula_str=model_formula_str,
fit_algorithm="sgd",
fit_algorithm_params={"alpha": 0.1})
pred_df = predict_ml(fut_df=df_test, trained_model=trained_model)
input_cols = ["x1", "x2", "x3", "x4", "x1_categ"]
assert_frame_equal(pred_df[input_cols].reset_index(drop=True),
df_test[input_cols].reset_index(drop=True))
y_test_pred = pred_df[y_col]
assert trained_model["ml_model"].alpha == 0.1
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert round(err[enum.get_metric_name()]) == 6.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert round(err[enum.get_metric_name()]) == 7.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
# Tests if `fitted_df` returned is correct
fitted_df_via_predict = predict_ml(fut_df=df, trained_model=trained_model)
assert trained_model["fitted_df"].equals(fitted_df_via_predict)
# Tests actual values for a smaller set
y_test_pred = predict_ml(fut_df=df_test[:10],
trained_model=trained_model)[y_col]
expected_values = [9.0, 9.0, 7.0, 10.0, 10.0, 10.0, 11.0, 9.0, 8.0, 6.0]
assert list(y_test_pred.round()) == expected_values
ml_model_summary = trained_model["ml_model_summary"].round(2)
assert list(ml_model_summary["variable"].values) == [
"Intercept", "x1", "x2", "x3", "x4"
]
assert list(ml_model_summary["coef"].round().values) == [
-0.0, -0.0, 2.0, 1.0, 10.0
]
# Testing the summary returned from statsmodels.
# The summary in this case is very informative with several tables.
# `table[1]` inlcudes the cofficients and p-values.
# Parameters without p-values are available directly
# through `trained_model["ml_model"].params` (names and coefficients).
# However `summary` does include more information (eg p-values) which is desirable.
# Here we test those values even though it is harder to get them through `summary`.
trained_model = fit_ml_model(df=df,
model_formula_str=model_formula_str,
fit_algorithm="statsmodels_ols",
fit_algorithm_params={"alpha": 0.1})
ml_model_summary = trained_model["ml_model_summary"]
ml_model_summary_table = ml_model_summary.tables[1]
assert ml_model_summary_table[0].data == ([
"", "coef", "std err", "t", "P>|t|", "[0.025", "0.975]"
])
assert ml_model_summary_table[1].data == ([
"Intercept", " -26.5445", " 0.456", " -58.197", " 0.000",
" -27.440", " -25.649"
])
assert ml_model_summary_table[2].data == ([
"x1", " 0.5335", " 0.409", " 1.304", " 0.192", " -0.269",
" 1.336"
])