def test_verbose():
X_train, y_train, X_test, y_test = _data("continuous")
scorecard = _fit_scorecard("continuous", X_train, y_train)
monitoring = ScorecardMonitoring(scorecard=scorecard, verbose=True)
with open("tests/test_scorecard_monitoring_verbose.txt", "w") as f:
with redirect_stdout(f):
monitoring.fit(X_test, y_test, X_train, y_train)
def test_plot_binary():
X_train, y_train, X_test, y_test = _data("binary")
scorecard = _fit_scorecard("binary", X_train, y_train)
monitoring = ScorecardMonitoring(scorecard=scorecard)
monitoring.fit(X_test, y_test, X_train, y_train)
with raises(TypeError):
monitoring.psi_plot(savefig=1)
monitoring.psi_plot(savefig="psi_plot_binary.png")
def test_plot_binary():
df_train, df_test = _data("binary")
scorecard = _fit_scorecard("binary", df_train)
monitoring = ScorecardMonitoring(target="target", scorecard=scorecard)
monitoring.fit(df_test, df_train)
with raises(TypeError):
monitoring.psi_plot(savefig=1)
monitoring.psi_plot(savefig="psi_plot_binary.png")
def test_plot_continuous():
X_train, y_train, X_test, y_test = _data("continuous")
scorecard = _fit_scorecard("continuous", X_train, y_train)
monitoring = ScorecardMonitoring(scorecard=scorecard)
monitoring.fit(X_test, y_test, X_train, y_train)
monitoring.psi_plot(savefig="psi_plot_continuous.png")
def test_plot_continuous():
df_train, df_test = _data("continuous")
scorecard = _fit_scorecard("continuous", df_train)
monitoring = ScorecardMonitoring(target="target", scorecard=scorecard)
monitoring.fit(df_test, df_train)
monitoring.psi_plot(savefig="psi_plot_continuous.png")
def test_input():
X_train, y_train, X_test, y_test = _data("binary")
scorecard = _fit_scorecard("binary", X_train, y_train)
monitoring = ScorecardMonitoring(scorecard=scorecard)
with raises(ValueError):
X_test_2 = X_test.copy()
columns = list(X_test.columns)
columns[3] = columns[3].upper()
X_test_2.columns = columns
monitoring.fit(X_test_2, y_test, X_train, y_train)
with raises(ValueError):
X_test_2 = X_test.copy()
y_test_2 = np.random.randint(0, 3, len(X_test))
monitoring.fit(X_test_2, y_test_2, X_train, y_train)
with raises(ValueError):
y_train_2 = np.random.randint(0, 3, len(X_train))
monitoring.fit(X_test, y_test, X_train, y_train_2)
with raises(ValueError):
y_train_2 = np.random.randn(len(X_train))
monitoring.fit(X_test, y_test, X_train, y_train_2)
def test_input():
df_train, df_test = _data("binary")
scorecard = _fit_scorecard("binary", df_train)
monitoring = ScorecardMonitoring(target="target", scorecard=scorecard)
with raises(ValueError):
df_test_2 = df_test.copy()
columns = list(df_test.columns)
columns[3] = columns[3].upper()
df_test_2.columns = columns
monitoring.fit(df_test_2, df_train)
with raises(ValueError):
df_test_2 = df_test.copy()
df_test_2["target"] = np.random.randint(0, 3, len(df_test))
monitoring.fit(df_test_2, df_train)
with raises(ValueError):
df_train["target"] = np.random.randint(0, 3, len(df_train))
monitoring.fit(df_test, df_train)
with raises(ValueError):
df_train["target"] = np.random.randn(len(df_train))
monitoring.fit(df_test, df_train)
def test_params():
X_train, y_train, X_test, y_test = _data("binary")
scorecard = _fit_scorecard("binary", X_train, y_train)
with raises(TypeError):
monitoring = ScorecardMonitoring(scorecard=None)
monitoring.fit(X_test, y_test, X_train, y_train)
with raises(ValueError):
monitoring = ScorecardMonitoring(scorecard=scorecard,
psi_method="new_method")
monitoring.fit(X_test, y_test, X_train, y_train)
with raises(ValueError):
monitoring = ScorecardMonitoring(scorecard=scorecard, psi_n_bins=-12.2)
monitoring.fit(X_test, y_test, X_train, y_train)
with raises(ValueError):
monitoring = ScorecardMonitoring(scorecard=scorecard,
psi_min_bin_size=0.8)
monitoring.fit(X_test, y_test, X_train, y_train)
with raises(ValueError):
monitoring = ScorecardMonitoring(scorecard=scorecard, show_digits=9)
monitoring.fit(X_test, y_test, X_train, y_train)
with raises(TypeError):
monitoring = ScorecardMonitoring(scorecard=scorecard, verbose=1)
monitoring.fit(X_test, y_test, X_train, y_train)
def test_information():
X_train, y_train, X_test, y_test = _data("continuous")
scorecard = _fit_scorecard("continuous", X_train, y_train)
monitoring = ScorecardMonitoring(scorecard=scorecard)
with raises(NotFittedError):
monitoring.information()
with raises(NotFittedError):
monitoring.psi_splits
monitoring.fit(X_test, y_test, X_train, y_train)
with raises(ValueError):
monitoring.information(print_level=-1)
with open("tests/test_scorecard_monitoring_information.txt", "w") as f:
with redirect_stdout(f):
monitoring.information(print_level=0)
monitoring.information(print_level=1)
monitoring.information(print_level=2)
def test_default_continuous():
X_train, y_train, X_test, y_test = _data("continuous")
scorecard = _fit_scorecard("continuous", X_train, y_train)
monitoring = ScorecardMonitoring(scorecard=scorecard)
monitoring.fit(X_test, y_test, X_train, y_train)
# Check psi_table
psi_table = monitoring.psi_table()
assert psi_table.PSI.sum() == approx(0.39483122757230243)
# Check psi_variable_table
assert monitoring.psi_variable_table(
name="CRIM", style="summary").values == approx(0.02469948)
assert monitoring.psi_variable_table(
name="CRIM", style="detailed")["PSI"].sum() == approx(0.02469948)
psi_variable_table_s = monitoring.psi_variable_table(style="summary")
psi_variable_table_d = monitoring.psi_variable_table(style="detailed")
psis = psi_variable_table_s[psi_variable_table_s.Variable == "CRIM"]["PSI"]
psid = psi_variable_table_d[psi_variable_table_d.Variable == "CRIM"]["PSI"]
assert psis.sum() == approx(0.02469948)
assert psid.sum() == approx(0.02469948)
# Check psi splits
assert monitoring.psi_splits[:4] == approx(
[27.18795586, 29.80329227, 31.72756577, 33.10686493], rel=1e-4)
# Check tests table
tests_table = monitoring.tests_table()
assert tests_table["p-value"].values[:2] == approx([0.595254, 0.006365],
rel=1e-4)
# Check system stability report
with open("tests/test_scorecard_monitoring_default_continuous.txt",
"w") as f:
with redirect_stdout(f):
monitoring.system_stability_report()
def test_default_binary():
X_train, y_train, X_test, y_test = _data("binary")
scorecard = _fit_scorecard("binary", X_train, y_train)
monitoring = ScorecardMonitoring(scorecard=scorecard)
monitoring.fit(X_test, y_test, X_train, y_train)
# Check psi_table
psi_table = monitoring.psi_table()
assert psi_table.PSI.sum() == approx(0.003536079105130241)
# Check psi_variable_table
with raises(ValueError):
monitoring.psi_variable_table(style="new_style")
with raises(ValueError):
monitoring.psi_variable_table(name="new variable")
assert monitoring.psi_variable_table(
name="mean radius", style="summary").values == approx(0.02463385)
# Check tests table
tests_table = monitoring.tests_table()
assert tests_table["p-value"].values[:2] == approx(
[0.00077184, 0.51953576], rel=1e-4)
# Check system stability report
with open("tests/test_scorecard_monitoring_default.txt", "w") as f:
with redirect_stdout(f):
monitoring.system_stability_report()
def test_default_continuous():
df_train, df_test = _data("continuous")
scorecard = _fit_scorecard("continuous", df_train)
monitoring = ScorecardMonitoring(target="target", scorecard=scorecard)
monitoring.fit(df_test, df_train)
# Check psi_table
psi_table = monitoring.psi_table()
assert psi_table.PSI.sum() == approx(0.24101307906596886)
# Check psi_variable_table
assert monitoring.psi_variable_table(
name="CRIM", style="summary").values == approx(0.02469948)
assert monitoring.psi_variable_table(
name="CRIM", style="detailed")["PSI"].sum() == approx(0.02469948)
psi_variable_table_s = monitoring.psi_variable_table(style="summary")
psi_variable_table_d = monitoring.psi_variable_table(style="detailed")
psis = psi_variable_table_s[psi_variable_table_s.Variable == "CRIM"]["PSI"]
psid = psi_variable_table_d[psi_variable_table_d.Variable == "CRIM"]["PSI"]
assert psis.sum() == approx(0.02469948)
assert psid.sum() == approx(0.02469948)
# Check psi splits
assert monitoring.psi_splits[:4] == approx(
[11.936903, 15.63085079, 18.19052601, 19.64347458], rel=1e-4)
# Check tests table
tests_table = monitoring.tests_table()
assert tests_table["p-value"].values[:2] == approx(
[0.80810407, 0.69939868], rel=1e-4)
# Check system stability report
with open("tests/test_scorecard_monitoring_default_continuous.txt",
"w") as f:
with redirect_stdout(f):
monitoring.system_stability_report()