def test_report_ignores_unrequired_columns_in_data(self, train_data_idx,
train_data_discrete,
test_data_c_discrete):
"""Classification report should ignore any columns that are no needed by predict"""
bn = BayesianNetwork(
from_pandas(train_data_idx,
w_threshold=0.3)).fit_node_states(train_data_discrete)
train_data_discrete["NEW_COL"] = [1] * len(train_data_discrete)
bn.fit_cpds(train_data_discrete)
classification_report(bn, test_data_c_discrete, "c")
def test_contains_expected_columns(self, test_data_c_discrete, bn):
"""Check that the report contains all of the required data"""
report = classification_report(bn, test_data_c_discrete, "c")
assert set(
report.columns) == {"recall", "precision", "support", "f1-score"}
def test_contains_all_class_data(self, test_data_c_discrete, bn,
test_data_c_likelihood):
"""Check that the report contains data on each possible class"""
report = classification_report(bn, test_data_c_discrete, "c")
assert (label in report for label in test_data_c_likelihood.columns)
def test_report_on_node_with_no_parents_based_on_modal_state(
self, bn, train_data_discrete):
"""Classification Report on a node with no parents should reflect that predictions are on modal state"""
report = classification_report(bn, train_data_discrete, "d")
assert report.loc["d_False",
"recall"] == 1 # always predicts most likely class
assert report.loc["d_True", "recall"] == 0
predictions.loc[18, :]
# %% markdown [markdown]
# Compare this prediction to the ground truth:
# %% codecell
print(f"Student 18 is predicted to {predictions.loc[18, 'G1_prediction']}")
print(f"Ground truth for student 18 is {discrData.loc[18, 'G1']}")
# %% markdown [markdown]
# # 4/ Model Quality
# To evaluate the quality of the model that has been learned, CausalNex supports two main approaches: Classification Report and Reciever Operating Characteristics (ROC) / Area Under the ROC Curve (AUC).
# ## Measure 1: Classification Report
# To obtain a classification report using a BN, we need to provide a test set and the node we are trying to classify. The classification report predicts the target node for all rows (observations) in the test set and evaluate how well those predictions are made, via the model.
# %% codecell
from causalnex.evaluation import classification_report
classification_report(bn=bayesNetCPD, data=test, node='G1')
# %% markdown [markdown]
# **Interpret Results of classification report:** this report shows that the model can classify reasonably well whether a student passs the exam. For predictions where the student fails, the precision is adequate but recall is bad. This implies that we can rely on predictions for `G1_Fail` but we are likely to miss some of the predictions we should have made. Perhaps these missing predictions are a result of something missing in our structure
# * ALERT - explore graph structure when the recall is bad
#
#
# ## ROC / AUC
# The ROC and AUC can be obtained with `roc_auc` method within CausalNex metrics module.
# ROC curve is computed by micro-averaging predictions made across all states (classes) of the target node.
# %% codecell
from causalnex.evaluation import roc_auc
roc, auc = roc_auc(bn=bayesNetCPD, data=test, node='G1')
print(f"ROC = \n{roc}\n")
print(f"AUC = {auc}")
bn = bn.fit_cpds(train, method="BayesianEstimator", bayes_prior="K2")
# 타겟 확인
print(bn.cpds["G1"]) # 시험 G1 성적 - Pass/Fail
# 타겟을 제외한 인풋(18번째 row) 확인
print(discretised_data.loc[18, discretised_data.columns != 'G1'])
# 예측
predictions = bn.predict(discretised_data, "G1")
print('The prediction is \'{prediction}\''.format(prediction=predictions.loc[18, 'G1_prediction']))
print('The ground truth is \'{truth}\''.format(truth=discretised_data.loc[18, 'G1']))
# 평가
classification_report(bn, test, "G1")
roc, auc = roc_auc(bn, test, "G1")
print(auc)
# 한계(Marginal) 확률 베이스라인 (위와 같음)
bn = bn.fit_cpds(discretised_data, method="BayesianEstimator", bayes_prior="K2")
# 모든 상태와 노드에 대해서 한계(Marginal) 우도(Likelihood) 계산
ie = InferenceEngine(bn)
marginals = ie.query()
print('Marginal Likelihood of Target: ', marginals["G1"])
# 실제 레이블 개수 분포를 세어서 계산한 우도와 비슷한지 확인
labels, counts = np.unique(discretised_data["G1"], return_counts=True)
predictions
# %% markdown [markdown]
# Compare this prediction to the ground truth:
# %% codecell
data
# %% markdown [markdown]
# # 4/ Model Quality
# To evaluate the quality of the model that has been learned, CausalNex supports two main approaches: Classification Report and Reciever Operating Characteristics (ROC) / Area Under the ROC Curve (AUC).
# ## Measure 1: Classification Report
# To obtain a classification report using a BN, we need to provide a test set and the node we are trying to classify. The classification report predicts the target node for all rows (observations) in the test set and evaluate how well those predictions are made, via the model.
# %% codecell
from causalnex.evaluation import classification_report
classification_report(bn=bayesNetCPD, data=data, node='absenteeism_level')
# %% markdown [markdown]
# **Interpret Results of classification report:** Precisions are very low for the no absentee level, and both precions and recall are very low for other absentee levels, implying we are likely to miss some of the predictions we should have made. Perhaps these missing predictions are a result of something missing in our structure
# * $\color{red}{\text{ALERT:}}$ explore graph structure when the recall is bad
#
#
# ## Measure 2: ROC / AUC
# The ROC and AUC can be obtained with `roc_auc` method within CausalNex metrics module.
# ROC curve is computed by micro-averaging predictions made across all states (classes) of the target node.
# %% codecell
from causalnex.evaluation import roc_auc
roc, auc = roc_auc(bn=bayesNetCPD, data=data, node='absenteeism_level')
print(f"ROC = \n{roc}\n")
print(f"AUC = {auc}")
def test_em_algorithm(self): # pylint: disable=too-many-locals
"""
Test if `BayesianNetwork` works with EM algorithm.
We use a naive bayes + parents + an extra node not related to the latent variable.
"""
# p0 p1 p2
# \ | /
# z
# / | \
# c0 c1 c2
# |
# cc0
np.random.seed(22)
data, sm, _, true_lv_values = naive_bayes_plus_parents(
percentage_not_missing=0.1,
samples=1000,
p_z=0.7,
p_c=0.7,
)
data["cc_0"] = np.where(
np.random.random(len(data)) < 0.5, data["c_0"],
(data["c_0"] + 1) % 3)
data.drop(columns=["z"], inplace=True)
complete_data = data.copy(deep=True)
complete_data["z"] = true_lv_values
# Baseline model: the structure of the figure trained with complete data. We try to reproduce it
complete_bn = BayesianNetwork(
StructureModel(list(sm.edges) + [("c_0", "cc_0")]))
complete_bn.fit_node_states_and_cpds(complete_data)
# BN without latent variable: All `p`s are connected to all `c`s + `c0` ->`cc0`
sm_no_lv = StructureModel([(f"p_{p}", f"c_{c}") for p in range(3)
for c in range(3)] + [("c_0", "cc_0")])
bn = BayesianNetwork(sm_no_lv)
bn.fit_node_states(data)
bn.fit_cpds(data)
# TEST 1: cc_0 does not depend on the latent variable so:
assert np.all(bn.cpds["cc_0"] == complete_bn.cpds["cc_0"])
# BN with latent variable
# When we add the latent variable, we add the edges in the image above
# and remove the connection among `p`s and `c`s
edges_to_add = list(sm.edges)
edges_to_remove = [(f"p_{p}", f"c_{c}") for p in range(3)
for c in range(3)]
bn.add_node("z", edges_to_add, edges_to_remove)
bn.fit_latent_cpds("z", [0, 1, 2], data, stopping_delta=0.001)
# TEST 2: cc_0 CPD should remain untouched by the EM algorithm
assert np.all(bn.cpds["cc_0"] == complete_bn.cpds["cc_0"])
# TEST 3: We should recover the correct CPDs quite accurately
assert bn.cpds.keys() == complete_bn.cpds.keys()
assert self.mean_absolute_error(bn.cpds, complete_bn.cpds) < 0.01
# TEST 4: Inference over recovered CPDs should be also accurate
eng = InferenceEngine(bn)
query = eng.query()
n_rows = complete_data.shape[0]
for node in query:
assert (np.abs(query[node][0] -
sum(complete_data[node] == 0) / n_rows) < 1e-2)
assert (np.abs(query[node][1] -
sum(complete_data[node] == 1) / n_rows) < 1e-2)
# TEST 5: Inference using predict and predict_probability functions
report = classification_report(bn, complete_data, "z")
_, auc = roc_auc(bn, complete_data, "z")
complete_report = classification_report(complete_bn, complete_data,
"z")
_, complete_auc = roc_auc(complete_bn, complete_data, "z")
for category, metrics in report.items():
if isinstance(metrics, dict):
for key, val in metrics.items():
assert np.abs(val - complete_report[category][key]) < 1e-2
else:
assert np.abs(metrics - complete_report[category]) < 1e-2
assert np.abs(auc - complete_auc) < 1e-2