def test_clf_fit_nm_inm(sparse):
data = SPARSE_DATA if sparse else DATA
cl = CleanLearning(seed=SEED)
nm = data["noise_matrix"]
inm = compute_inv_noise_matrix(
py=data["py"],
noise_matrix=nm,
ps=data["ps"],
)
cl.fit(
X=data["X_train"],
labels=data["labels"],
noise_matrix=nm,
inverse_noise_matrix=inm,
)
score_nm_inm = cl.score(data["X_test"], data["true_labels_test"])
# Learn with noisy labels and estimate the inv noise matrix.
cl2 = CleanLearning(seed=SEED)
cl2.fit(
data["X_train"],
data["labels"],
)
score = cl2.score(data["X_test"], data["true_labels_test"])
assert score < score_nm_inm + 1e-4
def test_fit_with_inm(
sparse,
seed=SEED,
used_by_another_test=False,
):
data = SPARSE_DATA if sparse else DATA
cl = CleanLearning(seed=seed, )
inm = compute_inv_noise_matrix(
py=data["py"],
noise_matrix=data["noise_matrix"],
ps=data["ps"],
)
# Learn with noisy labels with inverse noise matrix given
cl.fit(data["X_train"], data["labels"], inverse_noise_matrix=inm)
score_inm = cl.score(data["X_test"], data["true_labels_test"])
# Learn with noisy labels and estimate the inv noise matrix.
cl2 = CleanLearning(seed=seed, )
cl2.fit(
data["X_train"],
data["labels"],
)
score = cl2.score(data["X_test"], data["true_labels_test"])
if used_by_another_test:
return score, score_inm
else:
assert score < score_inm + 1e-4
def test_dimN(N):
cl = CleanLearning(clf=ReshapingLogisticRegression())
size = [100] + [3 for _ in range(N - 1)]
X = np.random.normal(size=size)
labels = np.random.randint(0, 4, size=100)
# ensure that every class is represented
labels[0:10] = 0
labels[11:20] = 1
labels[21:30] = 2
labels[31:40] = 3
# just make sure we don't crash...
cl.fit(X, labels)
cl.predict(X)
cl.predict_proba(X)
cl.score(X, labels)
def test_fit_pred_probs(sparse):
data = SPARSE_DATA if sparse else DATA
cl = CleanLearning()
pred_probs = estimate_cv_predicted_probabilities(
X=data["X_train"],
labels=data["true_labels_train"],
)
cl.fit(X=data["X_train"],
labels=data["true_labels_train"],
pred_probs=pred_probs)
score_with_pred_probs = cl.score(data["X_test"], data["true_labels_test"])
cl = CleanLearning()
cl.fit(
X=data["X_train"],
labels=data["true_labels_train"],
)
score_no_pred_probs = cl.score(data["X_test"], data["true_labels_test"])
assert abs(score_with_pred_probs - score_no_pred_probs) < 0.01
def test_no_score(sparse):
data = SPARSE_DATA if sparse else DATA
class Struct:
def fit(self):
pass
def predict_proba(self):
pass
def predict(self, X):
return data["true_labels_test"]
cl = CleanLearning(clf=Struct())
score = cl.score(data["X_test"], data["true_labels_test"])
assert abs(score - 1) < 1e-6
def test_score(sparse):
data = SPARSE_DATA if sparse else DATA
phrase = "cleanlab is dope"
class Struct:
def fit(self):
pass
def predict_proba(self):
pass
def predict(self):
pass
def score(self, X, y):
return phrase
cl = CleanLearning(clf=Struct())
score = cl.score(data["X_test"], data["true_labels_test"])
assert score == phrase
def test_aux_inputs():
data = DATA
K = len(np.unique(data["labels"]))
confident_joint = np.ones(shape=(K, K))
np.fill_diagonal(confident_joint, 10)
find_label_issues_kwargs = {
"confident_joint": confident_joint,
"min_examples_per_class": 2,
}
cl = CleanLearning(
clf=LogisticRegression(multi_class="auto",
solver="lbfgs",
random_state=SEED),
find_label_issues_kwargs=find_label_issues_kwargs,
verbose=1,
)
label_issues_df = cl.find_label_issues(data["X_train"],
data["labels"],
clf_kwargs={})
assert isinstance(label_issues_df, pd.DataFrame)
FIND_OUTPUT_COLUMNS = [
"is_label_issue", "label_quality", "given_label", "predicted_label"
]
assert list(label_issues_df.columns) == FIND_OUTPUT_COLUMNS
assert label_issues_df.equals(cl.get_label_issues())
cl.fit(
data["X_train"],
data["labels"],
label_issues=label_issues_df,
clf_kwargs={},
clf_final_kwargs={},
)
label_issues_df = cl.get_label_issues()
assert isinstance(label_issues_df, pd.DataFrame)
assert list(label_issues_df.columns) == (FIND_OUTPUT_COLUMNS +
["sample_weight"])
score = cl.score(data["X_test"], data["true_labels_test"])
# Test a second fit
cl.fit(data["X_train"], data["labels"])
# Test cl.find_label_issues with pred_prob input
pred_probs_test = cl.predict_proba(data["X_test"])
label_issues_df = cl.find_label_issues(X=None,
labels=data["true_labels_test"],
pred_probs=pred_probs_test)
assert isinstance(label_issues_df, pd.DataFrame)
assert list(label_issues_df.columns) == FIND_OUTPUT_COLUMNS
assert label_issues_df.equals(cl.get_label_issues())
cl.save_space()
assert cl.label_issues_df is None
# Verbose off
cl = CleanLearning(clf=LogisticRegression(multi_class="auto",
solver="lbfgs",
random_state=SEED),
verbose=0)
cl.save_space() # dummy call test
cl = CleanLearning(clf=LogisticRegression(multi_class="auto",
solver="lbfgs",
random_state=SEED),
verbose=0)
cl.find_label_issues(labels=data["true_labels_test"],
pred_probs=pred_probs_test,
save_space=True)
cl = CleanLearning(clf=LogisticRegression(multi_class="auto",
solver="lbfgs",
random_state=SEED),
verbose=1)
# Test with label_issues_mask input
label_issues_mask = find_label_issues(
labels=data["true_labels_test"],
pred_probs=pred_probs_test,
)
cl.fit(data["X_test"],
data["true_labels_test"],
label_issues=label_issues_mask)
label_issues_df = cl.get_label_issues()
assert isinstance(label_issues_df, pd.DataFrame)
assert set(label_issues_df.columns).issubset(FIND_OUTPUT_COLUMNS)
# Test with label_issues_indices input
label_issues_indices = find_label_issues(
labels=data["true_labels_test"],
pred_probs=pred_probs_test,
return_indices_ranked_by="confidence_weighted_entropy",
)
cl.fit(data["X_test"],
data["true_labels_test"],
label_issues=label_issues_indices)
label_issues_df2 = cl.get_label_issues().copy()
assert isinstance(label_issues_df2, pd.DataFrame)
assert set(label_issues_df2.columns).issubset(FIND_OUTPUT_COLUMNS)
assert label_issues_df2["is_label_issue"].equals(
label_issues_df["is_label_issue"])
# Test fit() with pred_prob input:
cl.fit(
data["X_test"],
data["true_labels_test"],
pred_probs=pred_probs_test,
label_issues=label_issues_mask,
)
label_issues_df = cl.get_label_issues()
assert isinstance(label_issues_df, pd.DataFrame)
assert set(label_issues_df.columns).issubset(FIND_OUTPUT_COLUMNS)
assert "label_quality" in label_issues_df.columns
def test_cl(data):
cl = CleanLearning(clf=LogisticRegression(
multi_class="auto", solver="lbfgs", random_state=SEED))
cl.fit(data["X_train"], data["labels"])
score = cl.score(data["X_test"], data["true_labels_test"])
print(score)