def test_clf_fit_nm_inm(sparse):
data = SPARSE_DATA if sparse else DATA
lnl = LearningWithNoisyLabels(seed=SEED)
nm = data['noise_matrix']
inm = compute_inv_noise_matrix(
data["py"],
nm,
data["ps"],
)
lnl.fit(
X=data['X_train'],
s=data['s'],
noise_matrix=nm,
inverse_noise_matrix=inm,
)
score_nm_inm = lnl.score(data['X_test'], data['y_test'])
# Learn with noisy labels and estimate the inv noise matrix.
lnl2 = LearningWithNoisyLabels(seed=SEED)
lnl2.fit(
data['X_train'],
data['s'],
)
score = lnl2.score(data['X_test'], data['y_test'])
assert (score < score_nm_inm + 1e-4)
def test_fit_with_inm(
prune_count_method='inverse_nm_dot_s',
seed=0,
used_by_another_test=False,
):
lnl = LearningWithNoisyLabels(
seed=seed,
prune_count_method=prune_count_method,
)
inm = compute_inv_noise_matrix(
data["py"],
data["noise_matrix"],
data["ps"],
)
# Learn with noisy labels with inverse noise matrix given
lnl.fit(data['X_train'], data['s'], inverse_noise_matrix=inm)
score_inm = lnl.score(data['X_test'], data['y_test'])
# Learn with noisy labels and estimate the inv noise matrix.
lnl2 = LearningWithNoisyLabels(
seed=seed,
prune_count_method=prune_count_method,
)
lnl2.fit(
data['X_train'],
data['s'],
)
score = lnl2.score(data['X_test'], data['y_test'])
if used_by_another_test:
return score, score_inm
else:
assert (score < score_inm + 1e-4)
def test_fit_with_inm(
sparse,
seed=SEED,
used_by_another_test=False,
):
data = SPARSE_DATA if sparse else DATA
lnl = LearningWithNoisyLabels(seed=seed, )
inm = compute_inv_noise_matrix(
data["py"],
data["noise_matrix"],
data["ps"],
)
# Learn with noisy labels with inverse noise matrix given
lnl.fit(data['X_train'], data['s'], inverse_noise_matrix=inm)
score_inm = lnl.score(data['X_test'], data['y_test'])
# Learn with noisy labels and estimate the inv noise matrix.
lnl2 = LearningWithNoisyLabels(seed=seed, )
lnl2.fit(
data['X_train'],
data['s'],
)
score = lnl2.score(data['X_test'], data['y_test'])
if used_by_another_test:
return score, score_inm
else:
assert (score < score_inm + 1e-4)
def converge_estimates(
ps,
py,
noise_matrix,
inverse_noise_matrix,
inv_noise_matrix_iterations = 5,
noise_matrix_iterations = 3,
):
'''Computes py := P(y=k) and both noise_matrix and inverse_noise_matrix,
by numerically converging ps := P(s=k), py, and the noise matrices.
Forces numerical consistency of estimates. Each is estimated
independently, but they are related mathematically with closed form
equivalences. This will iteratively make them mathematically consistent.
py := P(y=k) and the inverse noise matrix P(y=k_y|s=k_s) specify one another,
meaning one can be computed from the other and vice versa. When numerical
discrepancy exists due to poor estimation, they can be made to agree by repeatedly
computing one from the other, for some a certain number of iterations (3-10 works fine.)
Do not set iterations too high or performance will decrease as small deviations
will get perturbated over and over and potentially magnified.
Note that we have to first converge the inverse_noise_matrix and py,
then we can update the noise_matrix, then repeat. This is becauase the
inverse noise matrix depends on py (which is unknown/latent), but the
noise matrix depends on ps (which is known), so there will be no change
in the noise matrix if we recompute it when py and inverse_noise_matrix change.
Parameters
----------
ps : np.array (shape (K, ) or (1, K))
The fraction (prior probability) of each observed, noisy class label, P(y = k).
noise_matrix : np.array of shape (K, K), K = number of classes
A conditional probablity matrix of the form P(s=k_s|y=k_y) containing
the fraction of examples in every class, labeled as every other class.
Assumes columns of noise_matrix sum to 1.
inverse_noise_matrix : np.array of shape (K, K), K = number of classes
A conditional probablity matrix of the form P(y=k_y|s=k_s) representing
the estimated fraction observed examples in each class k_s, that are
mislabeled examples from every other class k_y. If None, the
inverse_noise_matrix will be computed from psx and s.
Assumes columns of inverse_noise_matrix sum to 1.
Output
------
Three np.arrays of the form (py, noise_matrix, inverse_noise_matrix) with py
and inverse_noise_matrix and noise_matrix having numerical agreement.'''
for j in range(noise_matrix_iterations):
for i in range(inv_noise_matrix_iterations):
inverse_noise_matrix = compute_inv_noise_matrix(py, noise_matrix, ps)
py = compute_py(ps, noise_matrix, inverse_noise_matrix)
noise_matrix = compute_noise_matrix_from_inverse(ps, inverse_noise_matrix, py)
return py, noise_matrix, inverse_noise_matrix
def make_data(
sparse=False,
means=[[3, 2], [7, 7], [0, 8]],
covs=[[[5, -1.5], [-1.5, 1]], [[1, 0.5], [0.5, 4]], [[5, 1], [1, 5]]],
sizes=[80, 40, 40],
avg_trace=0.8,
seed=1, # set to None for non-reproducible randomness
):
np.random.seed(seed=seed)
m = len(means) # number of classes
n = sum(sizes)
data = []
labels = []
test_data = []
test_labels = []
for idx in range(m):
data.append(
np.random.multivariate_normal(mean=means[idx],
cov=covs[idx],
size=sizes[idx]))
test_data.append(
np.random.multivariate_normal(mean=means[idx],
cov=covs[idx],
size=sizes[idx]))
labels.append(np.array([idx for i in range(sizes[idx])]))
test_labels.append(np.array([idx for i in range(sizes[idx])]))
X_train = np.vstack(data)
y_train = np.hstack(labels)
X_test = np.vstack(test_data)
y_test = np.hstack(test_labels)
if sparse:
X_train = scipy.sparse.csr_matrix(X_train)
X_test = scipy.sparse.csr_matrix(X_test)
# Compute p(y=k)
py = np.bincount(y_train) / float(len(y_train))
noise_matrix = generate_noise_matrix_from_trace(
m,
trace=avg_trace * m,
py=py,
valid_noise_matrix=True,
seed=seed,
)
# Generate our noisy labels using the noise_marix.
s = generate_noisy_labels(y_train, noise_matrix)
ps = np.bincount(s) / float(len(s))
# Compute inverse noise matrix
inv = compute_inv_noise_matrix(py, noise_matrix, ps)
# Estimate psx
latent = latent_estimation.estimate_py_noise_matrices_and_cv_pred_proba(
X=X_train,
s=s,
cv_n_folds=3,
)
return {
"X_train": X_train,
"y_train": y_train,
"X_test": X_test,
"y_test": y_test,
"s": s,
"ps": ps,
"py": py,
"noise_matrix": noise_matrix,
"inverse_noise_matrix": inv,
"est_py": latent[0],
"est_nm": latent[1],
"est_inv": latent[2],
"cj": latent[3],
"psx": latent[4],
"m": m,
"n": n,
}
def test_latent_inv():
ps, py, inv = test_latent_py_ps_inv()
inv2 = latent_algebra.compute_inv_noise_matrix(py, nm)
assert (np.all(abs(inv - inv2) < 1e-3))
clf = LogisticRegression(solver='lbfgs', multi_class='auto', max_iter=1000)
_ = clf.fit(X_train, s)
pred = clf.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test), 2))
print("\nNow we show improvement using cleanlab to characterize the noise")
print(
"and learn on the data that is (with high confidence) labeled correctly.")
print()
print('WITH confident learning (noise matrix given),', end=" ")
_ = rp.fit(X_train, s, noise_matrix=noise_matrix)
pred = rp.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test), 2))
print('WITH confident learning (noise / inverse noise matrix given),', end=" ")
inv = compute_inv_noise_matrix(py, noise_matrix)
_ = rp.fit(X_train, s, noise_matrix=noise_matrix, inverse_noise_matrix=inv)
pred = rp.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test), 2))
print('WITH confident learning noise not given,', end=" ")
clf = LogisticRegression(solver='lbfgs', multi_class='auto', max_iter=1000)
rp = LearningWithNoisyLabels(clf=clf, seed=seed)
_ = rp.fit(X_train, s)
pred = rp.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test), 2))
# ## Performance of confident learning across varying settings.
# To learn more, inspect ```cleanlab/pruning.py``` and ```cleanlab/classification.py```.
# In[ ]:
print('WITHOUT confident learning,', end=" ")
clf = logreg()
_ = clf.fit(X_train, s)
pred = clf.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test), 2))
print("\nNow we show the improvement using confident learning to characterize the noise")
print("and learn on the data that is (with high confidence) labeled correctly.")
print()
print('WITH confident learning (noise matrix given),', end=" ")
_ = rp.fit(X_train, s, noise_matrix = noise_matrix)
pred = rp.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test),2))
print('WITH confident learning (noise / inverse noise matrix given),', end=" ")
_ = rp.fit(X_train, s, noise_matrix = noise_matrix, inverse_noise_matrix=compute_inv_noise_matrix(py, noise_matrix))
pred = rp.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test),2))
print('WITH confident learning (using latent noise matrix estimation),', end=" ")
rp = LearningWithNoisyLabels(clf = logreg(), seed = seed, prune_count_method='inverse_nm_dot_s')
_ = rp.fit(X_train, s)
pred = rp.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test),2))
print('WITH confident learning (using calibrated confident joint),', end=" ")
rp = LearningWithNoisyLabels(clf = logreg(), seed = seed, prune_count_method='calibrate_confident_joint')
_ = rp.fit(X_train, s)
pred = rp.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test),2))
print(
"\nNow we show the improvement using confident learning to characterize the noise"
)
print(
"and learn on the data that is (with high confidence) labeled correctly.")
print()
print('WITH confident learning (noise matrix given),', end=" ")
_ = rp.fit(X_train, s, noise_matrix=noise_matrix)
pred = rp.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test), 2))
print('WITH confident learning (noise / inverse noise matrix given),', end=" ")
_ = rp.fit(X_train,
s,
noise_matrix=noise_matrix,
inverse_noise_matrix=compute_inv_noise_matrix(py, noise_matrix))
pred = rp.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test), 2))
print('WITH confident learning (using latent noise matrix estimation),',
end=" ")
clf = LogisticRegression(solver='lbfgs', multi_class='auto', max_iter=1000)
rp = LearningWithNoisyLabels(clf=clf, seed=seed)
_ = rp.fit(X_train, s)
pred = rp.predict(X_test)
print("Iris dataset test accuracy:", round(accuracy_score(pred, y_test), 2))
print('WITH confident learning (using calibrated confident joint),', end=" ")
clf = LogisticRegression(solver='lbfgs', multi_class='auto', max_iter=1000)
rp = LearningWithNoisyLabels(clf=clf, seed=seed)
_ = rp.fit(X_train, s)
