def _fit(Xy, classes, kwargs):
X, y = Xy
model = MNB(**kwargs)
model.partial_fit(X, y, classes=classes)
return model
def test_partial_fit(x_dtype, y_dtype, nlp_20news):
chunk_size = 500
X, y = nlp_20news
X = sparse_scipy_to_cp(X, x_dtype).astype(x_dtype)
y = y.astype(y_dtype)
X = X.tocsr()
model = MultinomialNB()
classes = np.unique(y)
total_fit = 0
for i in range(math.ceil(X.shape[0] / chunk_size)):
upper = i * chunk_size + chunk_size
if upper > X.shape[0]:
upper = -1
if upper > 0:
x = X[i * chunk_size:upper]
y_c = y[i * chunk_size:upper]
else:
x = X[i * chunk_size:]
y_c = y[i * chunk_size:]
model.partial_fit(x, y_c, classes=classes)
total_fit += (upper - (i * chunk_size))
if upper == -1:
break
y_hat = model.predict(X)
y_hat = cp.asnumpy(y_hat)
y = cp.asnumpy(y)
assert accuracy_score(y, y_hat) >= 0.924