def test_full_tuple_shape():
"""[Base] FoldIndex: test the tuple shape on generation."""
tups = [(tri, tei) for tri, tei in FoldIndex(5, X=X).generate()]
assert tups == [(((1, 5), ), (0, 1)), (((0, 1), (2, 5)), (1, 2)),
(((0, 2), (3, 5)), (2, 3)), (((0, 3), (4, 5)), (3, 4)),
(((0, 4), ), (4, 5))]
def test_scores_wo_folds_sklearn():
"""[SuperLearner] test scoring without folds on sklearn scorer."""
if not run_sklearn:
# Fail safe
return
with open(os.devnull, 'w') as f, redirect_stdout(f):
ens3.fit(X2, y2)
pred = ens3.predict(X2)
scores = dict()
for _, tei in FoldIndex(FOLDS, X2).generate(as_array=True):
col = 0
for est_name, __ in ECM:
s = mean_squared_error(y2[tei], F2[tei][:, col])
if est_name not in scores:
scores[est_name] = []
scores[est_name].append(s)
col += 1
for k in scores:
scores[k] = np.mean(scores[k])
for k in scores:
assert scores[k] == ens3.scores_['score_mean'][('layer-1', k)]
def test_full_index_is_fitted():
"""[Base] FoldIndex: check fit methods."""
idx = FoldIndex(4)
assert not hasattr(idx, 'n_samples')
idx.fit(X)
assert hasattr(idx, 'n_samples')
idx = FoldIndex(4)
assert not hasattr(idx, 'n_samples')
for _ in idx.generate(X): pass
assert hasattr(idx, 'n_samples')
idx = FoldIndex(4, X)
assert hasattr(idx, 'n_samples')
def test_full_array_shape():
"""[Base] FoldIndex: test the array shape on generation."""
tr = [np.array([2, 3, 4]), np.array([0, 1, 4]), np.array([0, 1, 2, 3])]
te = [np.array([0, 1]), np.array([2, 3]), np.array([4])]
for i, (tri, tei) in enumerate(FoldIndex(3, X).generate(as_array=True)):
np.testing.assert_array_equal(tri, tr[i])
np.testing.assert_array_equal(tei, te[i])
def test_scores_w_folds():
"""[SuperLearner] test scoring with folds."""
scores = {'no__null': [], 'no__offs': [], 'sc__offs': []}
for _, tei in FoldIndex(FOLDS, X1).generate(as_array=True):
col = 0
for case in sorted(PREPROCESSING):
for est_name, _ in ESTIMATORS[case]:
s = rmse(y1[tei], F1[tei][:, col])
scores['%s__%s' % (case, est_name)].append(s)
col += 1
def test_scores_w_folds_in_script():
"""[SuperLearner] test scoring with folds and in-script scorer."""
ens1_b.fit(X1, y1)
scores = {'no__null': [], 'no__offs': [], 'sc__offs': []}
for _, tei in FoldIndex(FOLDS, X1).generate(as_array=True):
col = 0
for case in sorted(PREPROCESSING):
for est_name, __ in ESTIMATORS[case]:
s = in_script_func(y1[tei], F1[tei][:, col])
scores['%s__%s' % (case, est_name)].append(s)
col += 1
def test_scores_wo_folds_in_script():
"""[SuperLearner] test scoring without folds and in-script scorer."""
ens2_b.fit(X2, y2)
scores = dict()
for _, tei in FoldIndex(FOLDS, X2).generate(as_array=True):
col = 0
for est_name, __ in ECM:
s = in_script_func(y2[tei], F2[tei][:, col])
if not est_name in scores:
scores[est_name] = []
scores[est_name].append(s)
col += 1
for k in scores:
scores[k] = np.mean(scores[k])
for k in scores:
assert scores[k] == ens2_b.scores_['score_mean'][('layer-1', k)]
def test_scores_wo_folds():
"""[SuperLearner] test scoring without folds."""
scores = dict()
for _, tei in FoldIndex(FOLDS, X2).generate(as_array=True):
col = 0
for est_name, _ in ECM:
s = rmse(y2[tei], F2[tei][:, col])
if not est_name in scores:
scores[est_name] = []
scores[est_name].append(s)
col += 1
for k in scores:
scores[k] = np.mean(scores[k])
for k in scores:
assert scores[k] == ens2.scores_['score_mean'][('layer-1', k)]
def test_full_raises_on_empty():
"""[Base] FoldIndex: check raises error on singular array."""
with np.testing.assert_raises(ValueError):
FoldIndex(2, np.empty(1))
def test_full_raises_on_float():
"""[Base] FoldIndex: check raises error on float."""
with np.testing.assert_raises(ValueError):
FoldIndex(0.5, X)
def test_full_warns_on_fold_1():
"""[Base] FoldIndex: check warns on folds=1 if not raise_on_exception."""
with np.testing.assert_warns(UserWarning):
FoldIndex(1, X, raise_on_exception=False)
def test_full_raises_on_fold_1():
"""[Base] FoldIndex: check raises error on folds=1."""
with np.testing.assert_raises(ValueError):
FoldIndex(1, X)
def test_full_raises_on_oversampling():
"""[Base] FoldIndex: check raises error."""
with np.testing.assert_raises(ValueError):
FoldIndex(100, X)
