def test_full_tuple_shape(): """[Base] FoldIndex: test the tuple shape on generation.""" tups = [(tri, tei) for tri, tei in FoldIndex(5, X=X).generate()] assert tups == [(((1, 5), ), (0, 1)), (((0, 1), (2, 5)), (1, 2)), (((0, 2), (3, 5)), (2, 3)), (((0, 3), (4, 5)), (3, 4)), (((0, 4), ), (4, 5))]
def test_scores_wo_folds_sklearn(): """[SuperLearner] test scoring without folds on sklearn scorer.""" if not run_sklearn: # Fail safe return with open(os.devnull, 'w') as f, redirect_stdout(f): ens3.fit(X2, y2) pred = ens3.predict(X2) scores = dict() for _, tei in FoldIndex(FOLDS, X2).generate(as_array=True): col = 0 for est_name, __ in ECM: s = mean_squared_error(y2[tei], F2[tei][:, col]) if est_name not in scores: scores[est_name] = [] scores[est_name].append(s) col += 1 for k in scores: scores[k] = np.mean(scores[k]) for k in scores: assert scores[k] == ens3.scores_['score_mean'][('layer-1', k)]
def test_full_index_is_fitted(): """[Base] FoldIndex: check fit methods.""" idx = FoldIndex(4) assert not hasattr(idx, 'n_samples') idx.fit(X) assert hasattr(idx, 'n_samples') idx = FoldIndex(4) assert not hasattr(idx, 'n_samples') for _ in idx.generate(X): pass assert hasattr(idx, 'n_samples') idx = FoldIndex(4, X) assert hasattr(idx, 'n_samples')
def test_full_array_shape(): """[Base] FoldIndex: test the array shape on generation.""" tr = [np.array([2, 3, 4]), np.array([0, 1, 4]), np.array([0, 1, 2, 3])] te = [np.array([0, 1]), np.array([2, 3]), np.array([4])] for i, (tri, tei) in enumerate(FoldIndex(3, X).generate(as_array=True)): np.testing.assert_array_equal(tri, tr[i]) np.testing.assert_array_equal(tei, te[i])
def test_scores_w_folds(): """[SuperLearner] test scoring with folds.""" scores = {'no__null': [], 'no__offs': [], 'sc__offs': []} for _, tei in FoldIndex(FOLDS, X1).generate(as_array=True): col = 0 for case in sorted(PREPROCESSING): for est_name, _ in ESTIMATORS[case]: s = rmse(y1[tei], F1[tei][:, col]) scores['%s__%s' % (case, est_name)].append(s) col += 1
def test_scores_w_folds_in_script(): """[SuperLearner] test scoring with folds and in-script scorer.""" ens1_b.fit(X1, y1) scores = {'no__null': [], 'no__offs': [], 'sc__offs': []} for _, tei in FoldIndex(FOLDS, X1).generate(as_array=True): col = 0 for case in sorted(PREPROCESSING): for est_name, __ in ESTIMATORS[case]: s = in_script_func(y1[tei], F1[tei][:, col]) scores['%s__%s' % (case, est_name)].append(s) col += 1
def test_scores_wo_folds_in_script(): """[SuperLearner] test scoring without folds and in-script scorer.""" ens2_b.fit(X2, y2) scores = dict() for _, tei in FoldIndex(FOLDS, X2).generate(as_array=True): col = 0 for est_name, __ in ECM: s = in_script_func(y2[tei], F2[tei][:, col]) if not est_name in scores: scores[est_name] = [] scores[est_name].append(s) col += 1 for k in scores: scores[k] = np.mean(scores[k]) for k in scores: assert scores[k] == ens2_b.scores_['score_mean'][('layer-1', k)]
def test_scores_wo_folds(): """[SuperLearner] test scoring without folds.""" scores = dict() for _, tei in FoldIndex(FOLDS, X2).generate(as_array=True): col = 0 for est_name, _ in ECM: s = rmse(y2[tei], F2[tei][:, col]) if not est_name in scores: scores[est_name] = [] scores[est_name].append(s) col += 1 for k in scores: scores[k] = np.mean(scores[k]) for k in scores: assert scores[k] == ens2.scores_['score_mean'][('layer-1', k)]
def test_full_raises_on_empty(): """[Base] FoldIndex: check raises error on singular array.""" with np.testing.assert_raises(ValueError): FoldIndex(2, np.empty(1))
def test_full_raises_on_float(): """[Base] FoldIndex: check raises error on float.""" with np.testing.assert_raises(ValueError): FoldIndex(0.5, X)
def test_full_warns_on_fold_1(): """[Base] FoldIndex: check warns on folds=1 if not raise_on_exception.""" with np.testing.assert_warns(UserWarning): FoldIndex(1, X, raise_on_exception=False)
def test_full_raises_on_fold_1(): """[Base] FoldIndex: check raises error on folds=1.""" with np.testing.assert_raises(ValueError): FoldIndex(1, X)
def test_full_raises_on_oversampling(): """[Base] FoldIndex: check raises error.""" with np.testing.assert_raises(ValueError): FoldIndex(100, X)