def test_str_repr(self):
        spa = SPA(self.benchmark, self.models)
        expected = 'SPA(studentization: asymptotic, bootstrap: ' + \
                   str(spa.bootstrap) + ')'
        assert_equal(str(spa), expected)
        expected = expected[:-1] + ', ID: ' + hex(id(spa)) + ')'
        assert_equal(spa.__repr__(), expected)

        expected = ('<strong>SPA</strong>(' +
                    '<strong>studentization</strong>: asymptotic, ' +
                    '<strong>bootstrap</strong>: ' + str(spa.bootstrap) +
                    ', <strong>ID</strong>: ' + hex(id(spa)) + ')')

        assert_equal(spa._repr_html_(), expected)
        spa = SPA(self.benchmark, self.models, studentize=False,
                  bootstrap='cbb')
        expected = 'SPA(studentization: none, bootstrap: ' + \
                   str(spa.bootstrap) + ')'
        assert_equal(str(spa), expected)

        spa = SPA(self.benchmark, self.models, nested=True,
                  bootstrap='moving_block')
        expected = 'SPA(studentization: bootstrap, bootstrap: ' + \
                   str(spa.bootstrap) + ')'
        assert_equal(str(spa), expected)
 def test_equivalence(self):
     spa = SPA(self.benchmark, self.models, block_size=10, reps=100)
     spa.seed(23456)
     spa.compute()
     numpy_pvalues = spa.pvalues
     spa = SPA(self.benchmark_df, self.models_df, block_size=10, reps=100)
     spa.seed(23456)
     spa.compute()
     pandas_pvalues = spa.pvalues
     assert_series_equal(numpy_pvalues, pandas_pvalues)
Beispiel #3
0
 def test_bootstrap_selection(self):
     spa = SPA(self.benchmark, self.models, bootstrap='sb')
     assert isinstance(spa.bootstrap, StationaryBootstrap)
     spa = SPA(self.benchmark, self.models, bootstrap='cbb')
     assert isinstance(spa.bootstrap, CircularBlockBootstrap)
     spa = SPA(self.benchmark, self.models, bootstrap='circular')
     assert isinstance(spa.bootstrap, CircularBlockBootstrap)
     spa = SPA(self.benchmark, self.models, bootstrap='mbb')
     assert isinstance(spa.bootstrap, MovingBlockBootstrap)
     spa = SPA(self.benchmark, self.models, bootstrap='moving block')
     assert isinstance(spa.bootstrap, MovingBlockBootstrap)
def test_bootstrap_selection(spa_data):
    spa = SPA(spa_data.benchmark, spa_data.models, bootstrap="sb")
    assert isinstance(spa.bootstrap, StationaryBootstrap)
    spa = SPA(spa_data.benchmark, spa_data.models, bootstrap="cbb")
    assert isinstance(spa.bootstrap, CircularBlockBootstrap)
    spa = SPA(spa_data.benchmark, spa_data.models, bootstrap="circular")
    assert isinstance(spa.bootstrap, CircularBlockBootstrap)
    spa = SPA(spa_data.benchmark, spa_data.models, bootstrap="mbb")
    assert isinstance(spa.bootstrap, MovingBlockBootstrap)
    spa = SPA(spa_data.benchmark, spa_data.models, bootstrap="moving block")
    assert isinstance(spa.bootstrap, MovingBlockBootstrap)
Beispiel #5
0
    def test_str_repr(self):
        spa = SPA(self.benchmark, self.models)
        expected = 'SPA(studentization: asymptotic, bootstrap: ' + \
                   str(spa.bootstrap) + ')'
        assert_equal(str(spa), expected)
        expected = expected[:-1] + ', ID: ' + hex(id(spa)) + ')'
        assert_equal(spa.__repr__(), expected)

        expected = ('<strong>SPA</strong>(' +
                    '<strong>studentization</strong>: asymptotic, ' +
                    '<strong>bootstrap</strong>: ' + str(spa.bootstrap) +
                    ', <strong>ID</strong>: ' + hex(id(spa)) + ')')

        assert_equal(spa._repr_html_(), expected)
        spa = SPA(self.benchmark,
                  self.models,
                  studentize=False,
                  bootstrap='cbb')
        expected = 'SPA(studentization: none, bootstrap: ' + \
                   str(spa.bootstrap) + ')'
        assert_equal(str(spa), expected)

        spa = SPA(self.benchmark,
                  self.models,
                  nested=True,
                  bootstrap='moving_block')
        expected = 'SPA(studentization: bootstrap, bootstrap: ' + \
                   str(spa.bootstrap) + ')'
        assert_equal(str(spa), expected)
def test_str_repr(spa_data):
    spa = SPA(spa_data.benchmark, spa_data.models)
    expected = "SPA(studentization: asymptotic, bootstrap: " + str(
        spa.bootstrap) + ")"
    assert_equal(str(spa), expected)
    expected = expected[:-1] + ", ID: " + hex(id(spa)) + ")"
    assert_equal(spa.__repr__(), expected)

    expected = ("<strong>SPA</strong>(" +
                "<strong>studentization</strong>: asymptotic, " +
                "<strong>bootstrap</strong>: " + str(spa.bootstrap) +
                ", <strong>ID</strong>: " + hex(id(spa)) + ")")

    assert_equal(spa._repr_html_(), expected)
    spa = SPA(spa_data.benchmark,
              spa_data.models,
              studentize=False,
              bootstrap="cbb")
    expected = "SPA(studentization: none, bootstrap: " + str(
        spa.bootstrap) + ")"
    assert_equal(str(spa), expected)

    spa = SPA(spa_data.benchmark,
              spa_data.models,
              nested=True,
              bootstrap="moving_block")
    expected = "SPA(studentization: bootstrap, bootstrap: " + str(
        spa.bootstrap) + ")"
    assert_equal(str(spa), expected)
Beispiel #7
0
 def test_seed_reset(self):
     spa = SPA(self.benchmark, self.models, reps=10)
     spa.seed(23456)
     initial_state = spa.bootstrap.random_state
     assert_equal(spa.bootstrap._seed, 23456)
     spa.compute()
     spa.reset()
     assert_equal(spa._pvalues, None)
     assert_equal(spa.bootstrap.random_state, initial_state)
 def test_seed_reset(self):
     spa = SPA(self.benchmark, self.models, reps=10)
     spa.seed(23456)
     initial_state = spa.bootstrap.random_state
     assert_equal(spa.bootstrap._seed, 23456)
     spa.compute()
     spa.reset()
     assert_equal(spa._pvalues, None)
     assert_equal(spa.bootstrap.random_state, initial_state)
Beispiel #9
0
 def test_superior_models(self):
     adj_models = self.models - linspace(-0.4, 0.4, self.k)
     stepm = StepM(self.benchmark, adj_models, reps=120)
     stepm.compute()
     superior_models = stepm.superior_models
     assert len(superior_models) > 0
     spa = SPA(self.benchmark, adj_models, reps=120)
     spa.compute()
     spa.pvalues
     spa.critical_values(0.05)
     spa.better_models(0.05)
     adj_models = self.models_df - linspace(-3.0, 3.0, self.k)
     stepm = StepM(self.benchmark_series, adj_models, reps=120)
     stepm.compute()
     superior_models = stepm.superior_models
     assert len(superior_models) > 0
def test_pvalues_and_critvals(spa_data):
    spa = SPA(spa_data.benchmark, spa_data.models, reps=100)
    spa.compute()
    spa.seed(23456)
    simulated_vals = spa._simulated_vals
    max_stats = np.max(simulated_vals, 0)
    max_loss_diff = np.max(spa._loss_diff.mean(0), 0)
    pvalues = np.mean(max_loss_diff <= max_stats, 0)
    pvalues = pd.Series(pvalues, index=["lower", "consistent", "upper"])
    assert_series_equal(pvalues, spa.pvalues)

    crit_vals = np.percentile(max_stats, 90.0, axis=0)
    crit_vals = pd.Series(crit_vals, index=["lower", "consistent", "upper"])
    assert_series_equal(spa.critical_values(0.10), crit_vals)
 def test_superior_models(self):
     adj_models = self.models - linspace(-1.0, 1.0, self.k)
     stepm = StepM(self.benchmark, adj_models, reps=120)
     stepm.compute()
     superior_models = stepm.superior_models
     assert len(superior_models) > 0
     spa = SPA(self.benchmark, adj_models, reps=120)
     spa.compute()
     spa.pvalues
     spa.critical_values(0.05)
     spa.better_models(0.05)
     adj_models = self.models_df - linspace(-3.0, 3.0, self.k)
     stepm = StepM(self.benchmark_series, adj_models, reps=120)
     stepm.compute()
     superior_models = stepm.superior_models
     assert len(superior_models) > 0
    def test_pvalues_and_critvals(self):
        spa = SPA(self.benchmark, self.models, reps=100)
        spa.compute()
        spa.seed(23456)
        simulated_vals = spa._simulated_vals
        max_stats = np.max(simulated_vals, 0)
        max_loss_diff = np.max(spa._loss_diff.mean(0), 0)
        pvalues = np.mean(max_loss_diff <= max_stats, 0)
        pvalues = pd.Series(pvalues, index=['lower', 'consistent', 'upper'])
        assert_series_equal(pvalues, spa.pvalues)

        crit_vals = np.percentile(max_stats, 90.0, axis=0)
        crit_vals = pd.Series(crit_vals,
                              index=['lower', 'consistent', 'upper'])
        assert_series_equal(spa.critical_values(0.10), crit_vals)
    def test_variances_and_selection(self):
        adj_models = self.models + linspace(-2, 0.5, self.k)
        spa = SPA(self.benchmark, adj_models, block_size=10, reps=10)
        spa.seed(23456)
        spa.compute()
        variances = spa._loss_diff_var
        loss_diffs = spa._loss_diff
        demeaned = spa._loss_diff - loss_diffs.mean(0)
        t = loss_diffs.shape[0]
        kernel_weights = np.zeros(t)
        p = 1 / 10.0
        for i in range(1, t):
            kernel_weights[i] = ((1.0 - (i / t)) * ((1 - p) ** i)) + \
                                ((i / t) * ((1 - p) ** (t - i)))
        direct_vars = (demeaned ** 2).sum(0) / t
        for i in range(1, t):
            direct_vars += 2 * kernel_weights[i] * \
                           (demeaned[:t - i, :] * demeaned[i:, :]).sum(0) / t
        assert_allclose(direct_vars, variances)

        selection_criteria = -1.0 * np.sqrt((direct_vars / t) * 2 *
                                            np.log(np.log(t)))
        valid = loss_diffs.mean(0) >= selection_criteria
        assert_equal(valid, spa._valid_columns)

        # Bootstrap variances
        spa = SPA(self.benchmark, self.models, block_size=10, reps=100,
                  nested=True)
        spa.seed(23456)
        spa.compute()
        spa.reset()
        bs = spa.bootstrap.clone(demeaned)
        variances = spa._loss_diff_var
        bootstrap_variances = t * bs.var(lambda x: x.mean(0), reps=100,
                                         recenter=True)
        assert_allclose(bootstrap_variances, variances)
def test_spa_nested(spa_data):
    spa = SPA(spa_data.benchmark, spa_data.models, nested=True, reps=100)
    spa.compute()
    def test_single_model(self):
        spa = SPA(self.benchmark, self.models[:, 0])
        spa.compute()

        spa = SPA(self.benchmark_series, self.models_df.iloc[:, 0])
        spa.compute()
 def test_spa_nested(self):
     spa = SPA(self.benchmark, self.models, nested=True, reps=100)
     spa.compute()
def test_single_model(spa_data):
    spa = SPA(spa_data.benchmark, spa_data.models[:, 0])
    spa.compute()

    spa = SPA(spa_data.benchmark_series, spa_data.models_df.iloc[:, 0])
    spa.compute()
Beispiel #18
0
 def test_spa_nested(self):
     spa = SPA(self.benchmark, self.models, nested=True, reps=100)
     spa.compute()
Beispiel #19
0
    def test_errors(self):
        spa = SPA(self.benchmark, self.models, reps=100)

        with pytest.raises(RuntimeError):
            spa.pvalues

        with pytest.raises(RuntimeError):
            spa.critical_values()
        with pytest.raises(RuntimeError):
            spa.better_models()

        with pytest.raises(ValueError):
            SPA(self.benchmark, self.models, bootstrap='unknown')
        spa.compute()
        with pytest.raises(ValueError):
            spa.better_models(pvalue_type='unknown')
        with pytest.raises(ValueError):
            spa.critical_values(pvalue=1.0)
Beispiel #20
0
    def test_variances_and_selection(self):
        adj_models = self.models + linspace(-2, 0.5, self.k)
        spa = SPA(self.benchmark, adj_models, block_size=10, reps=10)
        spa.seed(23456)
        spa.compute()
        variances = spa._loss_diff_var
        loss_diffs = spa._loss_diff
        demeaned = spa._loss_diff - loss_diffs.mean(0)
        t = loss_diffs.shape[0]
        kernel_weights = np.zeros(t)
        p = 1 / 10.0
        for i in range(1, t):
            kernel_weights[i] = ((1.0 - (i / t)) * ((1 - p) ** i)) + \
                                ((i / t) * ((1 - p) ** (t - i)))
        direct_vars = (demeaned**2).sum(0) / t
        for i in range(1, t):
            direct_vars += 2 * kernel_weights[i] * \
                           (demeaned[:t - i, :] * demeaned[i:, :]).sum(0) / t
        assert_allclose(direct_vars, variances)

        selection_criteria = -1.0 * np.sqrt(
            (direct_vars / t) * 2 * np.log(np.log(t)))
        valid = loss_diffs.mean(0) >= selection_criteria
        assert_equal(valid, spa._valid_columns)

        # Bootstrap variances
        spa = SPA(self.benchmark,
                  self.models,
                  block_size=10,
                  reps=100,
                  nested=True)
        spa.seed(23456)
        spa.compute()
        spa.reset()
        bs = spa.bootstrap.clone(demeaned)
        variances = spa._loss_diff_var
        bootstrap_variances = t * bs.var(
            lambda x: x.mean(0), reps=100, recenter=True)
        assert_allclose(bootstrap_variances, variances)
Beispiel #21
0
    def test_single_model(self):
        spa = SPA(self.benchmark, self.models[:, 0])
        spa.compute()

        spa = SPA(self.benchmark_series, self.models_df.iloc[:, 0])
        spa.compute()
    def test_errors(self):
        spa = SPA(self.benchmark, self.models, reps=100)

        with pytest.raises(RuntimeError):
            spa.pvalues

        with pytest.raises(RuntimeError):
            spa.critical_values()
        with pytest.raises(RuntimeError):
            spa.better_models()

        with pytest.raises(ValueError):
            SPA(self.benchmark, self.models, bootstrap='unknown')
        spa.compute()
        with pytest.raises(ValueError):
            spa.better_models(pvalue_type='unknown')
        with pytest.raises(ValueError):
            spa.critical_values(pvalue=1.0)
Beispiel #23
0
 def test_equivalence(self):
     spa = SPA(self.benchmark, self.models, block_size=10, reps=100)
     spa.seed(23456)
     spa.compute()
     numpy_pvalues = spa.pvalues
     spa = SPA(self.benchmark_df, self.models_df, block_size=10, reps=100)
     spa.seed(23456)
     spa.compute()
     pandas_pvalues = spa.pvalues
     assert_series_equal(numpy_pvalues, pandas_pvalues)