def test_seed_reset(self): spa = SPA(self.benchmark, self.models, reps=10) spa.seed(23456) initial_state = spa.bootstrap.random_state assert_equal(spa.bootstrap._seed, 23456) spa.compute() spa.reset() assert_equal(spa._pvalues, None) assert_equal(spa.bootstrap.random_state, initial_state)
def test_seed_reset(self): spa = SPA(self.benchmark, self.models, reps=10) spa.seed(23456) initial_state = spa.bootstrap.random_state assert_equal(spa.bootstrap._seed, 23456) spa.compute() spa.reset() assert_equal(spa._pvalues, None) assert_equal(spa.bootstrap.random_state, initial_state)
def test_equivalence(self): spa = SPA(self.benchmark, self.models, block_size=10, reps=100) spa.seed(23456) spa.compute() numpy_pvalues = spa.pvalues spa = SPA(self.benchmark_df, self.models_df, block_size=10, reps=100) spa.seed(23456) spa.compute() pandas_pvalues = spa.pvalues assert_series_equal(numpy_pvalues, pandas_pvalues)
def test_equivalence(self): spa = SPA(self.benchmark, self.models, block_size=10, reps=100) spa.seed(23456) spa.compute() numpy_pvalues = spa.pvalues spa = SPA(self.benchmark_df, self.models_df, block_size=10, reps=100) spa.seed(23456) spa.compute() pandas_pvalues = spa.pvalues assert_series_equal(numpy_pvalues, pandas_pvalues)
def test_pvalues_and_critvals(spa_data): spa = SPA(spa_data.benchmark, spa_data.models, reps=100) spa.compute() spa.seed(23456) simulated_vals = spa._simulated_vals max_stats = np.max(simulated_vals, 0) max_loss_diff = np.max(spa._loss_diff.mean(0), 0) pvalues = np.mean(max_loss_diff <= max_stats, 0) pvalues = pd.Series(pvalues, index=["lower", "consistent", "upper"]) assert_series_equal(pvalues, spa.pvalues) crit_vals = np.percentile(max_stats, 90.0, axis=0) crit_vals = pd.Series(crit_vals, index=["lower", "consistent", "upper"]) assert_series_equal(spa.critical_values(0.10), crit_vals)
def test_pvalues_and_critvals(self): spa = SPA(self.benchmark, self.models, reps=100) spa.compute() spa.seed(23456) simulated_vals = spa._simulated_vals max_stats = np.max(simulated_vals, 0) max_loss_diff = np.max(spa._loss_diff.mean(0), 0) pvalues = np.mean(max_loss_diff <= max_stats, 0) pvalues = pd.Series(pvalues, index=['lower', 'consistent', 'upper']) assert_series_equal(pvalues, spa.pvalues) crit_vals = np.percentile(max_stats, 90.0, axis=0) crit_vals = pd.Series(crit_vals, index=['lower', 'consistent', 'upper']) assert_series_equal(spa.critical_values(0.10), crit_vals)
def test_superior_models(self): adj_models = self.models - linspace(-0.4, 0.4, self.k) stepm = StepM(self.benchmark, adj_models, reps=120) stepm.compute() superior_models = stepm.superior_models assert len(superior_models) > 0 spa = SPA(self.benchmark, adj_models, reps=120) spa.compute() spa.pvalues spa.critical_values(0.05) spa.better_models(0.05) adj_models = self.models_df - linspace(-3.0, 3.0, self.k) stepm = StepM(self.benchmark_series, adj_models, reps=120) stepm.compute() superior_models = stepm.superior_models assert len(superior_models) > 0
def test_superior_models(self): adj_models = self.models - linspace(-1.0, 1.0, self.k) stepm = StepM(self.benchmark, adj_models, reps=120) stepm.compute() superior_models = stepm.superior_models assert len(superior_models) > 0 spa = SPA(self.benchmark, adj_models, reps=120) spa.compute() spa.pvalues spa.critical_values(0.05) spa.better_models(0.05) adj_models = self.models_df - linspace(-3.0, 3.0, self.k) stepm = StepM(self.benchmark_series, adj_models, reps=120) stepm.compute() superior_models = stepm.superior_models assert len(superior_models) > 0
def test_errors(self): spa = SPA(self.benchmark, self.models, reps=100) with pytest.raises(RuntimeError): spa.pvalues with pytest.raises(RuntimeError): spa.critical_values() with pytest.raises(RuntimeError): spa.better_models() with pytest.raises(ValueError): SPA(self.benchmark, self.models, bootstrap='unknown') spa.compute() with pytest.raises(ValueError): spa.better_models(pvalue_type='unknown') with pytest.raises(ValueError): spa.critical_values(pvalue=1.0)
def test_errors(self): spa = SPA(self.benchmark, self.models, reps=100) with pytest.raises(RuntimeError): spa.pvalues with pytest.raises(RuntimeError): spa.critical_values() with pytest.raises(RuntimeError): spa.better_models() with pytest.raises(ValueError): SPA(self.benchmark, self.models, bootstrap='unknown') spa.compute() with pytest.raises(ValueError): spa.better_models(pvalue_type='unknown') with pytest.raises(ValueError): spa.critical_values(pvalue=1.0)
def test_variances_and_selection(self): adj_models = self.models + linspace(-2, 0.5, self.k) spa = SPA(self.benchmark, adj_models, block_size=10, reps=10) spa.seed(23456) spa.compute() variances = spa._loss_diff_var loss_diffs = spa._loss_diff demeaned = spa._loss_diff - loss_diffs.mean(0) t = loss_diffs.shape[0] kernel_weights = np.zeros(t) p = 1 / 10.0 for i in range(1, t): kernel_weights[i] = ((1.0 - (i / t)) * ((1 - p) ** i)) + \ ((i / t) * ((1 - p) ** (t - i))) direct_vars = (demeaned**2).sum(0) / t for i in range(1, t): direct_vars += 2 * kernel_weights[i] * \ (demeaned[:t - i, :] * demeaned[i:, :]).sum(0) / t assert_allclose(direct_vars, variances) selection_criteria = -1.0 * np.sqrt( (direct_vars / t) * 2 * np.log(np.log(t))) valid = loss_diffs.mean(0) >= selection_criteria assert_equal(valid, spa._valid_columns) # Bootstrap variances spa = SPA(self.benchmark, self.models, block_size=10, reps=100, nested=True) spa.seed(23456) spa.compute() spa.reset() bs = spa.bootstrap.clone(demeaned) variances = spa._loss_diff_var bootstrap_variances = t * bs.var( lambda x: x.mean(0), reps=100, recenter=True) assert_allclose(bootstrap_variances, variances)
def test_variances_and_selection(self): adj_models = self.models + linspace(-2, 0.5, self.k) spa = SPA(self.benchmark, adj_models, block_size=10, reps=10) spa.seed(23456) spa.compute() variances = spa._loss_diff_var loss_diffs = spa._loss_diff demeaned = spa._loss_diff - loss_diffs.mean(0) t = loss_diffs.shape[0] kernel_weights = np.zeros(t) p = 1 / 10.0 for i in range(1, t): kernel_weights[i] = ((1.0 - (i / t)) * ((1 - p) ** i)) + \ ((i / t) * ((1 - p) ** (t - i))) direct_vars = (demeaned ** 2).sum(0) / t for i in range(1, t): direct_vars += 2 * kernel_weights[i] * \ (demeaned[:t - i, :] * demeaned[i:, :]).sum(0) / t assert_allclose(direct_vars, variances) selection_criteria = -1.0 * np.sqrt((direct_vars / t) * 2 * np.log(np.log(t))) valid = loss_diffs.mean(0) >= selection_criteria assert_equal(valid, spa._valid_columns) # Bootstrap variances spa = SPA(self.benchmark, self.models, block_size=10, reps=100, nested=True) spa.seed(23456) spa.compute() spa.reset() bs = spa.bootstrap.clone(demeaned) variances = spa._loss_diff_var bootstrap_variances = t * bs.var(lambda x: x.mean(0), reps=100, recenter=True) assert_allclose(bootstrap_variances, variances)
def test_single_model(spa_data): spa = SPA(spa_data.benchmark, spa_data.models[:, 0]) spa.compute() spa = SPA(spa_data.benchmark_series, spa_data.models_df.iloc[:, 0]) spa.compute()
def test_spa_nested(self): spa = SPA(self.benchmark, self.models, nested=True, reps=100) spa.compute()
def test_single_model(self): spa = SPA(self.benchmark, self.models[:, 0]) spa.compute() spa = SPA(self.benchmark_series, self.models_df.iloc[:, 0]) spa.compute()
def test_single_model(self): spa = SPA(self.benchmark, self.models[:, 0]) spa.compute() spa = SPA(self.benchmark_series, self.models_df.iloc[:, 0]) spa.compute()
def test_spa_nested(self): spa = SPA(self.benchmark, self.models, nested=True, reps=100) spa.compute()
def test_spa_nested(spa_data): spa = SPA(spa_data.benchmark, spa_data.models, nested=True, reps=100) spa.compute()