def test_variances_and_selection(self):
adj_models = self.models + linspace(-2, 0.5, self.k)
spa = SPA(self.benchmark, adj_models, block_size=10, reps=10)
spa.seed(23456)
spa.compute()
variances = spa._loss_diff_var
loss_diffs = spa._loss_diff
demeaned = spa._loss_diff - loss_diffs.mean(0)
t = loss_diffs.shape[0]
kernel_weights = np.zeros(t)
p = 1 / 10.0
for i in range(1, t):
kernel_weights[i] = ((1.0 - (i / t)) * ((1 - p) ** i)) + ((i / t) * ((1 - p) ** (t - i)))
direct_vars = (demeaned ** 2).sum(0) / t
for i in range(1, t):
direct_vars += 2 * kernel_weights[i] * (demeaned[:t - i, :] * demeaned[i:, :]).sum(0) / t
assert_allclose(direct_vars, variances)
selection_criteria = -1.0 * np.sqrt((direct_vars / t) * 2 * np.log(np.log(t)))
valid = loss_diffs.mean(0) >= selection_criteria
assert_equal(valid, spa._valid_columns)
# Bootstrap variances
spa = SPA(self.benchmark, self.models, block_size=10, reps=100, nested=True)
spa.seed(23456)
spa.compute()
spa.reset()
bs = spa.bootstrap.clone(demeaned)
variances = spa._loss_diff_var
bootstrap_variances = t * bs.var(lambda x: x.mean(0), reps=100, recenter=True)
assert_allclose(bootstrap_variances, variances)
def test_seed_reset(self):
spa = SPA(self.benchmark, self.models, reps=10)
spa.seed(23456)
initial_state = spa.bootstrap.random_state
assert_equal(spa.bootstrap._seed, 23456)
spa.compute()
spa.reset()
assert_equal(spa._pvalues, None)
assert_equal(spa.bootstrap.random_state, initial_state)
def test_seed_reset(self):
spa = SPA(self.benchmark, self.models, reps=10)
spa.seed(23456)
initial_state = spa.bootstrap.random_state
assert_equal(spa.bootstrap._seed, 23456)
spa.compute()
spa.reset()
assert_equal(spa._pvalues, None)
assert_equal(spa.bootstrap.random_state, initial_state)
def test_variances_and_selection(self):
adj_models = self.models + linspace(-2, 0.5, self.k)
spa = SPA(self.benchmark, adj_models, block_size=10, reps=10)
spa.seed(23456)
spa.compute()
variances = spa._loss_diff_var
loss_diffs = spa._loss_diff
demeaned = spa._loss_diff - loss_diffs.mean(0)
t = loss_diffs.shape[0]
kernel_weights = np.zeros(t)
p = 1 / 10.0
for i in range(1, t):
kernel_weights[i] = ((1.0 - (i / t)) *
((1 - p)**i)) + ((i / t) * ((1 - p)**(t - i)))
direct_vars = (demeaned**2).sum(0) / t
for i in range(1, t):
direct_vars += 2 * kernel_weights[i] * (demeaned[:t - i, :] *
demeaned[i:, :]).sum(0) / t
assert_allclose(direct_vars, variances)
selection_criteria = -1.0 * np.sqrt(
(direct_vars / t) * 2 * np.log(np.log(t)))
valid = loss_diffs.mean(0) >= selection_criteria
assert_equal(valid, spa._valid_columns)
# Bootstrap variances
spa = SPA(self.benchmark,
self.models,
block_size=10,
reps=100,
nested=True)
spa.seed(23456)
spa.compute()
spa.reset()
bs = spa.bootstrap.clone(demeaned)
variances = spa._loss_diff_var
bootstrap_variances = t * bs.var(
lambda x: x.mean(0), reps=100, recenter=True)
assert_allclose(bootstrap_variances, variances)
