def setup_sampler(self, score_mean, scaling=1, solve_args={ 'min_its': 20, 'tol': 1.e-10 }): X, _ = self.loss.data n, p = X.shape bootstrap_score = pairs_bootstrap_glm(self.loss, self._overall, beta_full=self._beta_full, inactive=~self._overall)[0] score_cov = bootstrap_cov( lambda: np.random.choice(n, size=(n, ), replace=True), bootstrap_score) #score_cov = np.zeros((p,p)) #X_E = X[:, self._active_groups] #X_minusE = X[:, ~self._active_groups] #score_cov[:self._active_groups.sum(), :self._active_groups.sum()] = np.linalg.inv(np.dot(X_E.T, X_E)) #residual_mat = np.identity(n)-np.dot(X_E, np.linalg.pinv(X_E)) #score_cov[self._active_groups.sum():, self._active_groups.sum():] = np.dot(X_minusE.T, np.dot(residual_mat, X_minusE)) self.score_cov = score_cov self.score_cov_inv = np.linalg.inv(self.score_cov) #self.score_mat = -self.score_transform[0] #self.score_mat_inv = np.linalg.inv(self.score_mat) #self.total_cov = np.dot(self.score_mat, self.score_cov).dot(self.score_mat.T) #self.total_cov_inv = np.linalg.inv(self.total_cov) self.reference = score_mean
def solve_approx(self): self.solve() (_opt_linear_term, _opt_affine_term) = self.opt_transform self._opt_linear_term = np.concatenate((_opt_linear_term[self._overall, :], _opt_linear_term[~self._overall, :]), 0) self._opt_affine_term = np.concatenate((_opt_affine_term[self._overall], _opt_affine_term[~self._overall]), 0) self.opt_transform = (self._opt_linear_term, self._opt_affine_term) (_score_linear_term, _) = self.score_transform self._score_linear_term = np.concatenate((_score_linear_term[self._overall, :], _score_linear_term[~self._overall, :]), 0) self.score_transform = (self._score_linear_term, np.zeros(self._score_linear_term.shape[0])) self.feasible_point = np.append(self.observed_score_state, np.abs(self.initial_soln[self._overall])) lagrange = self.penalty._weight_array self.inactive_lagrange = lagrange[~self._overall] X, _ = self.loss.data n, p = X.shape self.p = p nactive = self._overall.sum() self.nactive = nactive self.target_observed = self.observed_score_state[:self.nactive] if self.estimation == 'parametric': score_cov = np.zeros((p,p)) inv_X_active = np.linalg.inv(X[:, self._overall].T.dot(X[:, self._overall])) projection_X_active = X[:,self._overall].dot(np.linalg.inv(X[:, self._overall].T.dot(X[:, self._overall]))).dot(X[:,self._overall].T) score_cov[:self.nactive, :self.nactive] = inv_X_active score_cov[self.nactive:, self.nactive:] = X[:,~self._overall].T.dot(np.identity(n)- projection_X_active).dot(X[:,~self._overall]) elif self.estimation == 'bootstrap': bootstrap_score = pairs_bootstrap_glm(self.loss, self._overall, beta_full=self._beta_full, inactive=~self._overall)[0] score_cov = bootstrap_cov(lambda: np.random.choice(n, size=(n,), replace=True), bootstrap_score) self.score_cov = score_cov self.target_cov = score_cov[:nactive, :nactive] self.score_cov_inv = np.linalg.inv(self.score_cov) self.B = self._opt_linear_term self.A = self._score_linear_term self.B_active = self.B[:nactive, :nactive] self.B_inactive = self.B[nactive:, :nactive] self.A_active = self._score_linear_term[:nactive, :] self.A_inactive = self._score_linear_term[nactive:, :] self.offset_active = self._opt_affine_term[:nactive]
def solve_approx(self): self.solve() self.setup_sampler() p = self.inactive.sum() self.feasible_point = self.observed_scaling self._overall = np.zeros(p, dtype=bool) #print(self.selection_variable['variables']) self._overall[self.selection_variable['variables']] = 1 self.observed_opt_state = np.hstack( [self.observed_scaling, self.observed_subgradients]) _opt_linear_term = np.concatenate((np.atleast_2d( self.maximizing_subgrad).T, self.losing_padding_map), 1) self._opt_linear_term = np.concatenate( (_opt_linear_term[self._overall, :], _opt_linear_term[~self._overall, :]), 0) self.opt_transform = (self._opt_linear_term, np.zeros(p)) (self._score_linear_term, _) = self.score_transform self.inactive_lagrange = self.observed_scaling * self.penalty.weights[ 0] * np.ones(p - 1) X, _ = self.loss.data n, p = X.shape self.p = p bootstrap_score = pairs_bootstrap_glm(self.loss, self.active, inactive=~self.active)[0] bootstrap_target, target_observed = pairs_bootstrap_glm(self.loss, self._overall, beta_full=None, inactive=None) sampler = lambda: np.random.choice(n, size=(n, ), replace=True) self.target_cov, target_score_cov = bootstrap_cov( sampler, bootstrap_target, cross_terms=(bootstrap_score, )) self.score_target_cov = np.atleast_2d(target_score_cov).T self.target_observed = target_observed nactive = self._overall.sum() self.nactive = nactive self.B_active = self._opt_linear_term[:nactive, :nactive] self.B_inactive = self._opt_linear_term[nactive:, :nactive]
def solve_approx(self): self.solve() self.setup_sampler() #print("boundary", self.observed_opt_state, self.boundary) #self.feasible_point = self.observed_opt_state[self.boundary] self.observed_score_state = self.observed_internal_state self.feasible_point = np.ones(self.boundary.sum()) (_opt_linear_term, _opt_offset) = self.opt_transform print("shapes", _opt_linear_term[self.boundary, :].shape, _opt_linear_term[self.interior, :].shape) self._opt_linear_term = np.concatenate( (_opt_linear_term[self.boundary, :], _opt_linear_term[self.interior, :]), 0) self._opt_affine_term = np.concatenate( (_opt_offset[self.boundary], _opt_offset[self.interior]), 0) self.opt_transform = (self._opt_linear_term, self._opt_affine_term) (_score_linear_term, _) = self.score_transform self._score_linear_term = np.concatenate( (_score_linear_term[self.boundary, :], _score_linear_term[self.interior, :]), 0) self.score_transform = (self._score_linear_term, np.zeros(self._score_linear_term.shape[0])) self._overall = self.boundary self.inactive_lagrange = self.threshold[0] * np.ones( np.sum(~self.boundary)) X, _ = self.loss.data n, p = X.shape self.p = p bootstrap_score = pairs_bootstrap_glm(self.loss, self._overall, beta_full=self._beta_full, inactive=~self._overall)[0] score_cov = bootstrap_cov( lambda: np.random.choice(n, size=(n, ), replace=True), bootstrap_score) nactive = self._overall.sum() self.score_target_cov = score_cov[:, :nactive] self.target_cov = score_cov[:nactive, :nactive] self.target_observed = self.observed_score_state[:nactive] self.nactive = nactive self.B_active = self._opt_linear_term[:nactive, :nactive] self.B_inactive = self._opt_linear_term[nactive:, :nactive]
def solve_approx(self): self.solve() self.nactive = self._overall.sum() X, _ = self.loss.data n, p = X.shape self.p = p self.target_observed = self.observed_score_state[:self.nactive] self.feasible_point = np.concatenate([self.observed_score_state, np.fabs(self.observed_opt_state[:self.nactive]), self.observed_opt_state[self.nactive:]], axis = 0) (_opt_linear_term, _opt_affine_term) = self.opt_transform self._opt_linear_term = np.concatenate( (_opt_linear_term[self._overall, :], _opt_linear_term[~self._overall, :]), 0) self._opt_affine_term = np.concatenate((_opt_affine_term[self._overall], _opt_affine_term[~self._overall]), 0) self.opt_transform = (self._opt_linear_term, self._opt_affine_term) (_score_linear_term, _) = self.score_transform self._score_linear_term = np.concatenate( (_score_linear_term[self._overall, :], _score_linear_term[~self._overall, :]), 0) self.score_transform = (self._score_linear_term, np.zeros(self._score_linear_term.shape[0])) lagrange = self.penalty._weight_array #print("True or false", np.all(lagrange[0]-np.fabs(self.feasible_point[p+self.nactive:]))>0) #print("True or false", np.all(self.feasible_point[p:][:self.nactive]) > 0) self.inactive_lagrange = lagrange[~self._overall] self.bootstrap_score, self.randomization_cov = self.setup_sampler() if self.estimation == 'parametric': score_cov = np.zeros((p,p)) inv_X_active = np.linalg.inv(X[:, self._overall].T.dot(X[:, self._overall])) projection_X_active = X[:,self._overall].dot(np.linalg.inv(X[:, self._overall].T.dot(X[:, self._overall]))).dot(X[:,self._overall].T) score_cov[:self.nactive, :self.nactive] = inv_X_active score_cov[self.nactive:, self.nactive:] = X[:,~self._overall].T.dot(np.identity(n)- projection_X_active).dot(X[:,~self._overall]) elif self.estimation == 'bootstrap': score_cov = bootstrap_cov(lambda: np.random.choice(n, size=(n,), replace=True), self.bootstrap_score) self.score_cov = score_cov self.score_cov_inv = np.linalg.inv(self.score_cov)
def solve_approx(self): self.solve() (_opt_linear_term, _opt_affine_term) = self.opt_transform self._opt_linear_term = np.concatenate( (_opt_linear_term[self._overall, :], _opt_linear_term[~self._overall, :]), 0) self._opt_affine_term = np.concatenate( (_opt_affine_term[self._overall], _opt_affine_term[~self._overall]), 0) self.opt_transform = (self._opt_linear_term, self._opt_affine_term) (_score_linear_term, _) = self.score_transform self._score_linear_term = np.concatenate( (_score_linear_term[self._overall, :], _score_linear_term[~self._overall, :]), 0) self.score_transform = (self._score_linear_term, np.zeros(self._score_linear_term.shape[0])) self.feasible_point = np.abs(self.initial_soln[self._overall]) lagrange = [] for key, value in self.penalty.weights.iteritems(): lagrange.append(value) lagrange = np.asarray(lagrange) self.inactive_lagrange = lagrange[~self._overall] X, _ = self.loss.data n, p = X.shape self.p = p bootstrap_score = pairs_bootstrap_glm(self.loss, self._overall, beta_full=self._beta_full, inactive=~self._overall)[0] score_cov = bootstrap_cov( lambda: np.random.choice(n, size=(n, ), replace=True), bootstrap_score) nactive = self._overall.sum() self.score_target_cov = score_cov[:, :nactive] self.target_cov = score_cov[:nactive, :nactive] self.target_observed = self.observed_score_state[:nactive] self.nactive = nactive self.B_active = self._opt_linear_term[:nactive, :nactive] self.B_inactive = self._opt_linear_term[nactive:, :nactive]
def test_overall_null_two_queries(): s, n, p = 5, 200, 20 randomizer = randomization.laplace((p, ), scale=0.5) X, y, beta, _ = logistic_instance(n=n, p=p, s=s, rho=0.1, snr=14) nonzero = np.where(beta)[0] lam_frac = 1. loss = rr.glm.logistic(X, y) epsilon = 1. / np.sqrt(n) lam = lam_frac * np.mean( np.fabs(np.dot(X.T, np.random.binomial(1, 1. / 2, (n, 10000)))).max(0)) W = np.ones(p) * lam W[0] = 0 # use at least some unpenalized penalty = rr.group_lasso(np.arange(p), weights=dict(zip(np.arange(p), W)), lagrange=1.) # first randomization M_est1 = glm_group_lasso(loss, epsilon, penalty, randomizer) M_est1.solve() bootstrap_score1 = M_est1.setup_sampler(scaling=2.) # second randomization M_est2 = glm_group_lasso(loss, epsilon, penalty, randomizer) M_est2.solve() bootstrap_score2 = M_est2.setup_sampler(scaling=2.) # we take target to be union of two active sets active = M_est1.selection_variable[ 'variables'] + M_est2.selection_variable['variables'] if set(nonzero).issubset(np.nonzero(active)[0]): boot_target, target_observed = pairs_bootstrap_glm(loss, active) # target are all true null coefficients selected sampler = lambda: np.random.choice(n, size=(n, ), replace=True) target_cov, cov1, cov2 = bootstrap_cov(sampler, boot_target, cross_terms=(bootstrap_score1, bootstrap_score2)) active_set = np.nonzero(active)[0] inactive_selected = I = [ i for i in np.arange(active_set.shape[0]) if active_set[i] not in nonzero ] # is it enough only to bootstrap the inactive ones? # seems so... if not I: return None A1, b1 = M_est1.linear_decomposition(cov1[I], target_cov[I][:, I], target_observed[I]) A2, b2 = M_est2.linear_decomposition(cov2[I], target_cov[I][:, I], target_observed[I]) target_inv_cov = np.linalg.inv(target_cov[I][:, I]) initial_state = np.hstack([ target_observed[I], M_est1.observed_opt_state, M_est2.observed_opt_state ]) ntarget = len(I) target_slice = slice(0, ntarget) opt_slice1 = slice(ntarget, p + ntarget) opt_slice2 = slice(p + ntarget, 2 * p + ntarget) def target_gradient(state): # with many samplers, we will add up the `target_slice` component # many target_grads # and only once do the Gaussian addition of full_grad target = state[target_slice] opt_state1 = state[opt_slice1] opt_state2 = state[opt_slice2] target_grad1 = M_est1.randomization_gradient( target, (A1, b1), opt_state1) target_grad2 = M_est2.randomization_gradient( target, (A2, b2), opt_state2) full_grad = np.zeros_like(state) full_grad[opt_slice1] = -target_grad1[1] full_grad[opt_slice2] = -target_grad2[1] full_grad[target_slice] -= target_grad1[0] + target_grad2[0] full_grad[target_slice] -= target_inv_cov.dot(target) return full_grad def target_projection(state): opt_state1 = state[opt_slice1] state[opt_slice1] = M_est1.projection(opt_state1) opt_state2 = state[opt_slice2] state[opt_slice2] = M_est2.projection(opt_state2) return state target_langevin = projected_langevin(initial_state, target_gradient, target_projection, .5 / (2 * p + 1)) Langevin_steps = 10000 burning = 2000 samples = [] for i in range(Langevin_steps): target_langevin.next() if (i >= burning): samples.append(target_langevin.state[target_slice].copy()) test_stat = lambda x: np.linalg.norm(x) observed = test_stat(target_observed[I]) sample_test_stat = np.array([test_stat(x) for x in samples]) family = discrete_family(sample_test_stat, np.ones_like(sample_test_stat)) pval = family.ccdf(0, observed) return pval, False
def test_one_inactive_coordinate_handcoded(): s, n, p = 5, 200, 20 randomizer = randomization.laplace((p, ), scale=1.) X, y, beta, _ = logistic_instance(n=n, p=p, s=s, rho=0.1, snr=14) nonzero = np.where(beta)[0] lam_frac = 1. loss = rr.glm.logistic(X, y) epsilon = 1. lam = lam_frac * np.mean( np.fabs(np.dot(X.T, np.random.binomial(1, 1. / 2, (n, 10000)))).max(0)) W = np.ones(p) * lam W += lam * np.arange(p) / 200 W[0] = 0 penalty = rr.group_lasso(np.arange(p), weights=dict(zip(np.arange(p), W)), lagrange=1.) print(lam) # our randomization M_est1 = glm_group_lasso(loss, epsilon, penalty, randomizer) M_est1.solve() bootstrap_score1 = M_est1.setup_sampler() active = M_est1.selection_variable['variables'] if set(nonzero).issubset(np.nonzero(active)[0]): boot_target, target_observed = pairs_bootstrap_glm(loss, active) # target are all true null coefficients selected sampler = lambda: np.random.choice(n, size=(n, ), replace=True) target_cov, cov1 = bootstrap_cov(sampler, boot_target, cross_terms=(bootstrap_score1, )) # have checked that covariance up to here agrees with other test_glm_langevin example active_set = np.nonzero(active)[0] inactive_selected = I = [ i for i in np.arange(active_set.shape[0]) if active_set[i] not in nonzero ] # is it enough only to bootstrap the inactive ones? # seems so... if not I: return None # take the first inactive one I = I[:1] A1, b1 = M_est1.linear_decomposition(cov1[I], target_cov[I][:, I], target_observed[I]) print(I, 'I', target_observed[I]) target_inv_cov = np.linalg.inv(target_cov[I][:, I]) initial_state = np.hstack( [target_observed[I], M_est1.observed_opt_state]) ntarget = len(I) target_slice = slice(0, ntarget) opt_slice1 = slice(ntarget, p + ntarget) def target_gradient(state): # with many samplers, we will add up the `target_slice` component # many target_grads # and only once do the Gaussian addition of full_grad target = state[target_slice] opt_state1 = state[opt_slice1] target_grad1 = M_est1.randomization_gradient( target, (A1, b1), opt_state1) full_grad = np.zeros_like(state) full_grad[opt_slice1] = -target_grad1[1] full_grad[target_slice] -= target_grad1[0] full_grad[target_slice] -= target_inv_cov.dot(target) return full_grad def target_projection(state): opt_state1 = state[opt_slice1] state[opt_slice1] = M_est1.projection(opt_state1) return state target_langevin = projected_langevin(initial_state, target_gradient, target_projection, 1. / p) Langevin_steps = 10000 burning = 2000 samples = [] for i in range(Langevin_steps + burning): target_langevin.next() if (i > burning): samples.append(target_langevin.state[target_slice].copy()) test_stat = lambda x: x observed = test_stat(target_observed[I]) sample_test_stat = np.array([test_stat(x) for x in samples]) family = discrete_family(sample_test_stat, np.ones_like(sample_test_stat)) pval = family.ccdf(0, observed) pval = 2 * min(pval, 1 - pval) _i = I[0] naive_Z = target_observed[_i] / np.sqrt(target_cov[_i, _i]) naive_pval = ndist.sf(np.fabs(naive_Z)) naive_pval = 2 * min(naive_pval, 1 - naive_pval) print('naive Z', naive_Z, naive_pval) return pval, naive_pval, False
def solve_approx(self): self.solve() (_opt_linear_term, _opt_affine_term) = self.opt_transform self._opt_linear_term = np.concatenate( (_opt_linear_term[self._overall, :], _opt_linear_term[~self._overall, :]), 0) self._opt_affine_term = np.concatenate( (_opt_affine_term[self._overall], _opt_affine_term[~self._overall]), 0) self.opt_transform = (self._opt_linear_term, self._opt_affine_term) (_score_linear_term, _) = self.score_transform self._score_linear_term = np.concatenate( (_score_linear_term[self._overall, :], _score_linear_term[~self._overall, :]), 0) self.score_transform = (self._score_linear_term, np.zeros(self._score_linear_term.shape[0])) self.feasible_point = np.append( self.observed_score_state, np.abs(self.initial_soln[self._overall])) lagrange = [] for key, value in self.penalty.weights.iteritems(): lagrange.append(value) lagrange = np.asarray(lagrange) self.inactive_lagrange = lagrange[~self._overall] X, _ = self.loss.data n, p = X.shape self.p = p nactive = self._overall.sum() self.nactive = nactive self.target_observed = self.observed_score_state[:self.nactive] if self.estimation == 'parametric': score_cov = np.zeros((p, p)) vec = np.exp(X[:, self._overall].dot(self.target_observed)) #vec = np.exp(np.zeros(n)) pi = np.true_divide(vec, np.power(1. + vec, 2)) weights = np.diag(pi) Q_active = X[:, self._overall].T.dot(weights).dot(X[:, self._overall]) Q_active_inv = np.linalg.inv(Q_active) P_inactive = X[:, ~self._overall].T.dot( np.identity(n) - weights.dot(X[:, self._overall].dot( Q_active_inv).dot(X[:, self._overall].T))) score_cov[:self.nactive, :self.nactive] = Q_active_inv score_cov[self.nactive:, self.nactive:] = P_inactive.dot(weights).dot( P_inactive.T) elif self.estimation == 'bootstrap': bootstrap_score = pairs_bootstrap_glm(self.loss, self._overall, beta_full=self._beta_full, inactive=~self._overall)[0] score_cov = bootstrap_cov( lambda: np.random.choice(n, size=(n, ), replace=True), bootstrap_score) self.score_cov = score_cov self.target_cov = score_cov[:nactive, :nactive] self.score_cov_inv = np.linalg.inv(self.score_cov) self.B = self._opt_linear_term self.A = self._score_linear_term self.B_active = self.B[:nactive, :nactive] self.B_inactive = self.B[nactive:, :nactive] self.A_active = self._score_linear_term[:nactive, :] self.A_inactive = self._score_linear_term[nactive:, :] self.offset_active = self._opt_affine_term[:nactive]