def test_eq_slow(self):
x = np.atleast_2d(np.random.normal(size=(100, 2)))
y = np.atleast_2d(np.random.normal(size=(100, 1)))
print(x.shape)
for n_blocks in range(2, 49):
b1 = jk.LstsqJackknifeFast(x, y, n_blocks=n_blocks).est
b2 = jk.LstsqJackknifeSlow(x, y, n_blocks=n_blocks).est
assert_array_almost_equal(b1, b2)
def irwls(cls,
x,
y,
update_func,
n_blocks,
w,
slow=False,
separators=None):
'''
Iteratively re-weighted least squares (IRWLS).
Parameters
----------
x : np.matrix with shape (n, p)
Independent variable.
y : np.matrix with shape (n, 1)
Dependent variable.
update_func: function
Transforms output of np.linalg.lstsq to new weights.
n_blocks : int
Number of jackknife blocks (for estimating SE via block jackknife).
w : np.matrix with shape (n, 1)
Initial regression weights.
slow : bool
Use slow block jackknife? (Mostly for testing)
separators : list or None
Block jackknife block boundaries (optional).
Returns
-------
jknife : jk.LstsqJackknifeFast
Block jackknife regression with the final IRWLS weights.
'''
(n, p) = x.shape
if y.shape != (n, 1):
raise ValueError(
'y has shape {S}. y must have shape ({N}, 1).'.format(
S=y.shape, N=n))
if w.shape != (n, 1):
raise ValueError(
'w has shape {S}. w must have shape ({N}, 1).'.format(
S=w.shape, N=n))
w = np.sqrt(w)
for i in range(2): # update this later
new_w = np.sqrt(update_func(cls.wls(x, y, w)))
if new_w.shape != w.shape:
print('IRWLS update:', new_w.shape, w.shape)
raise ValueError('New weights must have same shape.')
else:
w = new_w
x = cls._weight(x, w)
y = cls._weight(y, w)
if slow:
jknife = jk.LstsqJackknifeSlow(x,
y,
n_blocks,
separators=separators)
else:
jknife = jk.LstsqJackknifeFast(x,
y,
n_blocks,
separators=separators)
return jknife
def __init__(self, y, x, w, N, M, n_blocks, intercept=None, slow=False, step1_ii=None, old_weights=False):
for i in [y, x, w, M, N]:
try:
if len(i.shape) != 2:
raise TypeError('Arguments must be 2D arrays.')
except AttributeError:
raise TypeError('Arguments must be arrays.')
n_snp, self.n_annot = x.shape
if any(i.shape != (n_snp, 1) for i in [y, w, N]):
raise ValueError(
'N, weights and response (z1z2 or chisq) must have shape (n_snp, 1).')
if M.shape != (1, self.n_annot):
raise ValueError('M must have shape (1, n_annot).')
M_tot = float(np.sum(M))
x_tot = np.sum(x, axis=1).reshape((n_snp, 1))
self.constrain_intercept = intercept is not None
self.intercept = intercept
self.n_blocks = n_blocks
tot_agg = self.aggregate(y, x_tot, N, M_tot, intercept)
initial_w = self._update_weights(
x_tot, w, N, M_tot, tot_agg, intercept)
Nbar = np.mean(N) # keep condition number low
x = np.multiply(N, x) / Nbar
if not self.constrain_intercept:
x, x_tot = append_intercept(x), append_intercept(x_tot)
yp = y
else:
yp = y - intercept
self.intercept_se = 'NA'
del y
self.twostep_filtered = None
if step1_ii is not None and self.constrain_intercept:
raise ValueError(
'twostep is not compatible with constrain_intercept.')
elif step1_ii is not None and self.n_annot > 1:
raise ValueError(
'twostep not compatible with partitioned LD Score yet.')
elif step1_ii is not None:
n1 = np.sum(step1_ii)
self.twostep_filtered = n_snp - n1
x1 = x[np.squeeze(step1_ii), :]
yp1, w1, N1, initial_w1 = map(
lambda a: a[step1_ii].reshape((n1, 1)), (yp, w, N, initial_w))
def update_func1(a): return self._update_func(
a, x1, w1, N1, M_tot, Nbar, ii=step1_ii)
step1_jknife = IRWLS(
x1, yp1, update_func1, n_blocks, slow=slow, w=initial_w1)
step1_int, _ = self._intercept(step1_jknife)
yp = yp - step1_int
x = remove_intercept(x)
x_tot = remove_intercept(x_tot)
def update_func2(a): return self._update_func(
a, x_tot, w, N, M_tot, Nbar, step1_int)
s = update_separators(step1_jknife.separators, step1_ii)
step2_jknife = IRWLS(
x, yp, update_func2, n_blocks, slow=slow, w=initial_w, separators=s)
c = np.sum(np.multiply(initial_w, x)) / \
np.sum(np.multiply(initial_w, np.square(x)))
jknife = self._combine_twostep_jknives(
step1_jknife, step2_jknife, M_tot, c, Nbar)
elif old_weights:
initial_w = np.sqrt(initial_w)
x = IRWLS._weight(x, initial_w)
y = IRWLS._weight(yp, initial_w)
jknife = jk.LstsqJackknifeFast(x, y, n_blocks)
else:
def update_func(a): return self._update_func(
a, x_tot, w, N, M_tot, Nbar, intercept)
jknife = IRWLS(
x, yp, update_func, n_blocks, slow=slow, w=initial_w)
self.coef, self.coef_cov, self.coef_se = self._coef(jknife, Nbar)
self.cat, self.cat_cov, self.cat_se =\
self._cat(jknife, M, Nbar, self.coef, self.coef_cov)
self.tot, self.tot_cov, self.tot_se = self._tot(self.cat, self.cat_cov)
self.prop, self.prop_cov, self.prop_se =\
self._prop(jknife, M, Nbar, self.cat, self.tot)
self.enrichment, self.M_prop = self._enrichment(
M, M_tot, self.cat, self.tot)
if not self.constrain_intercept:
self.intercept, self.intercept_se = self._intercept(jknife)
self.jknife = jknife
self.tot_delete_values = self._delete_vals_tot(jknife, Nbar, M)
self.part_delete_values = self._delete_vals_part(jknife, Nbar, M)
if not self.constrain_intercept:
self.intercept_delete_values = jknife.delete_values[
:, self.n_annot]
self.M = M