def estimate_mean(Y, sd):
""" Estimate the mean of a sample given information about
the standard deviations of each entry.
Parameters
----------
Y : ndarray
Data for which mean is to be estimated. Should have shape[0] ==
number of subjects.
sd : ndarray
Standard deviation (subject specific) of the data for which the
mean is to be estimated. Should have shape[0] == number of
subjects.
Returns
-------
value : dict
This dictionary has keys ['effect', 'scale', 't', 'resid', 'sd']
"""
nsubject = Y.shape[0]
squeeze = False
if Y.ndim == 1:
Y = Y.reshape(Y.shape[0], 1)
squeeze = True
_stretch = lambda x: np.multiply.outer(np.ones(nsubject), x)
W = pos_recipr(sd**2)
if W.shape in [(), (1,)]:
W = np.ones(Y.shape) * W
W.shape = Y.shape
# Compute the mean using the optimal weights
effect = (Y * W).sum(0) / W.sum(0)
resid = (Y - _stretch(effect)) * np.sqrt(W)
scale = np.add.reduce(np.power(resid, 2), 0) / (nsubject - 1)
var_total = scale * pos_recipr(W.sum(0))
value = {}
value['resid'] = resid
value['effect'] = effect
value['sd'] = np.sqrt(var_total)
value['t'] = value['effect'] * pos_recipr(value['sd'])
value['scale'] = np.sqrt(scale)
if squeeze:
for key in value.keys():
value[key] = np.squeeze(value[key])
return value
def _extract_sd(self, results):
delay = self.IRF.delay
self.T1 = np.zeros(self.gamma0.shape)
nrow = self.gamma0.shape[0]
for i in range(nrow):
self.T1[i] = self.gamma1[i] * pos_recipr(np.sqrt(results.cov_beta(matrix=self.deltamatrix[i])))
a1 = 1 + 1. * pos_recipr(self.T0sq)
gdot = np.array(([(self.r * (a1 - 2.) *
recipr0(self.gamma0 * a1**2)),
recipr0(self.gamma0 * a1)] *
recipr0(delay.dforward(self.deltahat))))
Cov = results.cov_beta
E = self.effectmatrix
D = self.deltamatrix
nrow = self.effectmatrix.shape[0]
cov = np.zeros((nrow,)*2 + self.T0sq.shape[1:])
for i in range(nrow):
for j in range(i + 1):
cov[i,j] = (gdot[0,i] * gdot[0,j] * Cov(matrix=E[i],
other=E[j]) +
gdot[0,i] * gdot[1,j] * Cov(matrix=E[i],
other=D[j]) +
gdot[1,i] * gdot[0,j] * Cov(matrix=D[i],
other=E[j]) +
gdot[1,i] * gdot[1,j] * Cov(matrix=D[i],
other=D[j]))
cov[j,i] = cov[i,j]
nout = self.weights.shape[0]
self._sd = np.zeros(self._effect.shape)
for r in range(nout):
var = 0
for i in range(nrow):
var += cov[i,i] * np.power(self.weights[r,i], 2)
for j in range(i):
var += 2 * cov[i,j] * self.weights[r,i] * self.weights[r,j]
self._sd[r] = np.sqrt(var)
def Fcontrast(self, matrix, dispersion=None, invcov=None):
"""
Compute an Fcontrast for a contrast matrix.
Here, matrix M is assumed to be non-singular. More precisely,
M pX pX' M'
is assumed invertible. Here, pX is the generalized inverse of the
design matrix of the model. There can be problems in non-OLS models
where the rank of the covariance of the noise is not full.
See the contrast module to see how to specify contrasts.
In particular, the matrices from these contrasts will always be
non-singular in the sense above.
"""
ctheta = np.dot(matrix, self.theta)
if matrix.ndim == 1:
matrix = matrix.reshape((1, matrix.shape[0]))
if dispersion is None:
dispersion = self.dispersion
q = matrix.shape[0]
if invcov is None:
invcov = inv(self.vcov(matrix=matrix, dispersion=1.0))
F = np.add.reduce(np.dot(invcov, ctheta) * ctheta, 0) * pos_recipr((q * dispersion))
return FContrastResults(F=F, df_den=self.df_resid, df_num=invcov.shape[0])
def resel2fwhm(self, resels):
"""
:Parameters:
resels : ``float``
Convert a resel value to an equivalent isotropic FWHM based on
step sizes in self.coordmap.
:Returns: FWHM
"""
return np.sqrt(4*np.log(2.)) * self.wedge * pos_recipr(np.power(resels, 1./self.D))
def fwhm2resel(self, fwhm):
"""
:Parameters:
fwhm : ``float``
Convert an FWHM value to an equivalent resels per voxel based on
step sizes in self.coordmap.
:Returns: resels
"""
return pos_recipr(np.power(fwhm / np.sqrt(4*np.log(2)) * self.wedge, self.D))
def Tcontrast(self, matrix, t=True, sd=True, dispersion=None):
"""
Compute a Tcontrast for a row vector matrix. To get the t-statistic
for a single column, use the 't' method.
"""
_t = _sd = None
_effect = np.dot(matrix, self.theta)
if sd:
_sd = np.sqrt(self.vcov(matrix=matrix, dispersion=dispersion))
if t:
_t = _effect * pos_recipr(_sd)
return TContrastResults(effect=_effect, t=_t, sd=_sd, df_den=self.df_resid)
def t(self, column=None):
"""
Return the (Wald) t-statistic for a given parameter estimate.
Use Tcontrast for more complicated (Wald) t-statistics.
"""
if column is None:
column = range(self.theta.shape[0])
column = np.asarray(column)
_theta = self.theta[column]
_cov = self.vcov(column=column)
if _cov.ndim == 2:
_cov = np.diag(_cov)
_t = _theta * pos_recipr(np.sqrt(_cov))
return _t
def __call__(self, results):
"""
:Parameters:
`results` : a scipy.stats.models.model.LikelihoodModelResults instance
:Returns: ``numpy.ndarray``
"""
resid = results.resid.reshape((results.resid.shape[0],
np.product(results.resid.shape[1:])))
sum_sq = results.scale.reshape(resid.shape[1:]) * results.df_resid
cov = np.zeros((self.p + 1,) + sum_sq.shape)
cov[0] = sum_sq
for i in range(1, self.p+1):
cov[i] = np.add.reduce(resid[i:] * resid[0:-i], 0)
cov = np.dot(self.invM, cov)
output = cov[1:] * pos_recipr(cov[0])
return np.squeeze(output)
def norm_resid(self):
"""
Residuals, normalized to have unit length.
Notes
-----
Is this supposed to return "stanardized residuals," residuals standardized
to have mean zero and approximately unit variance?
d_i = e_i/sqrt(MS_E)
Where MS_E = SSE/(n - k)
See: Montgomery and Peck 3.2.1 p. 68
Davidson and MacKinnon 15.2 p 662
"""
return self.resid * utils.pos_recipr(np.sqrt(self.dispersion))
def _extract_effect(self, results):
delay = self.IRF.delay
self.gamma0 = np.dot(self.effectmatrix, results.beta)
self.gamma1 = np.dot(self.deltamatrix, results.beta)
nrow = self.gamma0.shape[0]
self.T0sq = np.zeros(self.gamma0.shape)
for i in range(nrow):
self.T0sq[i] = (self.gamma0[i]**2 *
pos_recipr(results.cov_beta(matrix=self.effectmatrix[i])))
self.r = self.gamma1 * recipr0(self.gamma0)
self.rC = self.r * self.T0sq / (1. + self.T0sq)
self.deltahat = delay.inverse(self.rC)
self._effect = np.dot(self.weights, self.deltahat)
def test_pos_recipr():
X = np.array([2,1,-1,0], dtype=np.int8)
eX = np.array([0.5,1,0,0])
Y = pos_recipr(X)
yield assert_array_almost_equal, Y, eX
yield assert_equal, Y.dtype.type, np.float64
X2 = X.reshape((2,2))
Y2 = pos_recipr(X2)
yield assert_array_almost_equal, Y2, eX.reshape((2,2))
# check that lists have arrived
XL = [0, 1, -1]
yield assert_array_almost_equal, pos_recipr(XL), [0, 1, 0]
# scalars
yield assert_equal, pos_recipr(-1), 0
yield assert_equal, pos_recipr(0), 0
yield assert_equal, pos_recipr(2), 0.5
def estimateAR(resid, design, order=1):
"""
Estimate AR parameters using bias correction from fMRIstat.
Parameters
----------
resid: residual image
model: an OLS model used to estimate residuals
Returns
-------
output :
"""
p = order
R = np.identity(design.shape[0]) - np.dot(design, np.linalg.pinv(design))
M = np.zeros((p+1,)*2)
I = np.identity(R.shape[0])
for i in range(p+1):
Di = np.dot(R, toeplitz(I[i]))
for j in range(p+1):
Dj = np.dot(R, toeplitz(I[j]))
M[i,j] = np.diagonal((np.dot(Di, Dj))/(1.+(i>0))).sum()
invM = np.linalg.inv(M)
rresid = np.asarray(resid).reshape(resid.shape[0],
np.product(resid.shape[1:]))
sum_sq = np.sum(rresid**2, axis=0)
cov = np.zeros((p + 1,) + sum_sq.shape)
cov[0] = sum_sq
for i in range(1, p+1):
cov[i] = np.add.reduce(rresid[i:] * rresid[0:-i], 0)
cov = np.dot(invM, cov)
output = cov[1:] * pos_recipr(cov[0])
output = np.squeeze(output)
output.shape = resid.shape[1:]
return output
def estimate_varatio(Y, sd, df=None, niter=10):
"""
In a one-sample random effects problem, estimate
the ratio between the fixed effects variance and
the random effects variance.
Parameters
----------
Y : np.ndarray
Data for which mean is to be estimated.
Should have shape[0] == number of subjects.
sd : array
Standard deviation (subject specific)
of the data for which the mean is to be estimated.
Should have shape[0] == number of subjects.
df : int or None, optional
If supplied, these are used as weights when
deriving the fixed effects variance. Should have
length == number of subjects.
niter : int, optional
Number of EM iterations to perform (default 10)
Returns
-------
value : dict
This dictionary has keys ['fixed', 'ratio', 'random'], where
'fixed' is the fixed effects variance implied by the input
parameter 'sd'; 'random' is the random effects variance and
'ratio' is the estimated ratio of variances: 'random'/'fixed'.
"""
nsubject = Y.shape[0]
squeeze = False
if Y.ndim == 1:
Y = Y.reshape(Y.shape[0], 1)
squeeze = True
_stretch = lambda x: np.multiply.outer(np.ones(nsubject), x)
W = pos_recipr(sd**2)
if W.shape in [(), (1,)]:
W = np.ones(Y.shape) * W
W.shape = Y.shape
S = 1. / W
R = Y - np.multiply.outer(np.ones(Y.shape[0]), Y.mean(0))
sigma2 = np.squeeze((R**2).sum(0)) / (nsubject - 1)
Sreduction = 0.99
minS = S.min(0) * Sreduction
Sm = S - _stretch(minS)
for _ in range(niter):
Sms = Sm + _stretch(sigma2)
W = pos_recipr(Sms)
Winv = pos_recipr(W.sum(0))
mu = Winv * (W*Y).sum(0)
R = W * (Y - _stretch(mu))
ptrS = 1 + (Sm * W).sum(0) - (Sm * W**2).sum(0) * Winv
sigma2 = np.squeeze((sigma2 * ptrS + (sigma2**2) * (R**2).sum(0)) / nsubject)
sigma2 = sigma2 - minS
if df is None:
df = np.ones(nsubject)
df.shape = (1, nsubject)
_Sshape = S.shape
S.shape = (S.shape[0], np.product(S.shape[1:]))
value = {}
value['fixed'] = (np.dot(df, S) / df.sum()).reshape(_Sshape[1:])
value['ratio'] = np.nan_to_num(sigma2 / value['fixed'])
value['random'] = sigma2
if squeeze:
for key in value.keys():
value[key] = np.squeeze(value[key])
return value
def _extract_t(self):
t = self._effect * pos_recipr(self._sd)
t = np.clip(t, self.Tmin, self.Tmax)
return t
