def loessMA(m, windowSize, axis=0, approxMasked=True, verbose=False, callback=None):
"""Returns a new array with values at the given axis smoothed by loess;
if approxMasked==True: the masked values are approximated by loess;
assumes equidistant spacing of points on the given axis.
"""
assert 0 < windowSize <= m.shape[axis]+0.1, "0 < windowSize[%s] <= 1 OR windowSize in range(1.1,m.shape[axis]+1) expected, got %f" % ("%", windowSize)
m = MA.asarray(m)
if m.dtype.char <> Numeric.Float:
m = m.astype(Numeric.Float)
shp_other = list(m.shape)
shp_other.pop(axis)
# get a transposed and reshaped mask and data from m; if m.mask() == None, construct a new array of zeros
mask = Numeric.reshape(Numeric.transpose(MA.getmaskarray(m), [axis] + range(0,axis) + range(axis+1,len(m.shape))), (m.shape[axis], Numeric.multiply.reduce(shp_other)))
data = MA.reshape(MA.transpose(m, [axis] + range(0,axis) + range(axis+1,len(m.shape))), (m.shape[axis], Numeric.multiply.reduce(shp_other)))
maskInv = -1*(mask-1)
xall = Numeric.arange(data.shape[0])
xallList = xall.tolist()
for ii in Numeric.compress(Numeric.add.reduce(maskInv,0) > 1, range(data.shape[1])): # run loess if the profile contains more than 2 values
try:
data[:,ii] = MA.array(statc.loess(zip(MA.compress(maskInv[:,ii], xall).tolist(), MA.compress(maskInv[:,ii], data[:,ii]).tolist()), xallList, windowSize))[:,1]
except:
if verbose:
print "Warning: loessMA: could not loess axis %i index %i" % (axis, ii)
if callback:
callback()
if not approxMasked:
data = MA.array(data, mask=mask)
return MA.transpose(MA.reshape(data, [m.shape[axis]] + shp_other), [axis] + range(0,axis) + range(axis+1,len(m.shape)))
def loessMA(m,
windowSize,
axis=0,
approxMasked=True,
verbose=False,
callback=None):
"""Returns a new array with values at the given axis smoothed by loess;
if approxMasked==True: the masked values are approximated by loess;
assumes equidistant spacing of points on the given axis.
"""
assert 0 < windowSize <= m.shape[
axis] + 0.1, "0 < windowSize[%s] <= 1 OR windowSize in range(1.1,m.shape[axis]+1) expected, got %f" % (
"%", windowSize)
m = MA.asarray(m)
if m.dtype.char <> Numeric.Float:
m = m.astype(Numeric.Float)
shp_other = list(m.shape)
shp_other.pop(axis)
# get a transposed and reshaped mask and data from m; if m.mask() == None, construct a new array of zeros
mask = Numeric.reshape(
Numeric.transpose(MA.getmaskarray(m), [axis] + range(0, axis) +
range(axis + 1, len(m.shape))),
(m.shape[axis], Numeric.multiply.reduce(shp_other)))
data = MA.reshape(
MA.transpose(m,
[axis] + range(0, axis) + range(axis + 1, len(m.shape))),
(m.shape[axis], Numeric.multiply.reduce(shp_other)))
maskInv = -1 * (mask - 1)
xall = Numeric.arange(data.shape[0])
xallList = xall.tolist()
for ii in Numeric.compress(
Numeric.add.reduce(maskInv, 0) > 1, range(data.shape[1])
): # run loess if the profile contains more than 2 values
try:
data[:, ii] = MA.array(
statc.loess(
zip(
MA.compress(maskInv[:, ii], xall).tolist(),
MA.compress(maskInv[:, ii], data[:, ii]).tolist()),
xallList, windowSize))[:, 1]
except:
if verbose:
print "Warning: loessMA: could not loess axis %i index %i" % (
axis, ii)
if callback:
callback()
if not approxMasked:
data = MA.array(data, mask=mask)
return MA.transpose(MA.reshape(data, [m.shape[axis]] + shp_other), [axis] +
range(0, axis) + range(axis + 1, len(m.shape)))
def madMA(m,axis=0):
"""Returns Median Absolute Deviation of the given masked array along the given axis.
"""
m = MA.asarray(m)
mx = MA.asarray(medianMA(m,axis),Numeric.Float)
xt = MA.transpose(m, [axis]+range(axis)+range(axis+1,MA.rank(m))) # do not use swapaxes: (0,1,2) -swap-> (2,1,0); (0,1,2) -transpose-> (2,0,1)
return medianMA(MA.absolute(xt-mx))
def triangularPut(m1d, upper=1, lower=0):
"""Returns 2D masked array with elements of the given 1D array in the strictly upper (lower) triangle.
Elements of the 1D array should be ordered according to the upper triangular part of the 2D matrix.
The lower triangular part (if requested) equals to the transposed upper triangular part.
If upper == lower == 1 a symetric matrix is returned.
"""
assert upper in [0,1] and lower in [0,1], "[0|1] expected for upper / lower"
m1d = MA.asarray(m1d)
assert MA.rank(m1d) == 1, "1D masked array expected"
m2dShape0 = math.ceil(math.sqrt(2*m1d.shape[0]))
assert m1d.shape[0] == m2dShape0*(m2dShape0-1)/2, "the length of m1d does not correspond to n(n-1)/2"
if upper:
if lower:
mask = Numeric.fromfunction(lambda i,j: i==j, (m2dShape0, m2dShape0))
else:
mask = Numeric.fromfunction(lambda i,j: i>=j, (m2dShape0, m2dShape0))
else:
if lower:
mask = Numeric.fromfunction(lambda i,j: i<=j, (m2dShape0, m2dShape0))
else:
mask = Numeric.ones((m2dShape0, m2dShape0))
m2d = MA.ravel(MA.zeros((m2dShape0, m2dShape0), m1d.dtype.char))
condUpperTriang = Numeric.fromfunction(lambda i,j: i<j, (m2dShape0, m2dShape0))
putIndices = Numeric.compress(Numeric.ravel(condUpperTriang), Numeric.arange(0, m2dShape0**2, typecode=Numeric.Int))
MA.put(m2d, putIndices, m1d)
m2d = MA.reshape(m2d, (m2dShape0, m2dShape0))
m2d = MA.where(condUpperTriang, m2d, MA.transpose(m2d))
return MA.array(m2d, mask=Numeric.logical_or(mask, MA.getmaskarray(m2d)))
def swapaxesMA(ma, axis1, axis2):
if axis1 < axis2:
m,M = axis1, axis2
elif axis1 > axis2:
m,M = axis2, axis1
elif 0 <= axis1 == axis2 < len(ma.shape):
return ma
else:
raise ValueError("Bad axis1 or axis2 argument to swapaxes.")
axList = range(m) + [M] + range(m+1,M) + [m] + range(M+1,len(ma.shape))
return MA.transpose(ma, axList)
def anova1A(self, ma3d, repMeasures, callback):
"""conducts one-way ANOVA on individual examples wrt factor A (variables, ma3d axis 1);
returns Numeric array of p-values in shape (1, numExamples).
"""
ps = -1*Numeric.ones((ma3d.shape[0],), Numeric.Float)
if repMeasures:
fAnova = Anova.AnovaRM12LR
else:
fAnova = Anova.Anova1wayLR_2D
for eIdx in range(ma3d.shape[0]):
an = fAnova(MA.transpose(ma3d[eIdx]))
ps[eIdx] = an.Fprob
callback()
return ps
def ttest_ssmpl(self, ma3d, compToVal, callback):
"""conducts single-sample t-test on individual examples wrt factor A (variables, ma3d axis 1);
returns Numeric array of p-values in shape (1, numExamples).
"""
ps = -1*Numeric.ones((ma3d.shape[0],), Numeric.Float)
for eIdx in range(ma3d.shape[0]):
data = Numeric.asarray(MA.transpose(ma3d[eIdx]).compressed())
if len(data) >= 2:
try:
ps[eIdx] = scipy.stats.ttest_1samp(data, compToVal)[1]
except:
print "Warning: zero variance, check the example %i:" % eIdx, data
ps[eIdx] = 1.0
else:
ps[eIdx] = 1.0
callback()
return ps
def ttest_ssmpl(self, ma3d, popMeanVal, callback):
"""conducts single-sample t-test on individual examples wrt factor A (variables, ma3d axis 1);
returns Numeric array of p-values in shape (1, numExamples).
"""
ps = -1*Numeric.ones((ma3d.shape[0],), Numeric.Float)
for eIdx in range(ma3d.shape[0]):
data = Numeric.asarray(MA.transpose(ma3d[eIdx]).compressed())
if len(data) >= 2:
try:
ps[eIdx] = scipy.stats.ttest_1samp(data, popMeanVal)[1]
except:
print "Warning: zero variance, check the example %i:" % eIdx, data
ps[eIdx] = 1.0
else:
## print "Warning: removing example %i:\n%s\n%s\n" % (eIdx, str(data))
print "Warning: removing example %i:" % eIdx, str(data)
ps[eIdx] = 1.0
callback()
return ps
def maxMA(m,axis=0):
"""slow: remove sorting"""
m = MA.asarray(m, Numeric.Float)
transList = [axis] + range(0,axis) + range(axis+1,MA.rank(m))
m = MA.transpose(m, transList) # do not use swapaxes: (0,1,2) -swap-> (2,1,0); (0,1,2) -transpose-> (2,0,1)
return MA.sort(m, 0, fill_value=-1e20)[-1]
