def getAlphaNIEXS(lensParam,X): einRad = lensParam[0] f = lensParam[1] fp = np.sqrt(1 - f*f) bc = lensParam[2] t = lensParam[3] xshear = (lensParam[4] + 1j*lensParam[5]) #### The rotated coordinates Xr = X * np.exp(-1j*t) x1 = Xr.real x2 = Xr.imag #### The impact parameter b b = x1 + 1j*f*x2 #### The b-squared term bsq = np.absolute(b) bsq = np.multiply(bsq,bsq) #### The differentiation of bsq with respect to x bsqx = x1 + 1j*f*f*x2 #### The deflection angle alpha = einRad*(np.sqrt(f)/fp)*( np.arctanh( fp * np.sqrt(bsq+(bc*bc)) / bsqx) - np.arctanh(fp*bc/(f*Xr)) ) alpha = np.conj(alpha)*np.exp(1j*t) - xshear*np.conj(X) #### This part can return messages like #### "divide by zero encountered in divide" #### "invalid value encountered in divide" #### these messages can be ignored return Xr, alpha
def kappa(self, r):
from numpy import arctanh, arctan, arctan2, log, sin, cos, pi, logspace
x = self.b / self.rs
if x < 1.:
norm = x**2 / (4 * arctanh(
((1 - x) / (1 + x))**0.5) / (1 - x**2)**0.5 + 2 * log(x / 2))
elif x == 1.:
norm = 1. / (2. + 2 * log(0.5))
else:
norm = x**2 / (4 * arctan(
((x - 1) / (x + 1))**0.5) / (x**2 - 1)**0.5 + 2 * log(x / 2))
x = r / self.rs
A = x * 0.
C = x < 1.
X = x[C].copy()
A[C] = (1. - 2 * arctanh(
((1. - X) / (1. + X))**0.5) / (1 - X**2)**0.5) / (X**2 - 1.)
C = x == 1.
A[C] = 1. / 3
C = x > 1.
X = x[C].copy()
A[C] = (1. - 2 * arctan(
((X - 1.) / (1. + X))**0.5) / (X**2 - 1.)**0.5) / (X**2 - 1.)
return norm * A
def shear(self, r=None, zs=None):
rs = self.rs()
k = self.rhoC()*self.deltaC()
sigmaC = self.sigmaC(zs)
x = r/rs
if isinstance(r,np.ndarray):
f = np.piecewise(x, [x>1., x<1., x==1.], [lambda x: (rs*k)*((8*np.arctan(((x-1)/(1+x))**0.5)/(x**2*(x**2-1)**0.5)) \
+ (4*np.log(x/2.)/x**2) \
- (2./(x**2-1)) \
+ (4*np.arctan(((x-1)/(1+x))**0.5)/((x**2-1)**1.5))), \
lambda x: (rs*k)*((8*np.arctanh(((1-x)/(1+x))**0.5)/(x**2*(1-x**2)**0.5)) \
+ (4*np.log(x/2.)/x**2) \
- (2./(x**2-1)) \
+ (4*np.arctanh(((1-x)/(1+x))**0.5)/((x**2-1)*(1-x**2)**0.5))), \
lambda x: (rs*k)*(10./3+4.*np.log(0.5))])
else:
if (x<1.):
f = (rs*k)*((8*np.arctanh(((1-x)/(1+x))**0.5)/(x**2*(1-x**2)**0.5)) \
+ (4*np.log(x/2.)/x**2) \
- (2./(x**2-1)) \
+ (4*np.arctanh(((1-x)/(1+x))**0.5)/((x**2-1)*(1-x**2)**0.5)))
elif (x>1.):
f = (rs*k)*((8*np.arctan(((x-1)/(1+x))**0.5)/(x**2*(x**2-1)**0.5)) \
+ (4*np.log(x/2.)/x**2) \
- (2./(x**2-1)) \
+ (4*np.arctan(((x-1)/(1+x))**0.5)/((x**2-1)**1.5)))
else:
f = (rs*k)*(10./3+4.*np.log(0.5))
#print f/sigmaC
#print isinstance(f,np.ndarray)
return f/sigmaC
def deflections(self, xin, yin):
from numpy import arctanh, arctan, arctan2, log, sin, cos
#x,y = self.align_coords(xin,yin)
x = xin - self.x
y = yin - self.y
b, rs = self.b, self.rs
X = b / rs
if X < 1.:
amp = X**2 / (8 * arctanh(
((1 - X) / (1 + X))**0.5) / (1 - X**2)**0.5 + 4 * log(X / 2.))
elif X == 1:
amp = 0.25 / (1. + log(0.5))
else:
amp = X**2 / (8 * arctan(
((X - 1) / (1 + X))**0.5) / (X**2 - 1)**0.5 + 4 * log(X / 2.))
r2 = (x**2 + y**2) / rs**2
r = r2**0.5
F = r * 0.
F[r < 1.] = arctanh((1 - r2[r < 1.])**0.5) / (1 - r2[r < 1.])**0.5
F[r == 1.] = 1.
F[r > 1.] = arctan((r2[r > 1.] - 1.)**0.5) / (r2[r > 1.] - 1)**0.5
dr = 4 * amp * rs * (log(r / 2) + F) / r
A = arctan2(y, x)
return dr * cos(A), dr * sin(A)
def transfrom(N, E):
N = np.float(N)
xi = N/(A1*k0)
eta = (E-E0)/(A1*k0)
print xi, eta
xip1 = h1*np.sin(2*xi)*np.cosh(2*eta)
xip2 = h2*np.sin(4*xi)*np.cosh(4*eta)
xip3 = h3*np.sin(6*xi)*np.cosh(6*eta)
xip4 = h4*np.sin(8*xi)*np.cosh(8*eta)
print xip1, xip2, xip3, xip4
etap1 = h1*np.cos(2*xi)*np.sinh(2*eta)
etap2 = h2*np.cos(4*xi)*np.sinh(4*eta)
etap3 = h3*np.cos(6*xi)*np.sinh(6*eta)
etap4 = h4*np.cos(8*xi)*np.sinh(8*eta)
print etap1, etap2, etap4, etap4
xip = xi-xip1-xip2-xip3-xip4
etap = eta-etap1-etap2-etap3-etap4
print xip, etap
beta = np.arcsin(sech(etap)*np.sin(xip))
l = np.arcsin(np.tanh(etap)/np.cos(beta))
print beta, l
Q = np.arcsinh(np.tan(beta))
print Q
#e = np.e # ??
e = np.sqrt(0.006694380023)
Qp = Q + e*np.arctanh(e*np.tanh(Q))
for i in xrange(3):
Qp = Q + e*np.arctanh(e*np.tanh(Qp))
print Qp
rhoo = np.arctan(np.sinh(Qp))
lamb = lamb0 + l
return rhoo*180.0/np.pi, lamb*180.0/np.pi
def generateHeatMaps(GroupDF,goodsubj,GroupTrain=[]):
numberOfICs=10
columnNames=[]
for rsnNumber in range(numberOfICs):
columnNames.append('RSN%d' % rsnNumber)
heatmapDF=GroupDF[GroupDF.Subject_ID.isin(goodsubj)].groupby(['Subject_ID','FB','TR']).mean()
hmDiff=np.zeros((10,10,len(unique(GroupDF[GroupDF.Subject_ID.isin(goodsubj)]['Subject_ID']))))
hmFB=hmDiff.copy()
hmNFB=hmDiff.copy()
if len(GroupTrain)>0:
heatmapTrainDF=GroupTrain[GroupTrain.Subject_ID.isin(goodsubj)].groupby(['Subject_ID','TR']).mean()
hmTrain=hmDiff.copy()
hmFB_Train=hmDiff.copy()
hmNFB_Train=hmDiff.copy()
for indx,subj in enumerate(unique(GroupDF[GroupDF.Subject_ID.isin(goodsubj)]['Subject_ID'])):
hmFB[:,:,indx]=heatmapDF.loc[subj,'FEEDBACK'][columnNames].corr()
hmNFB[:,:,indx]=heatmapDF.loc[subj,'NOFEEDBACK'][columnNames].corr()
hmDiff[:,:,indx]=np.arctanh(heatmapDF.loc[subj,'FEEDBACK'][columnNames].corr())*np.sqrt(405)-np.arctanh(heatmapDF.loc[subj,'NOFEEDBACK'][columnNames].corr())*np.sqrt(405)
if len(GroupTrain)>0:
hmTrain[:,:,indx]=heatmapTrainDF.loc[subj][columnNames].corr()
hmFB_Train[:,:,indx]=np.arctanh(heatmapDF.loc[subj,'FEEDBACK'][columnNames].corr())*np.sqrt(405)-np.arctanh(heatmapTrainDF.loc[subj][columnNames].corr())*np.sqrt(175)
hmNFB_Train[:,:,indx]=np.arctanh(heatmapDF.loc[subj,'NOFEEDBACK'][columnNames].corr())*np.sqrt(405)-np.arctanh(heatmapTrainDF.loc[subj][columnNames].corr())*np.sqrt(175)
if len(GroupTrain)>0:
return hmFB,hmNFB,hmDiff,hmTrain,hmFB_Train,hmNFB_Train
else:
return hmFB,hmNFB,hmDiff
def slRSA_m_1Ss(ds, model, omit, partial_dsm = None, radius=3, cmetric='pearson'):
'''one subject
Executes slRSA on single subjects and returns tuple of arrays of 1-p's [0], and fisher Z transformed r's [1]
ds: pymvpa dsets for 1 subj
model: model DSM to be correlated with neural DSMs per searchlight center
partial_dsm: model DSM to be partialled out of model-neural DSM correlation
omit: list of targets omitted from pymvpa datasets
radius: sl radius, default 3
cmetric: default pearson, other optin 'spearman'
'''
if __debug__:
debug.active += ["SLC"]
for om in omit:
ds = ds[ds.sa.targets != om] # cut out omits
print('Target |%s| omitted from analysis' % (om))
ds = mean_group_sample(['targets'])(ds) #make UT ds
print('Mean group sample computed at size:',ds.shape,'...with UT:',ds.UT)
print('Beginning slRSA analysis...')
if partial_dsm == None: tdcm = rsa.TargetDissimilarityCorrelationMeasure(squareform(model), comparison_metric=cmetric)
elif partial_dsm != None: tdcm = rsa.TargetDissimilarityCorrelationMeasure(squareform(model), comparison_metric=cmetric, partial_dsm = squareform(partial_dsm))
sl = sphere_searchlight(tdcm,radius=radius)
slmap = sl(ds)
if partial_dsm == None:
print('slRSA complete with map of shape:',slmap.shape,'...p max/min:',slmap.samples[0].max(),slmap.samples[0].min(),'...r max/min',slmap.samples[1].max(),slmap.samples[1].min())
return 1-slmap.samples[1],np.arctanh(slmap.samples[0])
else:
print('slRSA complete with map of shape:',slmap.shape,'...r max/min:',slmap.samples[0].max(),slmap.samples[0].min())
return 1-slmap.samples[1],np.arctanh(slmap.samples[0])
def _surfdens(self,R,z,phi=0.,t=0.):
"""
NAME:
_surfdens
PURPOSE:
evaluate the surface density for this potential
INPUT:
R - Galactocentric cylindrical radius
z - vertical height
phi - azimuth
t - time
OUTPUT:
the surface density
HISTORY:
2018-08-19 - Written - Bovy (UofT)
"""
r= numpy.sqrt(R**2.+z**2.)
x= r/self.a
Rpa= numpy.sqrt(R**2.+self.a**2.)
Rma= numpy.sqrt(R**2.-self.a**2.+0j)
if Rma == 0:
za= z/self.a
return self.a**2./2.*((2.-2.*numpy.sqrt(za**2.+1)
+numpy.sqrt(2.)*za\
*numpy.arctan(za/numpy.sqrt(2.)))/z
+numpy.sqrt(2*za**2.+2.)\
*numpy.arctanh(za/numpy.sqrt(2.*(za**2.+1)))
/numpy.sqrt(self.a**2.+z**2.))
else:
return self.a**2.*(numpy.arctan(z/x/Rma)/Rma
+numpy.arctanh(z/x/Rpa)/Rpa
-numpy.arctan(z/Rma)/Rma
+numpy.arctan(z/Rpa)/Rpa).real
def coherr(C,J1,J2,p=0.05,Nsp1=None,Nsp2=None):
"""
Function to compute lower and upper confidence intervals on
coherency (absolute value of coherence).
C: coherence (real or complex)
J1,J2: tapered fourier transforms
p: the target P value (default 0.05)
Nsp1: number of spikes in J1, used for finite size correction.
Nsp2: number of spikes in J2, used for finite size correction.
Default is None, for no correction
Outputs:
CI: confidence interval for C, N x 2 array, (lower, upper)
phi_std: stanard deviation of phi, N array
"""
from numpy import iscomplexobj, absolute, fix, zeros, setdiff1d, real, sqrt,\
arctanh, tanh
from scipy.stats import t
J1 = _combine_trials(J1)
J2 = _combine_trials(J2)
N,K = J1.shape
assert J1.shape==J2.shape, "J1 and J2 must have the same dimensions."
assert N == C.size, "S and J lengths don't match"
if iscomplexobj(C): C = absolute(C)
pp = 1 - p/2
dof = 2*K
dof1 = dof if Nsp1 is None else fix(2.*Nsp1*dof/(2.*Nsp1+dof))
dof2 = dof if Nsp2 is None else fix(2.*Nsp2*dof/(2.*Nsp2+dof))
dof = min(dof1,dof2)
Cerr = zeros((N,2))
tcrit = t(dof-1).ppf(pp).tolist()
atanhCxyk = zeros((N,K))
phasefactorxyk = zeros((N,K),dtype='complex128')
for k in xrange(K):
indxk = setdiff1d(range(K),[k])
J1k = J1[:,indxk]
J2k = J2[:,indxk]
eJ1k = real(J1k * J1k.conj()).sum(1)
eJ2k = real(J2k * J2k.conj()).sum(1)
eJ12k = (J1k.conj() * J2k).sum(1)
Cxyk = eJ12k/sqrt(eJ1k*eJ2k)
absCxyk = absolute(Cxyk)
atanhCxyk[:,k] = sqrt(2*K-2)*arctanh(absCxyk)
phasefactorxyk[:,k] = Cxyk / absCxyk
atanhC = sqrt(2*K-2)*arctanh(C);
sigma12 = sqrt(K-1)* atanhCxyk.std(1)
Cu = atanhC + tcrit * sigma12
Cl = atanhC - tcrit * sigma12
Cerr[:,0] = tanh(Cl / sqrt(2*K-2))
Cerr[:,1] = tanh(Cu / sqrt(2*K-2))
phistd = (2*K-2) * (1 - absolute(phasefactorxyk.mean(1)))
return Cerr, phistd
def function_A(vega, a, b, t):
"""Helper function A(t) defined in thesis."""
kappa = 2 * b ** (-2) / vega
val = (1 - sqrt( 1 + kappa * (1 - exp( -vega * t )))) \
/ kappa + 1 / sqrt( 1 + kappa) \
* (arctanh( sqrt( 1 + kappa * ( 1 - exp( -vega * t)))/ \
sqrt(1 + kappa)) - arctanh(1 / sqrt( 1 + kappa)))
return val
def test_arctanh():
a = afnumpy.random.random((2,3))
b = numpy.array(a)
fassert(afnumpy.arctanh(a), numpy.arctanh(b))
c = afnumpy.random.random((2,3))
d = numpy.array(a)
fassert(afnumpy.arctanh(a, out=c), numpy.arctanh(b, out=d))
fassert(c, d)
def get_ttest_map(map_name):
subjs = map(lambda n: 'sub%03d' % n, SUBJECTS)
vols = load_maps(map_name, subjs)
# se = 1.0/np.sqrt(1199-6)
popmean = np.arctanh(0.03)
ttest = stats.ttest_1samp(np.arctanh(vols), popmean, axis=0)[0]
ttest[ttest < 4] = np.NaN
return get_volume(ttest, 4, 15)
def g1g2_to_e1e2(g1, g2):
"""
convert reduced shear g1,g2 to standard ellipticity
parameters e1,e2
uses eta representation but could also use
e1 = 2*g1/(1 + g1**2 + g2**2)
e2 = 2*g2/(1 + g1**2 + g2**2)
parameters
----------
g1,g2: scalars
Reduced shear space shapes
outputs
-------
e1,e2: tuple of scalars
shapes in (ixx-iyy)/(ixx+iyy) style space
"""
g=numpy.sqrt(g1*g1 + g2*g2)
if isinstance(g1, numpy.ndarray):
w,=numpy.where(g >= 1.0)
if w.size != 0:
raise GMixRangeError("some g were out of bounds")
eta = 2*numpy.arctanh(g)
e = numpy.tanh(eta)
numpy.clip(e, 0.0, 0.99999999, e)
e1=numpy.zeros(g.size)
e2=numpy.zeros(g.size)
w,=numpy.where(g != 0.0)
if w.size > 0:
fac = e[w]/g[w]
e1[w] = fac*g1[w]
e2[w] = fac*g2[w]
else:
if g >= 1.:
raise GMixRangeError("g out of bounds: %s" % g)
if g == 0.0:
return (0.0, 0.0)
eta = 2*numpy.arctanh(g)
e = numpy.tanh(eta)
if e >= 1.:
# round off?
e = 0.99999999
fac = e/g
e1 = fac*g1
e2 = fac*g2
return e1,e2
def e1e2_to_g1g2(e1, e2):
"""
convert e1,e2 to reduced shear style ellipticity
parameters
----------
e1,e2: tuple of scalars
shapes in (ixx-iyy)/(ixx+iyy) style space
outputs
-------
g1,g2: scalars
Reduced shear space shapes
"""
e = numpy.sqrt(e1*e1 + e2*e2)
if isinstance(e1, numpy.ndarray):
w,=numpy.where(e >= 1.0)
if w.size != 0:
raise GMixRangeError("some e were out of bounds")
eta=numpy.arctanh(e)
g = numpy.tanh(0.5*eta)
numpy.clip(g, 0.0, 0.99999999, g)
g1=numpy.zeros(g.size)
g2=numpy.zeros(g.size)
w,=numpy.where(e != 0.0)
if w.size > 0:
fac = g[w]/e[w]
g1[w] = fac*e1[w]
g2[w] = fac*e2[w]
else:
if e >= 1.:
raise GMixRangeError("e out of bounds: %s" % e)
if e == 0.0:
g1,g2=0.0,0.0
else:
eta=numpy.arctanh(e)
g = numpy.tanh(0.5*eta)
if g >= 1.:
# round off?
g = 0.99999999
fac = g/e
g1 = fac*e1
g2 = fac*e2
return g1,g2
def make_from_examples(cls, X, low, high, directed = True):
#for every pair of examples (i,j)
#make a feature that takes on the value low at i
#and value high at j
#if directed, order of (i,j) matters, otherwise it does not
m,n = X.shape
if directed:
h = m **2 - m
else:
h = m * (m-1)/2
W = N.zeros((n,h))
b = N.zeros(h)
idx = 0
inv_low = N.arctanh(low)
if N.abs(N.tanh(inv_low)-low) > 1e-6:
assert False
#
inv_high = N.arctanh(high)
for i in xrange(X.shape[0]):
if directed:
r = xrange(m)
else:
r = xrange(i+1,m)
for j in r:
if i == j:
continue
diff = X[j,:] - X[i,:]
direction = diff / N.sqrt(N.square(diff).sum())
pi = N.dot(X[i,:],direction)
pj = N.dot(X[j,:],direction)
wmag = (inv_high - inv_low) / (pj - pi)
b[idx] = (pj*inv_low - pi*inv_high) / (pj - pi)
W[:,idx] = wmag * direction
#check it
ival = N.tanh(N.dot(W[:,idx],X[i,:])+b[idx])
jval = N.tanh(N.dot(W[:,idx],X[j,:])+b[idx])
assert abs(ival-low) < 1e-6
assert abs(jval-high) < 1e-6
idx += 1
assert idx == h
return TanhFeatureExtractor(W,b)
def diffcorrcoeftest(rvalue1, rvalue2, N1, N2):
''' tests for difference between 2 r values by performing fishers transformation and performing t-test on the z values. returns z and two tailed p '''
r_z1=np.arctanh(rvalue1) #equivalent to 0.5 * np.log((1 + rvalue1)/(1 - rvalue1))
r_z2=np.arctanh(rvalue2)
se_diff_r = np.sqrt(1.0/(N1 - 3.0) + 1.0/(N2 - 3.0))
diff = r_z1 - r_z2
z = abs(diff / se_diff_r)
p = (1 - scipy.stats.norm.cdf(z))*2
return round(z,3), round(p,3)
def atanh(x):
"""
Inverse hyperbolic tangent
"""
if isinstance(x, UncertainFunction):
mcpts = np.arctanh(x._mcpts)
return UncertainFunction(mcpts)
else:
return np.arctanh(x)
def fisherTransformation(r1, r2, n1, n2):
z1 = numpy.arctanh(r1)
z2 = numpy.arctanh(r2)
se = numpy.sqrt(1/(float(n1)-3) + 1/(float(n2)-3))
z = (z1-z2)/se
return z
def test_arctanh(self):
import math
from numpy import arctanh
for v in [0.99, 0.5, 0, -0.5, -0.99]:
assert math.atanh(v) == arctanh(v)
for v in [2.0, -2.0]:
assert math.isnan(arctanh(v))
for v in [1.0, -1.0]:
assert arctanh(v) == math.copysign(float("inf"), v)
def compute_S(x):
''' compute the nonlinearity of the belief prop system
S(x) is obtained by applying arctanh to x at i in S_ '''
global S_, q
y = np.ones(q,1)
for i in (i,q):
for j in nonzero(S_[i,:]):
y[i] *= np.arctanh(x[j]/2)
y = 2 * np.arctanh(y)
def func_hernquist(x): res = x*0 idx1 = x>0 idx2 = x<1 idx = idx1&idx2 res[idx] = 1.0/np.sqrt(1-x[idx]*x[idx])*np.arctanh(np.sqrt(1-x[idx]*x[idx])) idx = x == 1 res[idx] = 1.0 idx = x>1 res[idx] = 1.0/np.sqrt(x[idx]*x[idx]-1.0)*np.arctanh(np.sqrt(x[idx]*x[idx]-1)) return res
def corr_ci(r, n, confidence=0.95):
""" Compute confidence interval for Spearman or Pearson correlation coefficients based on Fisher transformation
https://en.wikipedia.org/wiki/Pearson_product-moment_correlation_coefficient#Using_the_Fisher_transformation
:param r: correlation coefficient
:param n: sample size
:param confidence: sample size
:return: low and high
"""
delta = stats.norm.ppf(1.0 - (1 - confidence) / 2) / np.sqrt(n - 3)
lower = np.tanh(np.arctanh(r) - delta)
upper = np.tanh(np.arctanh(r) + delta)
return lower, upper
def g1g2_to_eta1eta2(g1, g2):
"""
convert reduced shear g1,g2 to eta style ellipticity
parameters
----------
g1,g2: scalars
Reduced shear space shapes
outputs
-------
eta1,eta2: tuple of scalars
eta space shapes
"""
if isinstance(g1, numpy.ndarray):
g=numpy.sqrt(g1*g1 + g2*g2)
w,=numpy.where(g >= 1.0)
if w.size != 0:
raise GMixRangeError("some g were out of bounds")
eta1=numpy.zeros(g.size)
eta2=eta1.copy()
w,=numpy.where(g > 0.0)
if w.size > 0:
eta = 2*numpy.arctanh(g[w])
fac = eta[w]/g[w]
eta1[w] = fac*g1[w]
eta2[w] = fac*g2[w]
else:
g=numpy.sqrt(g1*g1 + g2*g2)
if g >= 1.:
raise GMixRangeError("g out of bounds: %s converting to eta" % g)
if g == 0.0:
eta1, eta2=0.0, 0.0
else:
eta = 2*numpy.arctanh(g)
fac = eta/g
eta1 = fac*g1
eta2 = fac*g2
return eta1,eta2
def compute_best_J(im):
# A more precise computation of sisj could be done...
l_x = len(im)
X, Y = np.ogrid[:l_x, :l_x]
mask = ((X - l_x/2)**2 + (Y - l_x/2)**2 <= (l_x/2)**2)
grad = ndimage.morphological_gradient(im, footprint=np.ones((3, 3)))
sisj_average = 1 - (grad[mask] > 0).mean()
J1 = np.arctanh(sisj_average)
grad2 = np.abs(np.diff(im, axis=0))[:, :-1] + np.abs(np.diff(im, axis=1))[:-1]
sisj_average = 1 - 2*(grad2[mask[:-1, :-1]] > 0).mean()
J2 = np.arctanh(sisj_average)
return J1, J2
def spearmanr_ci_95(r,n):
"""Calculate the 95% confidence interval for a spearman r coefficient
See http://stats.stackexchange.com/a/18904
"""
if n < 5:
print('Cannot compute CI for Spearman R with less than five samples')
return (np.nan,np.nan)
delta = 1.96/np.sqrt(n-3)
lower = np.tanh(np.arctanh(r)-delta)
upper = np.tanh(np.arctanh(r)+delta)
return (lower,upper)
def getcaustics(b_I,q):
## Do not change the next line or face consequences
t=np.arange(0.0,pi/2.0+pi/100.0,pi/200.0);
den=(1+q**2)-(1-q**2)*np.cos(2*t)
r=sqrt(2.)*b_I/np.sqrt(den)
x=r*np.cos(t);
y=r*np.sin(t);
## Generate parametric function for the tangential caustic
xi=np.sqrt(2.*(1-q**2)/den);
u=np.cos(t)*r;
v=np.sin(t)*r;
if(q!=1.0):
u=u-b_I*np.arctan(xi*np.cos(t))/sqrt(1-q**2);
v=v-b_I*np.arctanh(xi*np.sin(t))/sqrt(1-q**2);
## Generate parametric function for the radial caustic
up=b_I*0.0;
vp=b_I*0.0;
if(q!=1.0):
up=-b_I*np.arctan(xi*np.cos(t))/sqrt(1-q**2);
vp=-b_I*np.arctanh(xi*np.sin(t))/sqrt(1-q**2);
## Set up splines for the tangential and radial caustic
r1=np.sqrt(u**2+v**2);
r2=np.sqrt(up**2+vp**2);
t1=[0.]*u.size;
t2=[0.]*up.size;
for ii in range(u.size):
if(u[ii]==0.):
t1[ii]=np.pi/2.;
else:
t1[ii]=arctan(np.abs(v[ii]/u[ii]));
if(t[ii]>pi/2.0):
t1[ii]=pi-t1[ii];
for ii in range(up.size):
if(up[ii]==0.):
t2[ii]=np.pi/2.;
else:
t2[ii]=arctan(np.abs(vp[ii]/up[ii]));
if(t[ii]>pi/2.0):
t2[ii]=pi-t2[ii];
c1=interp1d(t1,r1,kind='cubic');
c2=interp1d(t2,r2,kind='cubic');
## Calculate maximum radial distance between the caustics and the origin
maxr=np.max([r1,r2]);
return c1,c2,maxr;
def make_ttest(reg1, reg2):
masker = NiftiMasker(nib.load(MASK_FILE), standardize=False)
masker.fit()
subjects = [1, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
a = np.arctanh(join_all_subjects(reg1, subjects, masker))
b = np.arctanh(join_all_subjects(reg2, subjects, masker))
t, prob = ttest_rel(a, b)
tt = masker.inverse_transform(t)
pp = masker.inverse_transform(prob)
return tt, pp
def arctanh(x, out=None):
"""
Raises a ValueError if input cannot be rescaled to a dimensionless
quantity.
"""
if not isinstance(x, Quantity):
return np.arctanh(x, out)
return Quantity(
np.arctanh(x.rescale(dimensionless).magnitude, out),
dimensionless,
copy=False
)
def show_map():
sub_nums = map(int, raw_input('enter subject numbers --> ').split())
if len(sub_nums) == 0:
sub_nums = SUBJECTS
subjs = map(lambda n: 'sub%03d' % n, sub_nums)
maps = sorted(REGRESSORS)
for i, m in enumerate(maps):
print '%d. %s' % (i, m)
map_nums = map(int, raw_input('Choose map --> ').split())
if len(map_nums) == 1:
map_name = maps[map_nums[0]]
vols = load_maps(map_name, subjs)
volumes = dict(zip(subjs, vols))
# vol_objects = dict(map(lambda x: (map_name+' '+x[0], get_volume(x[1], 0.1, 0.4)), volumes.items()))
vol_objects = dict()
if len(vols) > 1:
vols = np.arctanh(vols)
se = 1.0/np.sqrt(1199-6) # ~0.03
popmean = np.arctanh(0.03)
ttest = stats.ttest_1samp(vols, popmean, axis=0)[0]
ttest[ttest < 4] = np.NaN
vol_objects.update(**{map_name + ' ttest': get_volume(ttest, 4, 15)})
std = np.tanh(np.std(vols, axis=0))
mean = np.tanh(mean_vol(vols))
vols = np.tanh(vols)
vol_objects.update(**{map_name + ' std': get_volume(std, 0.05, 0.15)})
vol_objects.update(**{map_name + ' mean': get_volume(mean, 0.1, 0.4)})
ds = cortex.dataset.Dataset(**vol_objects)
if len(map_nums) == 2:
mname1 = maps[map_nums[0]]
mname2 = maps[map_nums[1]]
maps1 = load_maps(mname1, subjs)
maps2 = load_maps(mname2, subjs)
diff_maps = [compute_diff_map(x[0], x[1]) for x in zip(maps1, maps2)]
vol = np.tanh(compute_diff_map(mean_vol(np.arctanh(maps1)), mean_vol(np.arctanh(maps2))))
d = get_volume(vol, -0.1, 0.1, cmap='coldwarm')
diff_vols = [get_volume(vol, -0.1, 0.1, cmap='coldwarm') for vol in diff_maps]
# vol_objects = dict(zip(subjs, diff_vols))
vol_objects = dict()
vol_objects[mname1+'-'+mname2] = d
ds = cortex.dataset.Dataset(**vol_objects)
web = cortex.webgl.show(ds, types=('inflated',), port=8081, open_browser=True)
raw_input()
def check_equation_20(q,gamma=3.):
'''
Kinematic factor difference between analytic and numerical for
gamma = 3
'''
Q = q/qpot_from_q(q)
F=.5*quad(lambda t: np.sin(t)**3*(np.sin(t)**2+np.cos(t)**2./Q/Q)**(-gamma/2.),0.,np.pi)[0]/quad(lambda t: np.sin(t)*np.cos(t)**2*(np.sin(t)**2+np.cos(t)**2./Q/Q)**(-gamma/2.),0.,np.pi)[0]
F2 = binney_tremaine_virial_ratio(q)
Qst = 1./Q/Q
if(gamma==3.):
if(Qst>1.):
Q = np.sqrt(Qst-1.)
G = .5*(Qst*Q-np.sqrt(Qst)*np.arcsinh(Q))/(np.sqrt(Qst)*np.arcsinh(Q)-Q)
else:
Q = np.sqrt(1.-Qst)
G = .5*(Qst*Q*Q-np.sqrt(Qst)*Q*np.arccos(np.sqrt(Qst)))/(np.sqrt(Qst)*Q*np.arccos(np.sqrt(Qst))-Q*Q)
if(gamma==2.):
if(Qst>1.):
Q = np.sqrt(Qst-1.)
T = np.arctan(Q)
G = .5*(Qst*T-Q)/(Q-T)
else:
Q = np.sqrt(1.-Qst)
T = np.arctanh(Q)
G = .5*(Qst*T-Q)/(Q-T)
if(gamma==4.):
if(Qst>1.):
Q = np.sqrt(Qst-1.)
G = (Qst*(Qst-2.)*np.arctan(Q)+Qst*Q)/(Qst*np.arctan(Q)-Q)/2.
else:
Q = np.sqrt(1.-Qst)
G = (Qst*(Qst-2.)*np.arctanh(Q)+Qst*Q)/(Qst*np.arctanh(Q)-Q)/2.
if(gamma==3.):
q = np.linspace(0.2,4.)
FF = np.zeros(len(q))
FF2 = np.zeros(len(q))
for n,i in enumerate(q):
Q = i/qpot_from_q(i)
FF[n]=.5*quad(lambda t: np.sin(t)**3*(np.sin(t)**2+np.cos(t)**2./Q/Q)**(-gamma/2.),0.,np.pi)[0]/quad(lambda t: np.sin(t)*np.cos(t)**2*(np.sin(t)**2+np.cos(t)**2./Q/Q)**(-gamma/2.),0.,np.pi)[0]
FF2[n] = binney_tremaine_virial_ratio(i)
plt.plot(q,np.log10(FF))
plt.plot(q,np.log10(FF2))
plt.savefig('tmp.pdf')
plt.clf()
if(gamma==3.):
return F-G,F2-G
else:
return F-G
def extractnetstats(ID,
network,
thr,
conn_model,
est_path,
mask,
out_file=None):
from pynets import thresholding, utils
pruning = True
##Load and threshold matrix
in_mat = np.array(np.genfromtxt(est_path))
in_mat = thresholding.autofix(in_mat)
##Normalize connectivity matrix (weights between 0-1)
in_mat = thresholding.normalize(in_mat)
##Get hyperbolic tangent of matrix if non-sparse (i.e. fischer r-to-z transform)
if conn_model == 'corr':
in_mat = np.arctanh(in_mat)
in_mat[np.isnan(in_mat)] = 0
in_mat[np.isinf(in_mat)] = 1
##Get dir_path
dir_path = os.path.dirname(os.path.realpath(est_path))
##Load numpy matrix as networkx graph
G_pre = nx.from_numpy_matrix(in_mat)
##Prune irrelevant nodes (i.e. nodes who are fully disconnected from the graph and/or those whose betweenness centrality are > 3 standard deviations below the mean)
if pruning == True:
[G_pruned, _, _] = most_important(G_pre)
else:
G_pruned = G_pre
##Make directed if sparse
if conn_model != 'corr' and conn_model != 'cov' and conn_model != 'tangent':
G_di = nx.DiGraph(G_pruned)
G_dir = G_di.to_directed()
G = G_pruned
else:
G = G_pruned
##Get corresponding matrix
in_mat = nx.to_numpy_array(G)
##Print graph summary
print('\n\nThreshold: ' + str(thr))
print('Source File: ' + str(est_path))
info_list = list(nx.info(G).split('\n'))[2:]
for i in info_list:
print(i)
try:
G_dir
print('Analyzing DIRECTED graph when applicable...')
except:
print('Graph is UNDIRECTED')
if conn_model == 'corr' or conn_model == 'cov' or conn_model == 'tangent':
if nx.is_connected(G) == True:
num_conn_comp = nx.number_connected_components(G)
print('Graph is CONNECTED with ' + str(num_conn_comp) +
' connected component(s)')
else:
print('Graph is DISCONNECTED')
print('\n')
##Create Length matrix
mat_len = thresholding.weight_conversion(in_mat, 'lengths')
##Load numpy matrix as networkx graph
G_len = nx.from_numpy_matrix(mat_len)
##Save G as gephi file
if mask:
if network:
nx.write_graphml(
G, dir_path + '/' + ID + '_' + network + '_' +
str(os.path.basename(mask).split('.')[0]) + '.graphml')
else:
nx.write_graphml(
G, dir_path + '/' + ID + '_' +
str(os.path.basename(mask).split('.')[0]) + '.graphml')
else:
if network:
nx.write_graphml(G,
dir_path + '/' + ID + '_' + network + '.graphml')
else:
nx.write_graphml(G, dir_path + '/' + ID + '.graphml')
###############################################################
########### Calculate graph metrics from graph G ##############
###############################################################
from networkx.algorithms import degree_assortativity_coefficient, average_clustering, average_shortest_path_length, degree_pearson_correlation_coefficient, graph_number_of_cliques, transitivity, betweenness_centrality, eigenvector_centrality, communicability_betweenness_centrality, clustering, degree_centrality
from pynets.netstats import average_local_efficiency, global_efficiency, local_efficiency, modularity_louvain_dir, smallworldness
##For non-nodal scalar metrics from custom functions, add the name of the function to metric_list and add the function (with a G-only input) to the netstats module.
metric_list = [
global_efficiency, average_local_efficiency, smallworldness,
degree_assortativity_coefficient, average_clustering,
average_shortest_path_length, degree_pearson_correlation_coefficient,
graph_number_of_cliques, transitivity
]
##Custom Weight Parameter
#custom_weight = 0.25
custom_weight = None
##Iteratively run functions from above metric list that generate single scalar output
num_mets = len(metric_list)
net_met_arr = np.zeros([num_mets, 2], dtype='object')
j = 0
for i in metric_list:
met_name = str(i).split('<function ')[1].split(' at')[0]
net_met = met_name
try:
if i is 'average_shortest_path_length':
try:
try:
net_met_val = float(i(G_dir))
print('Calculating from directed graph...')
except:
net_met_val = float(i(G))
except:
##case where G is not fully connected
net_met_val = float(
average_shortest_path_length_for_all(G))
if custom_weight is not None and i is 'degree_assortativity_coefficient' or i is 'global_efficiency' or i is 'average_local_efficiency' or i is 'average_clustering':
custom_weight_param = 'weight = ' + str(custom_weight)
try:
net_met_val = float(i(G_dir, custom_weight_param))
print('Calculating from directed graph...')
except:
net_met_val = float(i(G, custom_weight_param))
else:
try:
net_met_val = float(i(G_dir))
print('Calculating from directed graph...')
except:
net_met_val = float(i(G))
except:
net_met_val = np.nan
net_met_arr[j, 0] = net_met
net_met_arr[j, 1] = net_met_val
print(net_met)
print(str(net_met_val))
print('\n')
j = j + 1
net_met_val_list = list(net_met_arr[:, 1])
##Run miscellaneous functions that generate multiple outputs
##Calculate modularity using the Louvain algorithm
[community_aff, modularity] = modularity_louvain_dir(in_mat)
##Calculate core-periphery subdivision
[Coreness_vec, Coreness_q] = core_periphery_dir(in_mat)
##Local Efficiency
try:
try:
le_vector = local_efficiency(G_dir)
except:
le_vector = local_efficiency(G)
print('\nExtracting Local Efficiency vector for all network nodes...')
le_vals = list(le_vector.values())
le_nodes = list(le_vector.keys())
num_nodes = len(le_nodes)
le_arr = np.zeros([num_nodes + 1, 2], dtype='object')
j = 0
for i in range(num_nodes):
le_arr[j, 0] = str(le_nodes[j]) + '_local_efficiency'
#print('\n' + str(le_nodes[j]) + '_local_efficiency')
try:
le_arr[j, 1] = le_vals[j]
except:
le_arr[j, 1] = np.nan
#print(str(le_vals[j]))
j = j + 1
le_arr[num_nodes, 0] = 'MEAN_local_efficiency'
nonzero_arr_le = np.delete(le_arr[:, 1], [0])
le_arr[num_nodes, 1] = np.mean(nonzero_arr_le)
print('Mean Local Efficiency across nodes: ' +
str(le_arr[num_nodes, 1]))
print('\n')
except:
pass
##Local Clustering
try:
cl_vector = clustering(G)
print('\nExtracting Local Clustering vector for all network nodes...')
cl_vals = list(cl_vector.values())
cl_nodes = list(cl_vector.keys())
num_nodes = len(cl_nodes)
cl_arr = np.zeros([num_nodes + 1, 2], dtype='object')
j = 0
for i in range(num_nodes):
cl_arr[j, 0] = str(cl_nodes[j]) + '_local_clustering'
#print('\n' + str(cl_nodes[j]) + '_local_clustering')
try:
cl_arr[j, 1] = cl_vals[j]
except:
cl_arr[j, 1] = np.nan
#print(str(cl_vals[j]))
j = j + 1
cl_arr[num_nodes, 0] = 'MEAN_local_efficiency'
nonzero_arr_cl = np.delete(cl_arr[:, 1], [0])
cl_arr[num_nodes, 1] = np.mean(nonzero_arr_cl)
print('Mean Local Clustering across nodes: ' +
str(cl_arr[num_nodes, 1]))
print('\n')
except:
pass
##Degree centrality
try:
try:
dc_vector = degree_centrality(G_dir)
except:
dc_vector = degree_centrality(G)
print('\nExtracting Degree Centrality vector for all network nodes...')
dc_vals = list(dc_vector.values())
dc_nodes = list(dc_vector.keys())
num_nodes = len(dc_nodes)
dc_arr = np.zeros([num_nodes + 1, 2], dtype='object')
j = 0
for i in range(num_nodes):
dc_arr[j, 0] = str(dc_nodes[j]) + '_degree_centrality'
#print('\n' + str(dc_nodes[j]) + '_degree_centrality')
try:
dc_arr[j, 1] = dc_vals[j]
except:
dc_arr[j, 1] = np.nan
#print(str(cl_vals[j]))
j = j + 1
dc_arr[num_nodes, 0] = 'MEAN_degree_centrality'
nonzero_arr_dc = np.delete(dc_arr[:, 1], [0])
dc_arr[num_nodes, 1] = np.mean(nonzero_arr_dc)
print('Mean Degree Centrality across nodes: ' +
str(dc_arr[num_nodes, 1]))
print('\n')
except:
pass
##Betweenness Centrality
try:
bc_vector = betweenness_centrality(G_len, normalized=True)
print(
'\nExtracting Betweeness Centrality vector for all network nodes...'
)
bc_vals = list(bc_vector.values())
bc_nodes = list(bc_vector.keys())
num_nodes = len(bc_nodes)
bc_arr = np.zeros([num_nodes + 1, 2], dtype='object')
j = 0
for i in range(num_nodes):
bc_arr[j, 0] = str(bc_nodes[j]) + '_betweenness_centrality'
#print('\n' + str(bc_nodes[j]) + '_betw_cent')
try:
bc_arr[j, 1] = bc_vals[j]
except:
bc_arr[j, 1] = np.nan
#print(str(bc_vals[j]))
j = j + 1
bc_arr[num_nodes, 0] = 'MEAN_betw_cent'
nonzero_arr_betw_cent = np.delete(bc_arr[:, 1], [0])
bc_arr[num_nodes, 1] = np.mean(nonzero_arr_betw_cent)
print('Mean Betweenness Centrality across nodes: ' +
str(bc_arr[num_nodes, 1]))
print('\n')
except:
pass
##Eigenvector Centrality
try:
try:
ec_vector = eigenvector_centrality(G_dir, max_iter=1000)
except:
ec_vector = eigenvector_centrality(G, max_iter=1000)
print(
'\nExtracting Eigenvector Centrality vector for all network nodes...'
)
ec_vals = list(ec_vector.values())
ec_nodes = list(ec_vector.keys())
num_nodes = len(ec_nodes)
ec_arr = np.zeros([num_nodes + 1, 2], dtype='object')
j = 0
for i in range(num_nodes):
ec_arr[j, 0] = str(ec_nodes[j]) + '_eigenvector_centrality'
#print('\n' + str(ec_nodes[j]) + '_eig_cent')
try:
ec_arr[j, 1] = ec_vals[j]
except:
ec_arr[j, 1] = np.nan
#print(str(ec_vals[j]))
j = j + 1
ec_arr[num_nodes, 0] = 'MEAN_eig_cent'
nonzero_arr_eig_cent = np.delete(ec_arr[:, 1], [0])
ec_arr[num_nodes, 1] = np.mean(nonzero_arr_eig_cent)
print('Mean Eigenvector Centrality across nodes: ' +
str(ec_arr[num_nodes, 1]))
print('\n')
except:
pass
##Communicability Centrality
try:
cc_vector = communicability_betweenness_centrality(G, normalized=True)
print(
'\nExtracting Communicability Centrality vector for all network nodes...'
)
cc_vals = list(cc_vector.values())
cc_nodes = list(cc_vector.keys())
num_nodes = len(cc_nodes)
cc_arr = np.zeros([num_nodes + 1, 2], dtype='object')
j = 0
for i in range(num_nodes):
cc_arr[j, 0] = str(cc_nodes[j]) + '_communicability_centrality'
#print('\n' + str(cc_nodes[j]) + '_comm_cent')
try:
cc_arr[j, 1] = cc_vals[j]
except:
cc_arr[j, 1] = np.nan
#print(str(cc_vals[j]))
j = j + 1
cc_arr[num_nodes, 0] = 'MEAN_comm_cent'
nonzero_arr_comm_cent = np.delete(cc_arr[:, 1], [0])
cc_arr[num_nodes, 1] = np.mean(nonzero_arr_comm_cent)
print('Mean Communicability Centrality across nodes: ' +
str(cc_arr[num_nodes, 1]))
print('\n')
except:
pass
##Rich club coefficient
try:
rc_vector = rich_club_coefficient(G, normalized=True)
print(
'\nExtracting Rich Club Coefficient vector for all network nodes...'
)
rc_vals = list(rc_vector.values())
rc_edges = list(rc_vector.keys())
num_edges = len(rc_edges)
rc_arr = np.zeros([num_edges + 1, 2], dtype='object')
j = 0
for i in range(num_edges):
rc_arr[j, 0] = str(rc_edges[j]) + '_rich_club'
#print('\n' + str(rc_edges[j]) + '_rich_club')
try:
rc_arr[j, 1] = rc_vals[j]
except:
rc_arr[j, 1] = np.nan
#print(str(rc_vals[j]))
j = j + 1
##Add mean
rc_arr[num_edges, 0] = 'MEAN_rich_club'
nonzero_arr_rich_club = np.delete(rc_arr[:, 1], [0])
rc_arr[num_edges, 1] = np.mean(nonzero_arr_rich_club)
print('Mean Rich Club Coefficient across edges: ' +
str(rc_arr[num_edges, 1]))
print('\n')
except:
pass
##Create a list of metric names for scalar metrics
metric_list_names = []
net_met_val_list_final = net_met_val_list
for i in net_met_arr[:, 0]:
metric_list_names.append(i)
##Add modularity measure
try:
metric_list_names.append('Modularity')
net_met_val_list_final.append(modularity)
except:
pass
##Add Core/Periphery measure
try:
metric_list_names.append('Coreness')
net_met_val_list_final.append(Coreness_q)
except:
pass
##Add local efficiency measures
try:
for i in le_arr[:, 0]:
metric_list_names.append(i)
net_met_val_list_final = net_met_val_list_final + list(le_arr[:, 1])
except:
pass
##Add local clustering measures
try:
for i in cl_arr[:, 0]:
metric_list_names.append(i)
net_met_val_list_final = net_met_val_list_final + list(cl_arr[:, 1])
except:
pass
##Add centrality measures
try:
for i in dc_arr[:, 0]:
metric_list_names.append(i)
net_met_val_list_final = net_met_val_list_final + list(dc_arr[:, 1])
except:
pass
try:
for i in bc_arr[:, 0]:
metric_list_names.append(i)
net_met_val_list_final = net_met_val_list_final + list(bc_arr[:, 1])
except:
pass
try:
for i in ec_arr[:, 0]:
metric_list_names.append(i)
net_met_val_list_final = net_met_val_list_final + list(ec_arr[:, 1])
except:
pass
try:
for i in cc_arr[:, 0]:
metric_list_names.append(i)
net_met_val_list_final = net_met_val_list_final + list(cc_arr[:, 1])
except:
pass
##Add rich club measure
try:
for i in rc_arr[:, 0]:
metric_list_names.append(i)
net_met_val_list_final = net_met_val_list_final + list(rc_arr[:, 1])
except:
pass
##Save metric names as pickle
try:
import cPickle
except ImportError:
import _pickle as cPickle
if mask != None:
if network != None:
met_list_picke_path = os.path.dirname(os.path.abspath(
est_path)) + '/net_metric_list_' + network + '_' + str(
os.path.basename(mask).split('.')[0])
else:
met_list_picke_path = os.path.dirname(
os.path.abspath(est_path)) + '/net_metric_list_' + str(
os.path.basename(mask).split('.')[0])
else:
if network != None:
met_list_picke_path = os.path.dirname(
os.path.abspath(est_path)) + '/net_metric_list_' + network
else:
met_list_picke_path = os.path.dirname(
os.path.abspath(est_path)) + '/net_metric_list'
cPickle.dump(metric_list_names, open(met_list_picke_path, 'wb'))
##And save results to csv
out_path = utils.create_csv_path(ID, network, conn_model, thr, mask,
dir_path)
np.savetxt(out_path, net_met_val_list_final)
return (out_path)
def selcomps(seldict,
mmix,
mask,
ref_img,
manacc,
n_echos,
t2s,
s0,
olevel=2,
oversion=99,
filecsdata=True,
savecsdiag=True,
strict_mode=False):
"""
Labels ICA components to keep or remove from denoised data
The selection process uses pre-calculated parameters for each ICA component
inputted into this function in `seldict` such as
Kappa (a T2* weighting metric), Rho (an S0 weighting metric), and variance
explained. Additonal selection metrics are calculated within this function
and then used to classify each component into one of four groups.
Parameters
----------
seldict : :obj:`dict`
As output from `fitmodels_direct`
mmix : (C x T) array_like
Mixing matrix for converting input data to component space, where `C`
is components and `T` is the number of volumes in the original data
mask : (S,) array_like
Boolean mask array
ref_img : :obj:`str` or img_like
Reference image to dictate how outputs are saved to disk
manacc : :obj:`list`
Comma-separated list of indices of manually accepted components
n_echos : :obj:`int`
Number of echos in original data
t2s : (S,) array_like
Estimated T2* map
s0 : (S,) array_like
S0 map
olevel : :obj:`int`, optional
Default: 2
oversion : :obj:`int`, optional
Default: 99
filecsdata: :obj:`bool`, optional
Default: False
savecsdiag: :obj:`bool`, optional
Default: True
strict_mode: :obj:`bool`, optional
Default: False
Returns
-------
acc : :obj:`list`
Indices of accepted (BOLD) components in `mmix`
rej : :obj:`list`
Indices of rejected (non-BOLD) components in `mmix`
midk : :obj:`list`
Indices of mid-K (questionable) components in `mmix`
These components are typically removed from the data during denoising
ign : :obj:`list`
Indices of ignored components in `mmix`
Ignored components are considered to have too low variance to matter.
They are not processed through the accept vs reject decision tree and
are NOT removed during the denoising process
Notes
-----
The selection algorithm used in this function is from work by prantikk
It is from selcomps function in select_model_fft20e.py in
version 3.2 of MEICA at:
https://github.com/ME-ICA/me-ica/blob/b2781dd087ab9de99a2ec3925f04f02ce84f0adc/meica.libs/select_model_fft20e.py
Many of the early publications using and evaulating the MEICA method used a
different selection algorithm by prantikk. The final 2.5 version of that
algorithm in the selcomps function in select_model.py at:
https://github.com/ME-ICA/me-ica/blob/b2781dd087ab9de99a2ec3925f04f02ce84f0adc/meica.libs/select_model.py
In both algorithms, the ICA component selection process uses multiple
metrics that include: kappa, rho, variance explained, compent spatial
weighting maps, noise and spatial frequency metrics, and measures of
spatial overlap across metrics. The precise calculations may vary between
algorithms. The most notable difference is that the v2.5 algorithm is a
fixed decision tree where all sections were made based on whether
combinations of metrics crossed various thresholds. In the v3.5 algorithm,
clustering and support vector machines are also used to classify components
based on how similar metrics in one component are similar to metrics in
other components.
"""
if mmix.ndim != 2:
raise ValueError('Parameter mmix should be 2d, not {0}d'.format(
mmix.ndim))
elif t2s.ndim != 1: # FIT not necessarily supported
raise ValueError('Parameter t2s should be 1d, not {0}d'.format(
t2s.ndim))
elif s0.ndim != 1: # FIT not necessarily supported
raise ValueError('Parameter s0 should be 1d, not {0}d'.format(s0.ndim))
elif not (t2s.shape[0] == s0.shape[0] == mask.shape[0]):
raise ValueError('First dimensions (number of samples) of t2s ({0}), '
's0 ({1}), and mask ({2}) do not '
'match'.format(t2s.shape[0], s0.shape[0],
mask.shape[0]))
"""
handwerkerd and others are working to "hypercomment" this function to
help everyone understand it sufficiently with the goal of eventually
modularizing the algorithm. This is still a work-in-process with later
sections not fully commented, some points of uncertainty are noted, and the
summary of the full algorithm is not yet complete.
There are sections of this code that calculate metrics that are used in
the decision tree for the selection process and other sections that
are part of the decision tree. Certain comments are prefaced with METRIC
and variable names to make clear which are metrics and others are prefaced
with SELECTION to make clear which are for applying metrics. METRICs tend
to be summary values that contain a signal number per component.
Note there are some variables that are calculated in one section of the code
that are later transformed into another metric that is actually part of a
selection criterion. This running list is an attempt to summarize
intermediate metrics vs the metrics that are actually used in decision
steps. For applied metrics that are made up of intermediate metrics defined
in earlier sections of the code, the constituent metrics are noted. More
metrics will be added to the applied metrics section as the commenting of
this function continues.
Intermediate Metrics: seldict['F_S0_clmaps'] seldict['F_R2_clmaps']
seldict['Br_clmaps_S0'] seldict['Br_clmaps_R2'] seldict['Z_maps']
dice_tbl countnoise
counts_FR2_Z tt_table mmix_kurt mmix_std
spr fproj_arr_val fdist
Rtz, Dz
Applied Metrics:
seldict['Rhos']
seldict['Kappas']
seldict['varex']
countsigFS0
countsigFR2
fz (a combination of multiple z-scored metrics: tt_table,
seldict['varex'], seldict['Kappa'], seldict['Rho'], countnoise,
mmix_kurt, fdist)
tt_table[:,0]
spz (z score of spr)
KRcut
"""
"""
If seldict exists, save it into a pickle file called compseldata.pklbz
that can be loaded directly into python for future analyses
If seldict=None, load it from the pre-saved pickle file to use for the
rest of this function
"""
if filecsdata:
import bz2
if seldict is not None:
LGR.info('Saving component selection data')
with bz2.BZ2File('compseldata.pklbz', 'wb') as csstate_f:
pickle.dump(seldict, csstate_f)
else:
try:
with bz2.BZ2File('compseldata.pklbz', 'rb') as csstate_f:
seldict = pickle.load(csstate_f)
except FileNotFoundError:
LGR.warning('Failed to load component selection data')
return None
"""
List of components
all_comps and acc_comps start out as an ordered list of the component numbers
all_comps is constant throughout the function.
acc_comps changes through his function as components are assigned to other
categories (i.e. components that are classified as rejected are removed
from acc_comps)
"""
midk = []
ign = []
all_comps = np.arange(len(seldict['Kappas']))
acc_comps = np.arange(len(seldict['Kappas']))
"""
If user has specified components to accept manually, just assign those
components to the accepted and rejected comp lists and end the function
"""
if manacc:
acc = sorted([int(vv) for vv in manacc.split(',')])
midk = []
rej = sorted(np.setdiff1d(all_comps, acc))
ign = []
return acc, rej, midk, ign # Add string for ign
"""
METRICS: countsigFS0 countsigFR2
F_S0_clmaps & F_R2_clmaps are the thresholded & binarized clustered maps of
significant fits for the separate S0 and R2 cross-echo models per component.
Since the values are 0 or 1, the countsig variables are a count of the
significant voxels per component.
The cluster size is a function of the # of voxels in the mask.
The cluster threshold is based on the # of echos acquired
"""
countsigFS0 = seldict['F_S0_clmaps'].sum(0)
countsigFR2 = seldict['F_R2_clmaps'].sum(0)
countnoise = np.zeros(len(all_comps))
"""
Make table of dice values
METRICS: dice_tbl
dice_FR2, dice_FS0 are calculated for each component and the concatenated
values are in dice_tbl
Br_clmaps_R2 and Br_clmaps_S0 are binarized clustered Z_maps.
The volume being clustered is the rank order indices of the absolute value
of the beta values for the fit between the optimally combined time series
and the mixing matrix (i.e. the lowest beta value is 1 and the highest is
the # of voxels).
The cluster size is a function of the # of voxels in the mask.
The cluster threshold are the voxels with beta ranks greater than
countsigFS0 or countsigFR2 (i.e. roughly the same number of voxels will be
in the countsig clusters as the ICA beta map clusters)
These dice values are the Dice-Sorenson index for the Br_clmap_?? and the
F_??_clmap.
If handwerkerd understands this correctly, if the voxels with the above
threshold F stats are clustered in the same voxels with the highest beta
values, then the dice coefficient will be 1. If the thresholded F or betas
aren't spatially clustered (i.e. the component map is less spatially smooth)
or the clusters are in different locations (i.e. voxels with high betas
are also noiser so they have lower F values), then the dice coefficients
will be lower
"""
dice_tbl = np.zeros([all_comps.shape[0], 2])
for comp_num in all_comps:
dice_FR2 = utils.dice(
utils.unmask(seldict['Br_clmaps_R2'][:, comp_num], mask)[t2s != 0],
seldict['F_R2_clmaps'][:, comp_num])
dice_FS0 = utils.dice(
utils.unmask(seldict['Br_clmaps_S0'][:, comp_num], mask)[t2s != 0],
seldict['F_S0_clmaps'][:, comp_num])
dice_tbl[comp_num, :] = [dice_FR2, dice_FS0] # step 3a here and above
dice_tbl[np.isnan(dice_tbl)] = 0
"""
Make table of noise gain
METRICS: countnoise, counts_FR2_Z, tt_table
(This is a bit confusing & is handwerkerd's attempt at making sense of this)
seldict['Z_maps'] is the Fisher Z normalized beta fits for the optimally
combined time series and the mixing matrix. Z_clmaps is a binarized cluster
of Z_maps with the cluster size based on the # of voxels and the cluster
threshold of 1.95. utils.andb is a sum of the True values in arrays so
comp_noise_sel is true for voxels where the Z values are greater than 1.95
but not part of a cluster of Z values that are greater than 1.95.
Spatially unclustered voxels with high Z values could be considerd noisy.
countnoise is the # of voxels per component where comp_noise_sel is true.
counts_FR2_Z is the number of voxels with Z values above the threshold
that are in clusters (signal) and the number outside of clusters (noise)
tt_table is a bit confusing. For each component, the first index is
some type of normalized, log10, signal/noise t statistic and the second is
the p value for the signal/noise t statistic (for the R2 model).
In general, these should be bigger t or have lower p values when most of
the Z values above threshold are inside clusters.
Because of the log10, values below 1 are negative, which is later used as
a threshold. It doesn't seem like the p values are ever used.
"""
tt_table = np.zeros([len(all_comps), 4])
counts_FR2_Z = np.zeros([len(all_comps), 2])
for comp_num in all_comps:
comp_noise_sel = utils.andb([
np.abs(seldict['Z_maps'][:, comp_num]) > 1.95,
seldict['Z_clmaps'][:, comp_num] == 0
]) == 2
countnoise[comp_num] = np.array(comp_noise_sel, dtype=np.int).sum()
noise_FR2_Z_mask = utils.unmask(comp_noise_sel, mask)[t2s != 0]
noise_FR2_Z = np.log10(
np.unique(seldict['F_R2_maps'][noise_FR2_Z_mask, comp_num]))
signal_FR2_Z_mask = utils.unmask(seldict['Z_clmaps'][:, comp_num],
mask)[t2s != 0] == 1
signal_FR2_Z = np.log10(
np.unique(seldict['F_R2_maps'][signal_FR2_Z_mask, comp_num]))
counts_FR2_Z[comp_num, :] = [len(signal_FR2_Z), len(noise_FR2_Z)]
ttest = stats.ttest_ind(signal_FR2_Z, noise_FR2_Z, equal_var=True)
# avoid DivideByZero RuntimeWarning
if signal_FR2_Z.size > 0 and noise_FR2_Z.size > 0:
mwu = stats.norm.ppf(
stats.mannwhitneyu(signal_FR2_Z, noise_FR2_Z)[1])
else:
mwu = -np.inf
tt_table[comp_num, 0] = np.abs(mwu) * ttest[0] / np.abs(ttest[0])
tt_table[comp_num, 1] = ttest[1]
tt_table[np.isnan(tt_table)] = 0
tt_table[np.isinf(tt_table[:, 0]),
0] = np.percentile(tt_table[~np.isinf(tt_table[:, 0]), 0], 98)
"""
Time series derivative kurtosis
METRICS: mmix_kurt and mmix_std
Take the derivative of the time series for each component in the ICA
mixing matrix and calculate the kurtosis & standard deviation.
handwerkerd thinks these metrics are later used to calculate measures
of time series spikiness and drift in the component time series.
"""
mmix_dt = (mmix[:-1, :] - mmix[1:, :])
mmix_kurt = stats.kurtosis(mmix_dt)
mmix_std = np.std(mmix_dt, axis=0)
"""
SELECTION #1 (prantikk labeled "Step 1")
Reject anything that is obviously an artifact
Obvious artifacts are components with Rho>Kappa or with more clustered,
significant voxels for the S0 model than the R2 model
"""
LGR.debug('Rejecting gross artifacts based on Rho/Kappa values and S0/R2 '
'counts')
rej = acc_comps[utils.andb(
[seldict['Rhos'] > seldict['Kappas'], countsigFS0 > countsigFR2]) > 0]
acc_comps = np.setdiff1d(acc_comps, rej)
"""
prantikk labeled "Step 2"
Compute 3-D spatial FFT of Beta maps to detect high-spatial
frequency artifacts
METRIC spr, fproj_arr_val, fdist
PSC is the mean centered beta map for each ICA component
The FFT is sequentially calculated across each dimension of PSC & the max
value is removed (probably the 0Hz bin). The maximum remaining frequency
magnitude along the z dimenions is calculated leaving a 2D matrix.
spr contains a count of the number of frequency bins in the 2D matrix where
the frequency magnitude is greater than 4* the maximum freq in the matrix.
spr is later z-normed across components into spz and this is actually used
as a selection metric.
handwerkerd interpretation: spr is bigger the more values of the fft are
>1/4 the max. Thus, if you assume the highest mag bins are low frequency, &
all components have roughly the same low freq power (i.e. a brain-shaped
blob), then spr will be bigger the more high frequency bins have magnitudes
that are more than 1/4 of the low frequency bins.
fproj_arr_val is a flattened 1D vector of the 2D max projection fft
of each component. This seems to be later used in an SVM to train on
this value for rejected components to classify some remaining n_components
as midk
Note: fproj_arr is created here and is a ranked list of FFT values, but is
not used anywhere in the code. Was fproj_arr_val supposed to contain this
ranking?
fdist isn't completely clear to handwerkerd yet but it looks like the fit of
the fft of the spatial map to a Gaussian distribution. If so, then the
worse the fit, the more high frequency power would be in the component
"""
LGR.debug(
'Computing 3D spatial FFT of beta maps to detect high-spatial frequency artifacts'
)
# spatial information is important so for NIFTI we convert back to 3D space
dim1 = np.prod(check_niimg(ref_img).shape[:2])
fproj_arr = np.zeros([dim1, len(all_comps)])
fproj_arr_val = np.zeros([dim1, len(all_comps)])
spr = []
fdist = []
for comp_num in all_comps:
# convert data back to 3D array
tproj = utils.new_nii_like(
ref_img,
utils.unmask(seldict['PSC'], mask)[:, comp_num]).get_data()
fproj = np.fft.fftshift(np.abs(np.fft.rfftn(tproj)))
fproj_z = fproj.max(axis=-1)
fproj[fproj == fproj.max()] = 0
spr.append(np.array(fproj_z > fproj_z.max() / 4, dtype=np.int).sum())
fproj_arr[:, comp_num] = stats.rankdata(fproj_z.flatten())
fproj_arr_val[:, comp_num] = fproj_z.flatten()
fprojr = np.array([fproj, fproj[:, :, ::-1]]).max(0)
fdist.append(
np.max([
utils.fitgaussian(fproj.max(jj))[3:].max()
for jj in range(fprojr.ndim)
]))
if type(fdist) is not np.ndarray:
fdist = np.array(fdist)
spr = np.array(spr)
# import ipdb; ipdb.set_trace()
"""
prantikk labelled Step 3
Create feature space of component properties
METRIC fz, spz, Rtz, Dz
fz is matrix of multiple other metrics described above and calculated
in this section. Most are all of these have one number per component and
they are z-scored across components
Attempted explanations in order:
Tz: The z-scored t statistics of the spatial noisiness metric in tt_table
Vz: The z-scored the natural log of the non-normalized variance explained
of each component
Ktz: The z-scored natural log of the Kappa values
(the '/ 2' does not seem necessary beacuse it will be removed by z-scoring)
KRr: The z-scored ratio of the natural log of Kappa / nat log of Rho
(unclear why sometimes using stats.zcore and other times writing the eq out)
cnz: The z-scored measure of a noisy voxel count where the noisy voxels are
the voxels with large beta estimates, but aren't part of clusters
Rz: z-scored rho values (why aren't this log scaled, like kappa in Ktz?)
mmix_kurt: Probably a rough measure of the spikiness of each component's
time series in the ICA mixing matrix
fdist_z: z-score of fdist, which is probably a measure of high freq info
in the spatial FFT of the components (with lower being more high freq?)
NOT in fz:
spz: Z-scored measure probably of how much high freq is in the data. Larger
values mean more bins of the FFT have over 1/4 the power of the maximum
bin (read about spr above for more info)
Rtz: Z-scored natural log of the Rho values
Dz: Z-scored Fisher Z transformed dice values of the overlap between
clusters for the F stats and clusters of the ICA spatial beta maps with
roughly the same number of voxels as in the clustered F maps.
Dz saves this for the R2 model, there are also Dice coefs for the S0
model in dice_tbl
"""
LGR.debug('Creating feature space of component properties')
fdist_pre = fdist.copy()
fdist_pre[fdist > np.median(fdist) * 3] = np.median(fdist) * 3
fdist_z = (fdist_pre - np.median(fdist_pre)) / fdist_pre.std() # not z
spz = stats.zscore(spr)
Tz = stats.zscore(tt_table[:, 0])
varex_log = np.log(seldict['varex'])
Vz = stats.zscore(varex_log)
Rz = stats.zscore(seldict['Rhos'])
Ktz = stats.zscore(np.log(seldict['Kappas']) / 2)
# Rtz = stats.zscore(np.log(seldict['Rhos']) / 2)
KRr = stats.zscore(np.log(seldict['Kappas']) / np.log(seldict['Rhos']))
cnz = stats.zscore(countnoise)
Dz = stats.zscore(np.arctanh(dice_tbl[:, 0] + 0.001))
fz = np.array([Tz, Vz, Ktz, KRr, cnz, Rz, mmix_kurt, fdist_z])
"""
METRICS Kcut, Rcut, KRcut, KRcutguesses, Khighelbowval
Step 3: Make initial guess of where BOLD components are and use DBSCAN
to exclude noise components and find a sample set of 'good' components
"""
LGR.debug('Making initial guess of BOLD components')
# The F threshold for the echo fit (based on the # of echos) for p<0.05
# p<0.025, and p<0.001 (Confirm this is accurate since the function
# contains a lookup table rather than a calculation)
F05, F025, F01 = utils.getfbounds(n_echos)
# epsmap is [index,level of overlap with dicemask,
# number of high Rho components]
epsmap = []
Rhos_sorted = np.array(sorted(seldict['Rhos']))[::-1]
"""
Make an initial guess as to number of good components based on
consensus of control points across Rhos and Kappas
For terminology later, typically getelbow _aggr > _mod > _cons
though this might not be universally true. A more "inclusive" threshold
has a lower kappa since that means more components are above that thresh
and are likely to be accepted. For Rho, a more "inclusive" threshold is
higher since that means fewer components will be rejected based on rho.
KRcut seems weird to handwerkerd. I see that the thresholds are slightly
shifted for kappa & rho later in the code, but why would we ever want to
set a common threhsold reference point for both? These are two different
elbows on two different data sets.
"""
KRcutguesses = [
getelbow_mod(seldict['Rhos']),
getelbow_cons(seldict['Rhos']),
getelbow_aggr(seldict['Rhos']),
getelbow_mod(seldict['Kappas']),
getelbow_cons(seldict['Kappas']),
getelbow_aggr(seldict['Kappas'])
]
KRcut = np.median(KRcutguesses)
"""
Also a bit weird to handwerkerd. This is the 75th percentile of Kappa F
stats of the components with the 3 elbow selection criteria and the
F states for 3 significance thresholds based on the # of echos.
This is some type of way to get a significance criterion for a component
fit, but it's include why this specific criterion is useful.
"""
Khighelbowval = stats.scoreatpercentile([
getelbow_mod(seldict['Kappas'], return_val=True),
getelbow_cons(seldict['Kappas'], return_val=True),
getelbow_aggr(seldict['Kappas'], return_val=True)
] + list(utils.getfbounds(n_echos)),
75,
interpolation_method='lower')
"""
Default to the most inclusive kappa threshold (_cons) unless:
1. That threshold is more than twice the median of Kappa & Rho thresholds
2. and the moderate elbow is more inclusive than a p=0.01
handwerkerd: This actually seems like a way to avoid using the theoretically
most liberal threshold only when there was a bad estimate and _mod is
is more inclusive. My one concern is that it's an odd way to test that
the _mod elbow is any better. Why not at least see if _mod < _cons?
prantikk's orig comment for this section is:
"only use exclusive when inclusive is extremely inclusive - double KRcut"
"""
cond1 = getelbow_cons(seldict['Kappas']) > KRcut * 2
cond2 = getelbow_mod(seldict['Kappas'], return_val=True) < F01
if cond1 and cond2:
Kcut = getelbow_mod(seldict['Kappas'], return_val=True)
else:
Kcut = getelbow_cons(seldict['Kappas'], return_val=True)
"""
handwerkerd: The goal seems to be to maximize the rejected components
based on the rho cut by defaulting to a lower Rcut value. Again, if
that is the goal, why not just test if _mod < _cons?
prantikk's orig comment for this section is:
only use inclusive when exclusive is extremely exclusive - half KRcut
(remember for Rho inclusive is higher, so want both Kappa and Rho
to defaut to lower)
"""
if getelbow_cons(seldict['Rhos']) > KRcut * 2:
Rcut = getelbow_mod(seldict['Rhos'], return_val=True)
# for above, consider something like:
# min([getelbow_mod(Rhos,True),sorted(Rhos)[::-1][KRguess] ])
else:
Rcut = getelbow_cons(seldict['Rhos'], return_val=True)
# Rcut should never be higher than Kcut (handwerkerd: not sure why)
if Rcut > Kcut:
Kcut = Rcut
# KRelbow has a 2 for components that are above the Kappa accept threshold
# and below the rho reject threshold
KRelbow = utils.andb([seldict['Kappas'] > Kcut, seldict['Rhos'] < Rcut])
"""
Make guess of Kundu et al 2011 plus remove high frequencies,
generally high variance, and high variance given low Kappa
the first index of tt_table is a t static of a what handwerkerd thinks
is a spatial noise metric. Since log10 of these values are taken the >0
threshold means the metric is >1. tt_lim seems to be a fairly aggressive
percentile that is then divided by 3.
"""
tt_lim = stats.scoreatpercentile(
tt_table[tt_table[:, 0] > 0, 0], 75, interpolation_method='lower') / 3
"""
KRguess is a list of components to potentially accept. it starts with a
list of components that cross the Kcut and Rcut threshold and weren't
previously rejected for other reasons. From that list, it removes more
components based on several additional criteria:
1. tt_table less than the tt_lim threshold (spatial noisiness metric)
2. spz (a z-scored probably high spatial freq metric) >1
3. Vz (a z-scored variance explained metric) >2
4. If both (seems to be if a component has a relatively high variance
the acceptance threshold for Kappa values is doubled):
A. The variance explained is greater than half the KRcut highest
variance component
B. Kappa is less than twice Kcut
"""
temp = all_comps[utils.andb([
seldict['varex'] > 0.5 * sorted(seldict['varex'])[::-1][int(KRcut)],
seldict['Kappas'] < 2 * Kcut
]) == 2]
KRguess = np.setdiff1d(
np.setdiff1d(all_comps[KRelbow == 2], rej),
np.union1d(
all_comps[tt_table[:, 0] < tt_lim],
np.union1d(np.union1d(all_comps[spz > 1], all_comps[Vz > 2]),
temp)))
guessmask = np.zeros(len(all_comps))
guessmask[KRguess] = 1
"""
Throw lower-risk bad components out based on 3 criteria all being true:
1. tt_table (a spatial noisiness metric) <0
2. A components variance explains is greater than the median variance
explained
3. The component index is greater than the KRcut index. Since the
components are sorted by kappa, this is another kappa thresholding)
"""
rejB = acc_comps[utils.andb([
tt_table[acc_comps, 0] < 0, seldict['varex'][acc_comps] > np.median(
seldict['varex']), acc_comps > KRcut
]) == 3]
rej = np.union1d(rej, rejB)
# adjust acc_comps again to only contain the remaining non-rejected components
acc_comps = np.setdiff1d(acc_comps, rej)
"""
This is where handwerkerd has paused in hypercommenting the function.
"""
LGR.debug('Using DBSCAN to find optimal set of "good" BOLD components')
for ii in range(20000):
eps = .005 + ii * .005
db = DBSCAN(eps=eps, min_samples=3).fit(fz.T)
# it would be great to have descriptive names, here
# DBSCAN found at least three non-noisy clusters
cond1 = db.labels_.max() > 1
# DBSCAN didn't detect more classes than the total # of components / 6
cond2 = db.labels_.max() < len(all_comps) / 6
# TODO: confirm if 0 is a special label for DBSCAN
# my intuition here is that we're confirming DBSCAN labelled previously
# rejected components as noise (i.e., no overlap between `rej` and
# labelled DBSCAN components)
cond3 = np.intersect1d(rej, all_comps[db.labels_ == 0]).shape[0] == 0
# DBSCAN labelled less than half of the total components as noisy
cond4 = np.array(db.labels_ == -1, dtype=int).sum() / float(
len(all_comps)) < .5
if cond1 and cond2 and cond3 and cond4:
epsmap.append([
ii,
utils.dice(guessmask, db.labels_ == 0),
np.intersect1d(
all_comps[db.labels_ == 0],
all_comps[seldict['Rhos'] > getelbow_mod(
Rhos_sorted, return_val=True)]).shape[0]
])
db = None
epsmap = np.array(epsmap)
LGR.debug('Found DBSCAN solutions for {}/20000 eps resolutions'.format(
len(epsmap)))
group0 = []
dbscanfailed = False
if len(epsmap) != 0:
# Select index that maximizes Dice with guessmask but first
# minimizes number of higher Rho components
ii = int(
epsmap[np.argmax(epsmap[epsmap[:, 2] == np.min(epsmap[:, 2]),
1], 0), 0])
LGR.debug('Component selection tuning: {:.05f}'.format(
epsmap[:, 1].max()))
db = DBSCAN(eps=.005 + ii * .005, min_samples=3).fit(fz.T)
acc_comps = all_comps[db.labels_ == 0]
acc_comps = np.setdiff1d(acc_comps, rej)
acc_comps = np.setdiff1d(
acc_comps, acc_comps[acc_comps > len(all_comps) - len(rej)])
group0 = acc_comps.copy()
group_n1 = all_comps[db.labels_ == -1]
to_clf = np.setdiff1d(all_comps, np.union1d(acc_comps, rej))
if len(group0) == 0 or len(group0) < len(KRguess) * .5:
dbscanfailed = True
LGR.debug('DBSCAN guess failed; using elbow guess method instead')
temp = all_comps[utils.andb([
seldict['varex'] > 0.5 *
sorted(seldict['varex'])[::-1][int(KRcut)], seldict['Kappas'] < 2 *
Kcut
]) == 2]
acc_comps = np.setdiff1d(
np.setdiff1d(all_comps[KRelbow == 2], rej),
np.union1d(
all_comps[tt_table[:, 0] < tt_lim],
np.union1d(np.union1d(all_comps[spz > 1], all_comps[Vz > 2]),
temp)))
group0 = acc_comps.copy()
group_n1 = []
to_clf = np.setdiff1d(all_comps, np.union1d(group0, rej))
if len(group0) < 2 or (len(group0) < 4
and float(len(rej)) / len(group0) > 3):
LGR.warning('Extremely limited reliable BOLD signal space! '
'Not filtering components beyond BOLD/non-BOLD guesses.')
midkfailed = True
min_acc = np.array([])
if len(group0) != 0:
# For extremes, building in a 20% tolerance
toacc_hi = np.setdiff1d(
all_comps[utils.andb([
fdist <= np.max(fdist[group0]), seldict['Rhos'] < F025,
Vz > -2
]) == 3], np.union1d(group0, rej))
min_acc = np.union1d(group0, toacc_hi)
to_clf = np.setdiff1d(all_comps, np.union1d(min_acc, rej))
else:
toacc_hi = []
min_acc = []
diagstep_keys = [
'Rejected components', 'Kappa-Rho cut point', 'Kappa cut point',
'Rho cut point', 'DBSCAN failed to converge',
'Mid-Kappa failed (limited BOLD signal)', 'Kappa-Rho guess',
'min_acc', 'toacc_hi'
]
diagstep_vals = [
list(rej), KRcut, Kcut, Rcut, dbscanfailed, midkfailed,
list(KRguess),
list(min_acc),
list(toacc_hi)
]
with open('csstepdata.json', 'w') as ofh:
json.dump(dict(zip(diagstep_keys, diagstep_vals)),
ofh,
indent=4,
sort_keys=True,
default=str)
return list(sorted(min_acc)), list(sorted(rej)), [], list(
sorted(to_clf))
# Find additional components to reject based on Dice - doing this here
# since Dice is a little unstable, need to reference group0
rej_supp = []
dice_rej = False
if not dbscanfailed and len(rej) + len(group0) < 0.75 * len(all_comps):
dice_rej = True
temp = all_comps[dice_tbl[all_comps, 0] <= dice_tbl[all_comps, 1]]
rej_supp = np.setdiff1d(np.setdiff1d(np.union1d(rej, temp), group0),
group_n1)
rej = np.union1d(rej, rej_supp)
# Temporal features
# larger is worse - spike
mmix_kurt_z = (mmix_kurt -
mmix_kurt[group0].mean()) / mmix_kurt[group0].std()
# smaller is worse - drift
mmix_std_z = -1 * (
(mmix_std - mmix_std[group0].mean()) / mmix_std[group0].std())
mmix_kurt_z_max = np.max([mmix_kurt_z, mmix_std_z], 0)
"""
Step 2: Classifiy midk and ignore using separate SVMs for
different variance regimes
# To render hyperplane:
min_x = np.min(spz2);max_x=np.max(spz2)
# plotting separating hyperplane
ww = clf_.coef_[0]
aa = -ww[0] / ww[1]
# make sure the next line is long enough
xx = np.linspace(min_x - 2, max_x + 2)
yy = aa * xx - (clf_.intercept_[0]) / ww[1]
plt.plot(xx, yy, '-')
"""
LGR.debug('Attempting to classify midk components')
# Tried getting rid of accepting based on SVM altogether,
# now using only rejecting
toacc_hi = np.setdiff1d(
all_comps[utils.andb([
fdist <= np.max(fdist[group0]), seldict['Rhos'] < F025, Vz > -2
]) == 3], np.union1d(group0, rej))
temp = utils.andb([
spz < 1, Rz < 0, mmix_kurt_z_max < 5, Dz > -1, Tz > -1, Vz < 0,
seldict['Kappas'] >= F025,
fdist < 3 * np.percentile(fdist[group0], 98)
]) == 8
toacc_lo = np.intersect1d(to_clf, all_comps[temp])
midk_clf, clf_ = do_svm(fproj_arr_val[:, np.union1d(group0, rej)].T,
[0] * len(group0) + [1] * len(rej),
fproj_arr_val[:, to_clf].T,
svmtype=2)
midk = np.setdiff1d(
to_clf[utils.andb([
midk_clf == 1,
seldict['varex'][to_clf] > np.median(seldict['varex'][group0])
]) == 2], np.union1d(toacc_hi, toacc_lo))
# only use SVM to augment toacc_hi only if toacc_hi isn't already
# conflicting with SVM choice
if len(
np.intersect1d(
to_clf[utils.andb([midk_clf == 1, Vz[to_clf] > 0]) == 2],
toacc_hi)) == 0:
svm_acc_fail = True
toacc_hi = np.union1d(toacc_hi, to_clf[midk_clf == 0])
else:
svm_acc_fail = False
"""
Step 3: Compute variance associated with low T2* areas
(e.g. draining veins and low T2* areas)
# To write out veinmask
veinout = np.zeros(t2s.shape)
veinout[t2s!=0] = veinmaskf
utils.filewrite(veinout, 'veinmaskf', ref_img)
veinBout = utils.unmask(veinmaskB, mask)
utils.filewrite(veinBout, 'veins50', ref_img)
"""
LGR.debug('Computing variance associated with low T2* areas (e.g., '
'draining veins)')
tsoc_B_Zcl = np.zeros(seldict['tsoc_B'].shape)
tsoc_B_Zcl[seldict['Z_clmaps'] != 0] = np.abs(
seldict['tsoc_B'])[seldict['Z_clmaps'] != 0]
sig_B = [
stats.scoreatpercentile(tsoc_B_Zcl[tsoc_B_Zcl[:, ii] != 0, ii], 25)
if len(tsoc_B_Zcl[tsoc_B_Zcl[:, ii] != 0, ii]) != 0 else 0
for ii in all_comps
]
sig_B = np.abs(seldict['tsoc_B']) > np.tile(
sig_B, [seldict['tsoc_B'].shape[0], 1])
veinmask = utils.andb([
t2s < stats.scoreatpercentile(
t2s[t2s != 0], 15, interpolation_method='lower'), t2s != 0
]) == 2
veinmaskf = veinmask[mask]
veinR = np.array(sig_B[veinmaskf].sum(0),
dtype=float) / sig_B[~veinmaskf].sum(0)
veinR[np.isnan(veinR)] = 0
veinc = np.union1d(rej, midk)
rej_veinRZ = ((veinR - veinR[veinc].mean()) / veinR[veinc].std())[veinc]
rej_veinRZ[rej_veinRZ < 0] = 0
rej_veinRZ[countsigFR2[veinc] > np.array(veinmaskf, dtype=int).sum()] = 0
t2s_lim = [
stats.scoreatpercentile(t2s[t2s != 0],
50,
interpolation_method='lower'),
stats.scoreatpercentile(
t2s[t2s != 0], 80, interpolation_method='lower') / 2
]
phys_var_zs = []
for t2sl_i in range(len(t2s_lim)):
t2sl = t2s_lim[t2sl_i]
veinW = sig_B[:, veinc] * np.tile(rej_veinRZ, [sig_B.shape[0], 1])
veincand = utils.unmask(
utils.andb([
s0[t2s != 0] < np.median(s0[t2s != 0]), t2s[t2s != 0] < t2sl
]) >= 1, t2s != 0)[mask]
veinW[~veincand] = 0
invein = veinW.sum(
axis=1)[(utils.unmask(veinmaskf, mask) *
utils.unmask(veinW.sum(axis=1) > 1, mask))[mask]]
minW = 10 * (np.log10(invein).mean()) - 1 * 10**(
np.log10(invein).std())
veinmaskB = veinW.sum(axis=1) > minW
tsoc_Bp = seldict['tsoc_B'].copy()
tsoc_Bp[tsoc_Bp < 0] = 0
vvex = np.array([
(tsoc_Bp[veinmaskB, ii]**2.).sum() / (tsoc_Bp[:, ii]**2.).sum()
for ii in all_comps
])
group0_res = np.intersect1d(KRguess, group0)
phys_var_zs.append(
(vvex - vvex[group0_res].mean()) / vvex[group0_res].std())
veinBout = utils.unmask(veinmaskB, mask)
utils.filewrite(veinBout.astype(float), 'veins_l%i' % t2sl_i, ref_img)
# Mask to sample veins
phys_var_z = np.array(phys_var_zs).max(0)
Vz2 = (varex_log - varex_log[group0].mean()) / varex_log[group0].std()
"""
Step 4: Learn joint TE-dependence spatial and temporal models to move
remaining artifacts to ignore class
"""
LGR.debug(
'Learning joint TE-dependence spatial/temporal models to ignore remaining artifacts'
)
to_ign = []
minK_ign = np.max([F05, getelbow_cons(seldict['Kappas'], return_val=True)])
newcest = len(group0) + len(
toacc_hi[seldict['Kappas'][toacc_hi] > minK_ign])
phys_art = np.setdiff1d(
all_comps[utils.andb([phys_var_z > 3.5, seldict['Kappas'] < minK_ign])
== 2], group0)
rank_diff = stats.rankdata(phys_var_z) - stats.rankdata(seldict['Kappas'])
phys_art = np.union1d(
np.setdiff1d(
all_comps[utils.andb(
[phys_var_z > 2, rank_diff > newcest / 2, Vz2 > -1]) == 3],
group0), phys_art)
# Want to replace field_art with an acf/SVM based approach
# instead of a kurtosis/filter one
field_art = np.setdiff1d(
all_comps[utils.andb(
[mmix_kurt_z_max > 5, seldict['Kappas'] < minK_ign]) == 2], group0)
temp = (stats.rankdata(mmix_kurt_z_max) -
stats.rankdata(seldict['Kappas'])) > newcest / 2
field_art = np.union1d(
np.setdiff1d(
all_comps[utils.andb([
mmix_kurt_z_max > 2, temp, Vz2 > 1, seldict['Kappas'] < F01
]) == 4], group0), field_art)
temp = seldict['Rhos'] > np.percentile(seldict['Rhos'][group0], 75)
field_art = np.union1d(
np.setdiff1d(
all_comps[utils.andb([mmix_kurt_z_max > 3, Vz2 > 3, temp]) == 3],
group0), field_art)
field_art = np.union1d(
np.setdiff1d(
all_comps[utils.andb([mmix_kurt_z_max > 5, Vz2 > 5]) == 2],
group0), field_art)
misc_art = np.setdiff1d(
all_comps[utils.andb([(stats.rankdata(Vz) -
stats.rankdata(Ktz)) > newcest /
2, seldict['Kappas'] < Khighelbowval]) == 2],
group0)
ign_cand = np.unique(list(field_art) + list(phys_art) + list(misc_art))
midkrej = np.union1d(midk, rej)
to_ign = np.setdiff1d(list(ign_cand), midkrej)
toacc = np.union1d(toacc_hi, toacc_lo)
acc_comps = np.setdiff1d(np.union1d(acc_comps, toacc),
np.union1d(to_ign, midkrej))
ign = np.setdiff1d(all_comps, list(acc_comps) + list(midk) + list(rej))
orphan = np.setdiff1d(
all_comps,
list(acc_comps) + list(to_ign) + list(midk) + list(rej))
# Last ditch effort to save some transient components
if not strict_mode:
Vz3 = (varex_log -
varex_log[acc_comps].mean()) / varex_log[acc_comps].std()
temp = utils.andb([
seldict['Kappas'] > F05, seldict['Rhos'] < F025,
seldict['Kappas'] > seldict['Rhos'], Vz3 <= -1, Vz3 > -3,
mmix_kurt_z_max < 2.5
])
acc_comps = np.union1d(acc_comps,
np.intersect1d(orphan, all_comps[temp == 6]))
ign = np.setdiff1d(all_comps, list(acc_comps) + list(midk) + list(rej))
orphan = np.setdiff1d(
all_comps,
list(acc_comps) + list(to_ign) + list(midk) + list(rej))
if savecsdiag:
diagstep_keys = [
'Rejected components', 'Kappa-Rho cut point', 'Kappa cut',
'Rho cut', 'DBSCAN failed to converge', 'Kappa-Rho guess',
'Dice rejected', 'rej_supp', 'to_clf', 'Mid-kappa components',
'svm_acc_fail', 'toacc_hi', 'toacc_lo', 'Field artifacts',
'Physiological artifacts', 'Miscellaneous artifacts', 'acc_comps',
'Ignored components'
]
diagstep_vals = [
list(rej),
KRcut.item(),
Kcut.item(),
Rcut.item(), dbscanfailed,
list(KRguess), dice_rej,
list(rej_supp),
list(to_clf),
list(midk), svm_acc_fail,
list(toacc_hi),
list(toacc_lo),
list(field_art),
list(phys_art),
list(misc_art),
list(acc_comps),
list(ign)
]
with open('csstepdata.json', 'w') as ofh:
json.dump(dict(zip(diagstep_keys, diagstep_vals)),
ofh,
indent=4,
sort_keys=True,
default=str)
allfz = np.array([Tz, Vz, Ktz, KRr, cnz, Rz, mmix_kurt, fdist_z])
np.savetxt('csdata.txt', allfz)
return list(sorted(acc_comps)), list(sorted(rej)), list(
sorted(midk)), list(sorted(ign))
def testComplexOps(self):
for dtype in self.complex_types:
self._assertOpOutputMatchesExpected(
math_ops.acosh,
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype),
expected=np.arccosh(
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.asinh,
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype),
expected=np.arcsinh(
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.atanh,
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype),
expected=np.arctanh(
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.cosh,
np.array([1j, 2 - 3j, 3, 4 + 2j], dtype=dtype),
expected=np.cosh(np.array([1j, 2 - 3j, 3, 4 + 2j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.sinh,
np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype),
expected=np.sinh(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.exp,
np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype),
expected=np.exp(np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.expm1,
np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype),
expected=np.expm1(np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype)),
rtol=1e-6,
atol=1e-6)
# For real part close to zero, or imaginary part close to a multiple of
# pi.
self._assertOpOutputMatchesExpected(
math_ops.expm1,
np.array([[1e-11 + 1j, -1e-11 - 1j, 1. + 1e-11j,
-1. - 1e-11j, 1e-13j + 1e-13j]], dtype=dtype),
# TODO(srvasude): Use numpy as the source of truth after we depend on
# latest numpy with this pull request:
# https://github.com/numpy/numpy/pull/15110.
# The numbers below were generated by scipy.special.expm1.
expected=np.array([[
-4.59697694e-01+8.41470985e-01j,
-4.59697694e-01-8.41470985e-01j,
1.71828183e+00+2.71828183e-11j,
-6.32120559e-01-3.67879441e-12j,
-2.00000000e-26+2.00000000e-13j]], dtype=dtype),
rtol=1e-09,
atol=1e-20)
self._assertOpOutputMatchesExpected(
math_ops.reciprocal,
np.array([[1, 2j, 2 + 3j]], dtype=dtype),
expected=1.0 / np.array([[1, 2j, 2 + 3j]], dtype=dtype))
self._assertOpOutputMatchesExpected(
math_ops.log,
np.array([[5j, 3 - 2j]], dtype=dtype),
expected=np.log(np.array([[5j, 3 - 2j]], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.sin,
np.array([[5j, 3 - 2j]], dtype=dtype),
expected=np.sin(np.array([[5j, 3 - 2j]], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.cos,
np.array([[5j, 3 - 2j]], dtype=dtype),
expected=np.cos(np.array([[5j, 3 - 2j]], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.log1p,
np.array([[1e-14, 1e-15j, 0.6 - 0.3j]], dtype=dtype),
expected=np.log1p(
np.array([[1e-14, 1e-15j, 0.6 - 0.3j]], dtype=dtype)),
rtol=1e-4,
atol=1e-6)
val = np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)
self._assertOpOutputMatchesExpected(
math_ops.rsqrt, val, expected=1 / np.sqrt(val))
self._assertOpOutputMatchesExpected(
math_ops.sigmoid, val, expected=1 / (1 + np.exp(-val)))
self._assertOpOutputMatchesExpected(
math_ops.sqrt, val, expected=np.sqrt(val))
self._assertOpOutputMatchesExpected(
math_ops.tanh,
np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype),
expected=np.tanh(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.tan,
np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype),
expected=np.tan(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)))
ctypes = {np.complex64: np.float32, np.complex128: np.float64}
self._assertOpOutputMatchesExpected(
math_ops.abs,
np.array([[3 - 4j, -1j, np.inf]], dtype=dtype),
expected=np.array([[5, 1, np.inf]], dtype=ctypes[dtype]))
self._assertOpOutputMatchesExpected(
math_ops.negative,
np.array([[-1 + 2j, -3j]], dtype=dtype),
expected=np.array([[1 - 2j, 3j]], dtype=dtype))
self._assertOpOutputMatchesExpected(
math_ops.square,
np.array([[-2 - 3j, 3 + 4j, 5j]], dtype=dtype),
expected=np.array([[-2 - 3j, 3 + 4j, 5j]], dtype=dtype)**2)
self._assertOpOutputMatchesExpected(
array_ops.zeros_like,
np.array([[4j, 3 - 2j], [2, -1j]], dtype=dtype),
expected=np.array([[0, 0], [0, 0]], dtype=dtype))
self._assertOpOutputMatchesExpected(
array_ops.ones_like,
np.array([[-4j, 3 + 2j], [2, -1j]], dtype=dtype),
expected=np.array([[1, 1], [1, 1]], dtype=dtype))
self._assertOpOutputMatchesExpected(
math_ops.angle,
np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
expected=np.angle(np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.conj,
np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
expected=np.array([1 - 3j, -4 - 7j, 2.7, 3j], dtype=dtype))
self._assertOpOutputMatchesExpected(
math_ops.imag,
np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
expected=np.array([3, 7, 0, -3], dtype=ctypes[dtype]))
self._assertOpOutputMatchesExpected(
math_ops.real,
np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
expected=np.array([1, -4, 2.7, 0], dtype=ctypes[dtype]))
def fyi(x):
return np.arctanh(x)
def unscale(serie):
return np.arctanh(serie)
class Atanh(Activation):
op = 'Atanh'
version = 'opset4'
operation = staticmethod(lambda x: np.arctanh(x))
def _eval_model(model_id, perm=None):
# set vars from model
mm = _global_meld[model_id]
_R = mm._R
# Calculate R
R = []
ind_b = np.arange(len(mm._groups))
# loop over group vars
ind = {}
for i, k in enumerate(mm._groups[ind_b]):
# grab the A and M
A = mm._A[k]
M = mm._M[k]
# gen a perm for that subj
if perm is None:
ind[k] = np.arange(len(A))
else:
ind[k] = perm[k]
if perm is None and mm._R is not None:
# reuse R we already calculated
R.append(mm._R[ind_b[i]])
else:
# calc the correlation
#R.append(np.dot(A.T,M[ind[k]].copy()))
#R.append(blockwise_dot(M[:][ind[k]].T, A).T)
R.append(blockwise_dot(M[ind[k]].T, A).T)
# turn R into array
R_nocat = np.array(R)
# zero invariant features
feat_mask = np.isnan(R)
R_nocat[feat_mask] = 0.0
# turn to Z
# make sure we are not 1/-1
R_nocat[R_nocat > .9999] = .9999
R_nocat[R_nocat < -.9999] = -.9999
R_nocat = np.arctanh(R_nocat)
if mm._do_tfce:
# run TFCE
Ztemp = R_to_tfce(R_nocat,
mm._dep_mask,
connectivity=mm._connectivity,
shape=mm._feat_shape,
dt=mm._dt,
E=mm._E,
H=mm._H)
else:
# just use the current Z without TFCE
Ztemp = R_nocat
# pick only stable features
# NOTE: Ztemp is no longer R, it's either TFCE or Z
Rtbr = pick_stable_features(Ztemp, nboot=mm._feat_nboot)
# actually use the TFCE for SVD
R_nocat = Ztemp
# apply the thresh
stable_ind = Rtbr < mm._feat_thresh
stable_ind = stable_ind.reshape((stable_ind.shape[0], -1))
# zero out non-stable
feat_mask[:, ~stable_ind] = True
R_nocat[:, ~stable_ind] = 0.0
# save R before concatenating if not a permutation
if perm is None and _R is None:
_R = R_nocat.copy()
# concatenate R for SVD
# NOTE: It's really Z or TFCE now
R = np.concatenate([R_nocat[i] for i in range(len(R_nocat))])
# perform svd
U, s, Vh = np.linalg.svd(R, full_matrices=False)
# fix near zero vals from SVD
Vh[np.abs(Vh) < (.00000001 * mm._dt)] = 0.0
s[np.abs(s) < .00000001] = 0.0
# calc prop of variance accounted for
if mm._ss is None:
# _ss = np.sqrt((s*s).sum())
_ss = s.sum()
else:
_ss = mm._ss
# ss /= ss.sum()
ss = s
# set up lmer
O = None
lmer = None
if mm._mer is None:
O = [mm._O[i].copy() for i in ind_b]
lmer = LMER(mm._formula_str,
np.concatenate(O),
factors=mm._factors,
resid_formula_str=mm._resid_formula_str,
**mm._lmer_opts)
mer = None
else:
mer = mm._mer
# loop over LVs performing LMER
res = []
for i in range(len(Vh)):
if ss[i] <= 0.0:
# print 'skipped ',str(i)
continue
# flatten then weigh features via dot product
Dw = np.concatenate([ #np.dot(mm._D[k][ind[k]].copy(),Vh[i])
#blockwise_dot(mm._D[k][:][ind[k]], Vh[i])
blockwise_dot(mm._D[k][ind[k]], Vh[i])
for g, k in enumerate(mm._groups[ind_b])
])
# run the main model
if mer is None:
# run the model for the first time and save it
res.append(lmer.run(vals=Dw))
mer = lmer._ms
else:
# use the saved model and just refit it for speed
mer = r['refit'](mer, FloatVector(Dw))
df = r['data.frame'](r_coef(r['summary'](mer)))
rows = list(r['row.names'](df))
new_tvals = np.rec.fromarrays([[tv]
for tv in tuple(df.rx2('t.value'))],
names=','.join(rows))
new_ll = float(r['logLik'](mer)[0])
res.append((new_tvals, np.array([new_ll])))
if len(res) == 0:
# must make dummy data
if lmer is None:
O = [mm._O[i].copy() for i in ind_b]
# if boot is not None:
# # replace the group
# for i, k in enumerate(mm._groups):
# O[i][mm._re_group] = k
lmer = LMER(mm._formula_str,
np.concatenate(O),
factors=mm._factors,
resid_formula_str=mm._resid_formula_str,
**mm._lmer_opts)
Dw = np.random.randn(len(np.concatenate(O)))
temp_t, temp_ll = lmer.run(vals=Dw)
for n in temp_t.dtype.names:
temp_t[n] = 0.0
temp_ll[0] = 0.0
res.append((temp_t, temp_ll))
# must make ss, too
ss = np.array([1.0])
# print "perm fail"
# pull out data from all the components
tvals, log_likes = zip(*res)
tvals = np.concatenate(tvals)
log_likes = np.concatenate(log_likes)
# recombine and scale the tvals across components
ts = np.rec.fromarrays(
[
np.dot(tvals[k], ss[ss > 0.0] / _ss) # /(ss>0.).sum()
for k in tvals.dtype.names
],
names=','.join(tvals.dtype.names))
# scale tvals across features
tfs = []
for k in tvals.dtype.names:
# tfs.append(np.dot(tvals[k],
# np.dot(diagsvd(ss[ss > 0],
# len(ss[ss > 0]),
# len(ss[ss > 0])),
# Vh[ss > 0, ...]))) # /(ss>0).sum())
tfs.append(
blockwise_dot(
tvals[k],
blockwise_dot(
diagsvd(ss[ss > 0], len(ss[ss > 0]), len(ss[ss > 0])),
Vh[ss > 0, ...])))
tfs = np.rec.fromarrays(tfs, names=','.join(tvals.dtype.names))
# decide what to return
if perm is None:
# return tvals, tfs, and R for actual non-permuted data
out = (ts, tfs, _R, feat_mask, _ss, mer)
else:
# return the tvals for the terms
out = (ts, tfs, ~feat_mask[0])
return out
def attack_batch(self, imgs, labs):
"""
Run the attack on a batch of images and labels.
"""
# print("imgs, labs in attack_batch", imgs, labs) #shape=(1, 28, 28, 1), dtype=float32) [array([0., 0., 0., 0., 0., 0., 1., 0., 0., 0.])]
batch_size = self.batch_size
def compare(x, y):
if not isinstance(x, (float, int, np.int64)):
x = x.numpy()
x = np.copy(x)
if self.TARGETED:
x[y] -= self.CONFIDENCE
else:
x[y] += self.CONFIDENCE
x = np.argmax(x)
if self.TARGETED:
return x == y
else:
return x != y
# @tf.function
def train_step(modifier, timg, tlab, const):
with tf.GradientTape() as tape:
newimg = tf.tanh(modifier + timg) * self.boxmul + self.boxplus
# newimg = np.random.rand(1, 28, 28, 1)
output = model.predict(newimg)
output = tf.cast(output, dtype=tf.float32)
l2dist = tf.reduce_sum(
tf.square(newimg -
(tf.tanh(timg) * self.boxmul + self.boxplus)),
[1, 2, 3])
real = tf.math.reduce_sum((tlab) * output, 1)
other = tf.math.reduce_max(
(1 - tlab) * output - (tlab * 10000), 1)
if self.TARGETED:
# if targetted, optimize for making the other class most likely
loss1 = tf.maximum(0.0, other - real + self.CONFIDENCE)
else:
# if untargeted, optimize for making this class least likely.
loss1 = tf.maximum(0.0, real - other + self.CONFIDENCE)
loss2 = tf.reduce_sum(l2dist)
loss1 = tf.reduce_sum(const * loss1)
loss = loss1 + loss2
optimizer = optimizers.Adam(self.LEARNING_RATE)
loss_metric = tf.keras.metrics.Mean(name='train_loss')
# optimizer.minimize(self.loss, var_list=[modifier])
grads = tape.gradient(loss, [modifier])
optimizer.apply_gradients(zip(grads, [modifier]))
loss_metric.update_state(loss)
return loss, l2dist, output, newimg, loss1, loss2
# convert to tanh-space
imgs = np.arctanh((imgs - self.boxplus) / self.boxmul * 0.999999)
# print(np.shape(imgs))
lower_bound = np.zeros(batch_size)
CONST = np.ones(batch_size) * self.initial_const
upper_bound = np.ones(batch_size) * 1e10
# the best l2, score, and image attack
o_bestl2 = [1e10] * batch_size
o_bestscore = [-1] * batch_size
o_bestattack = [np.zeros(imgs[0].shape)] * batch_size
print(np.shape(o_bestattack),
"np.shape(o_bestattack)") # (1, 28, 28, 1)
for outer_step in range(self.BINARY_SEARCH_STEPS):
batch = tf.Variable(imgs[:batch_size], dtype=tf.float32)
batchlab = tf.Variable(labs[:batch_size], dtype=tf.float32)
# print("*******batchlab***********", batchlab) # shape=(1, 10)
bestl2 = [1e10] * batch_size
bestscore = [-1] * batch_size
if self.repeat == True and outer_step == self.BINARY_SEARCH_STEPS - 1:
CONST = upper_bound
modifier = tf.Variable(np.zeros((1, 28, 28, 1), dtype=np.float32))
const = tf.Variable(CONST, dtype=tf.float32)
prev = np.inf
for iteration in range(self.MAX_ITERATIONS):
# perform the attack
l, l2s, scores, nimg, loss1, loss2 = train_step(
modifier, batch, batchlab, const)
if np.all(scores >= -.0001) and np.all(scores <= 1.0001):
if np.allclose(np.sum(scores, axis=1), 1.0, atol=1e-3):
if not self.I_KNOW_WHAT_I_AM_DOING_AND_WANT_TO_OVERRIDE_THE_PRESOFTMAX_CHECK:
raise Exception(
"The output of model.predict should return the pre-softmax layer. It looks like you are returning the probability vector (post-softmax). If you are sure you want to do that, set attack.I_KNOW_WHAT_I_AM_DOING_AND_WANT_TO_OVERRIDE_THE_PRESOFTMAX_CHECK = True"
)
if iteration % (self.MAX_ITERATIONS // 10) == 0:
print(iteration, l, loss1, loss2)
# check if we should abort search if we're getting nowhere.
if self.ABORT_EARLY and iteration % (self.MAX_ITERATIONS //
10) == 0:
if l > prev * .9999:
break
prev = l
# adjust the best result found so far
for e, (l2, sc, ii) in enumerate(zip(l2s, scores, nimg)):
# print("batchlab", np.argmax(batchlab[e]))
# print("(sc, np.argmax(batchlab))", sc, np.argmax(sc))
# print("l2 and bestl2[e]", l2, bestl2[e])
# print("compare(sc, tf.argmax(batchlab))",
# compare(sc, tf.argmax(batchlab[e])))
if l2 < bestl2[e] and compare(sc, np.argmax(batchlab[e])):
bestl2[e] = l2
bestscore[e] = np.argmax(sc)
if l2 < o_bestl2[e] and compare(sc, np.argmax(
batchlab[e])):
o_bestl2[e] = l2
o_bestscore[e] = np.argmax(sc)
o_bestattack[e] = ii
# adjust the constant as needed
for e in range(batch_size):
print("bestscore[e]", bestscore[e])
if compare(bestscore[e], np.argmax(
batchlab[e])) and bestscore[e] != -1:
# success, divide const by two
upper_bound[e] = min(upper_bound[e], CONST[e])
if upper_bound[e] < 1e9:
CONST[e] = (lower_bound[e] + upper_bound[e]) / 2
else:
# failure, either multiply by 10 if no solution found yet
# or do binary search with the known upper bound
lower_bound[e] = max(lower_bound[e], CONST[e])
if upper_bound[e] < 1e9:
CONST[e] = (lower_bound[e] + upper_bound[e]) / 2
else:
CONST[e] *= 10
o_bestl2 = np.array(o_bestl2)
return o_bestattack
between_subj_data = []
for subj in subj_overlap:
print('%s: %s %s Getting corr data' % (pu.ctime(), sess, subj))
within_subj_data = []
for roi in rois:
cfc_file = h5py.File(pac_first_level, 'r')
rval_path = subj + '/' + sess + '/' + roi + '/' + 'r_vals'
dset = cfc_file.get(rval_path).value
within_subj_data.append(dset[:, :])
cfc_file.close()
between_subj_data.append(np.arctanh(np.asarray(within_subj_data)))
del within_subj_data
between_subj_array = np.asarray(between_subj_data)
del between_subj_data
print('%s: %s Reorganizing data by ROI' % (pu.ctime(), sess))
roi_dict = {}
for r, roi in enumerate(rois):
band_dict = {}
for s, slow in enumerate(slow_bands):
key_name = slow + ' with supraslow bands'
data_to_grab = between_subj_array[:, r, s, :]
band_dict[key_name] = np.ndarray.squeeze(data_to_grab)
def return_conf_cor_mat(ts_mat, weight_vect, conf_interval_prob=0.01,
normalize=False):
"""
Compute correlation matrices over a time series and weight vector,
either Rsquared-value matrix (cor_mat),
or Z (after R-to-Z values) matrix (Z_cor_mat)
It also return possibly thresholded matrices (conf_cor_mat and
Z_conf_cor_mat) according to a confidence interval probability
conf_interval_prob
"""
if ts_mat.shape[1] == len(weight_vect):
print("Transposing data_matrix shape {} -> {}".format(
ts_mat.shape, np.transpose(ts_mat).shape))
ts_mat = np.transpose(ts_mat)
assert ts_mat.shape[0] == len(weight_vect), \
("Error, incompatible regressor length {} {}".format(ts_mat.shape[0],
len(weight_vect)))
if normalize: # pragma: no cover
print("Normalising data before computing Correlation")
ts_mat = stats.zscore(ts_mat, axis=0, nan_policy="omit")
keep = weight_vect > 0.0
w = weight_vect[keep]
ts_mat = ts_mat[keep, :]
# confidence interval for variance computation
norm = stats.norm.ppf(1-conf_interval_prob/2)
deg_freedom = w.sum()/w.max()-3
s, n = ts_mat.shape
Z_cor_mat = np.zeros((n, n), dtype=float)
Z_conf_cor_mat = np.zeros((n, n), dtype=float)
cor_mat = np.zeros((n, n), dtype=float)
conf_cor_mat = np.zeros((n, n), dtype=float)
ts_mat2 = ts_mat*np.sqrt(w)[:, np.newaxis]
for i, j in it.combinations(list(range(n)), 2):
keep_val = ~(np.isnan(ts_mat2[:, i]) | np.isnan(ts_mat2[:, j]))
s1 = ts_mat2[keep_val, i]
s2 = ts_mat2[keep_val, j]
cor_mat[i, j] = (s1*s2).sum()/np.sqrt((s1*s1).sum() * (s2*s2).sum())
Z_cor_mat[i, j] = np.arctanh(cor_mat[i, j])
assert not np.isnan(Z_cor_mat[i, j]), \
("Error Z_cor_mat {} {} should not be NAN value".format(i, j))
assert not np.isinf(Z_cor_mat[i, j]), \
("Error Z_cor_mat {} {} should not be infinite value".format(i, j))
pos_Z = (np.sign(Z_cor_mat) == +1.0)
neg_Z = (np.sign(Z_cor_mat) == -1.0)
signif_pos = (Z_cor_mat > norm/np.sqrt(deg_freedom)) & pos_Z
signif_neg = (Z_cor_mat < -norm/np.sqrt(deg_freedom)) & neg_Z
Z_conf_cor_mat[signif_pos] = Z_cor_mat[signif_pos]
Z_conf_cor_mat[signif_neg] = Z_cor_mat[signif_neg]
conf_cor_mat[signif_pos] = cor_mat[signif_pos]
conf_cor_mat[signif_neg] = cor_mat[signif_neg]
return cor_mat, Z_cor_mat, conf_cor_mat, Z_conf_cor_mat
def sie_grad(x, y, par):
"""
NAME: sie_grad
PURPOSE: compute the deflection of an SIE potential
USAGE: (xg, yg) = sie_grad(x, y, par)
ARGUMENTS:
x, y: vectors or images of coordinates;
should be matching numpy ndarrays
par: vector of parameters with 1 to 5 elements, defined as follows:
par[0]: lens strength, or 'Einstein radius'
par[1]: (optional) x-center (default = 0.0)
par[2]: (optional) y-center (default = 0.0)
par[3]: (optional) axis ratio (default=1.0)
par[4]: (optional) major axis Position Angle
in degrees c.c.w. of x axis. (default = 0.0)
RETURNS: tuple (xg, yg) of gradients at the positions (x, y)
NOTES: This routine implements an 'intermediate-axis' convention.
Analytic forms for the SIE potential can be found in:
Kassiola & Kovner 1993, ApJ, 417, 450
Kormann et al. 1994, A&A, 284, 285
Keeton & Kochanek 1998, ApJ, 495, 157
The parameter-order convention in this routine differs from that
of a previous IDL routine of the same name by ASB.
WRITTEN: Adam S. Bolton, U of Utah, 2009
"""
# Set parameters:
b = N.abs(par[0]) # can't be negative!!!
xzero = 0. if (len(par) < 2) else par[1]
yzero = 0. if (len(par) < 3) else par[2]
q = 1. if (len(par) < 4) else N.abs(par[3])
phiq = 0. if (len(par) < 5) else par[4]
eps = 0.001 # for sqrt(1/q - q) < eps, a limit expression is used.
# Handle q > 1 gracefully:
if (q > 1.):
q = 1.0 / q
phiq = phiq + 90.0
# Go into shifted coordinats of the potential:
phirad = N.deg2rad(phiq)
xsie = (x - xzero) * N.cos(phirad) + (y - yzero) * N.sin(phirad)
ysie = (y - yzero) * N.cos(phirad) - (x - xzero) * N.sin(phirad)
# Compute potential gradient in the transformed system:
r_ell = N.sqrt(q * xsie**2 + ysie**2 / q)
qfact = N.sqrt(1. / q - q)
# (r_ell == 0) terms prevent divide-by-zero problems
if (qfact >= eps):
xtg = (b / qfact) * N.arctan(qfact * xsie / (r_ell + (r_ell == 0)))
ytg = (b / qfact) * N.arctanh(qfact * ysie / (r_ell + (r_ell == 0)))
else:
xtg = b * xsie / (r_ell + (r_ell == 0))
ytg = b * ysie / (r_ell + (r_ell == 0))
# Transform back to un-rotated system:
xg = xtg * N.cos(phirad) - ytg * N.sin(phirad)
yg = ytg * N.cos(phirad) + xtg * N.sin(phirad)
# Return value:
return (xg, yg)
import numpy as np
import matplotlib.pyplot as plt
maxes = [0.981439, 0.643811, 0.36434, 0.136199]
bluber = np.tanh(maxes)
plt.plot(np.tanh(maxes), np.tanh(maxes), "ro")
plt.rc('text', usetex=True)
plt.ylabel(r'$tanh(\beta J)$')
plt.xlabel(r'$tanh(\beta J)$')
x = np.linspace(1, 0, 100)
y = np.tanh(np.arctanh(x))**2
f = lambda x: np.tanh(np.arctanh(x))**2
interpol = np.tanh(maxes)
params = np.polyfit(interpol, interpol, 1)
plt.plot(x, y, "b")
plt.plot(x, params[0] * x + params[1], "b")
#plot connections
line1 = []
print(line1)
for i in range(0, len(bluber)):
plt.plot([bluber[i], bluber[i]], [bluber[i], f(bluber[i])], "r-.")
plt.plot([bluber[i], f(bluber[i])], [f(bluber[i]), f(bluber[i])], "r-.")
plt.show()
def arctanh(x):
return np.arctanh(x)
def test_squashed_gaussian(self):
"""Tests the SquashedGaussia ActionDistribution (tf-eager only)."""
with eager_mode():
input_space = Box(-1.0, 1.0, shape=(200, 10))
low, high = -2.0, 1.0
# Batch of size=n and deterministic.
inputs = input_space.sample()
means, _ = np.split(inputs, 2, axis=-1)
squashed_distribution = SquashedGaussian(inputs, {},
low=low,
high=high)
expected = ((np.tanh(means) + 1.0) / 2.0) * (high - low) + low
# Sample n times, expect always mean value (deterministic draw).
out = squashed_distribution.deterministic_sample()
check(out, expected)
# Batch of size=n and non-deterministic -> expect roughly the mean.
inputs = input_space.sample()
means, log_stds = np.split(inputs, 2, axis=-1)
squashed_distribution = SquashedGaussian(inputs, {},
low=low,
high=high)
expected = ((np.tanh(means) + 1.0) / 2.0) * (high - low) + low
values = squashed_distribution.sample()
self.assertTrue(np.max(values) < high)
self.assertTrue(np.min(values) > low)
check(np.mean(values), expected.mean(), decimals=1)
# Test log-likelihood outputs.
sampled_action_logp = squashed_distribution.sampled_action_logp()
# Convert to parameters for distr.
stds = np.exp(
np.clip(log_stds, MIN_LOG_NN_OUTPUT, MAX_LOG_NN_OUTPUT))
# Unsquash values, then get log-llh from regular gaussian.
unsquashed_values = np.arctanh((values - low) /
(high - low) * 2.0 - 1.0)
log_prob_unsquashed = \
np.sum(np.log(norm.pdf(unsquashed_values, means, stds)), -1)
log_prob = log_prob_unsquashed - \
np.sum(np.log(1 - np.tanh(unsquashed_values) ** 2),
axis=-1)
check(np.mean(sampled_action_logp), np.mean(log_prob), rtol=0.01)
# NN output.
means = np.array([[0.1, 0.2, 0.3, 0.4, 50.0],
[-0.1, -0.2, -0.3, -0.4, -1.0]])
log_stds = np.array([[0.8, -0.2, 0.3, -1.0, 2.0],
[0.7, -0.3, 0.4, -0.9, 2.0]])
squashed_distribution = SquashedGaussian(np.concatenate(
[means, log_stds], axis=-1), {},
low=low,
high=high)
# Convert to parameters for distr.
stds = np.exp(log_stds)
# Values to get log-likelihoods for.
values = np.array([[0.9, 0.2, 0.4, -0.1, -1.05],
[-0.9, -0.2, 0.4, -0.1, -1.05]])
# Unsquash values, then get log-llh from regular gaussian.
unsquashed_values = np.arctanh((values - low) /
(high - low) * 2.0 - 1.0)
log_prob_unsquashed = \
np.sum(np.log(norm.pdf(unsquashed_values, means, stds)), -1)
log_prob = log_prob_unsquashed - \
np.sum(np.log(1 - np.tanh(unsquashed_values) ** 2),
axis=-1)
out = squashed_distribution.logp(values)
check(out, log_prob)
def corr_cint(r, n):
z = np.arctanh(r)
sigma = (1 / (n - 3)**0.5)
cint = z + np.array([-1, 1]) * sigma * st.norm.ppf((1 + 0.95) / 2)
return np.tanh(cint)
top_patch = blocks.create_patch_nb_faces(name='top', nb_faces=1)
top_patch[0] = [5, 4]
right_patch = blocks.create_patch_nb_faces(name='right', nb_faces=2)
right_patch[0] = [1, 3]
right_patch[1] = [3, 5]
#blocks.partition_blocks(nb_partitions = 4, direction = 0)
mesh = domain.create_component('Mesh', 'cf3.mesh.Mesh')
blocks.create_mesh(mesh.uri())
coordmap = {}
b = 0.9544
xi = np.linspace(-h, h, y_segs * 2 + 1)
y_graded = h / b * np.tanh(xi * np.arctanh(b))
coords = mesh.geometry.coordinates
for i in range(len(coords)):
y_key = int(coords[i][1])
coords[i][1] = y_graded[y_key]
# Make the boundary global, to allow wall distance and periodics to work correctly
make_boundary_global = domain.create_component(
'MakeBoundaryGlobal', 'cf3.mesh.actions.MakeBoundaryGlobal')
make_boundary_global.mesh = mesh
make_boundary_global.execute()
link_horizontal = domain.create_component(
'LinkHorizontal', 'cf3.mesh.actions.LinkPeriodicNodes')
link_horizontal.mesh = mesh
import numpy as np from pylab import * x = np.linspace(-np.pi, np.pi, 2001) sines = np.sin(x) hist(np.arctanh(np.clip(sines, -0.999, 0.999)), bins=40) show()
def attack_batch(self, imgs, labs):
"""
Run the attack on a batch of instance and labels.
"""
def compare(x, y):
if not isinstance(x, (float, int, np.int64)):
x = np.copy(x)
if self.TARGETED:
x[y] -= self.CONFIDENCE
else:
x[y] += self.CONFIDENCE
x = np.argmax(x)
if self.TARGETED:
return x == y
else:
return x != y
batch_size = self.batch_size
oimgs = np.clip(imgs, self.clip_min, self.clip_max)
# re-scale instances to be within range [0, 1]
imgs = (imgs - self.clip_min) / (self.clip_max - self.clip_min)
imgs = np.clip(imgs, 0, 1)
# now convert to [-1, 1]
imgs = (imgs * 2) - 1
# convert to tanh-space
imgs = np.arctanh(imgs * .999999)
# set the lower and upper bounds accordingly
lower_bound = np.zeros(batch_size)
CONST = np.ones(batch_size) * self.initial_const
upper_bound = np.ones(batch_size) * 1e10
# placeholders for the best l2, score, and instance attack found so far
o_bestl2 = [1e10] * batch_size
o_bestscore = [-1] * batch_size
o_bestattack = np.copy(oimgs)
for outer_step in range(self.BINARY_SEARCH_STEPS):
# completely reset adam's internal state.
self.sess.run(self.init)
batch = imgs[:batch_size]
batchlab = labs[:batch_size]
bestl2 = [1e10] * batch_size
bestscore = [-1] * batch_size
_logger.debug(" Binary search step %s of %s",
outer_step, self.BINARY_SEARCH_STEPS)
# The last iteration (if we run many steps) repeat the search once.
if self.repeat and outer_step == self.BINARY_SEARCH_STEPS - 1:
CONST = upper_bound
# set the variables so that we don't have to send them over again
self.sess.run(
self.setup, {
self.assign_timg: batch,
self.assign_tlab: batchlab,
self.assign_const: CONST
})
prev = 1e6
for iteration in range(self.MAX_ITERATIONS):
# perform the attack
_, l, l2s, scores, nimg = self.sess.run([
self.train, self.loss, self.l2dist, self.output,
self.newimg
])
if iteration % ((self.MAX_ITERATIONS // 10) or 1) == 0:
_logger.debug((" Iteration {} of {}: loss={:.3g} " +
"l2={:.3g} f={:.3g}").format(
iteration, self.MAX_ITERATIONS, l,
np.mean(l2s), np.mean(scores)))
# check if we should abort search if we're getting nowhere.
if self.ABORT_EARLY and \
iteration % ((self.MAX_ITERATIONS // 10) or 1) == 0:
if l > prev * .9999:
msg = " Failed to make progress; stop early"
_logger.debug(msg)
break
prev = l
# adjust the best result found so far
for e, (l2, sc, ii) in enumerate(zip(l2s, scores, nimg)):
lab = np.argmax(batchlab[e])
if l2 < bestl2[e] and compare(sc, lab):
bestl2[e] = l2
bestscore[e] = np.argmax(sc)
if l2 < o_bestl2[e] and compare(sc, lab):
o_bestl2[e] = l2
o_bestscore[e] = np.argmax(sc)
o_bestattack[e] = ii
# adjust the constant as needed
for e in range(batch_size):
if compare(bestscore[e], np.argmax(batchlab[e])) and \
bestscore[e] != -1:
# success, divide const by two
upper_bound[e] = min(upper_bound[e], CONST[e])
if upper_bound[e] < 1e9:
CONST[e] = (lower_bound[e] + upper_bound[e]) / 2
else:
# failure, either multiply by 10 if no solution found yet
# or do binary search with the known upper bound
lower_bound[e] = max(lower_bound[e], CONST[e])
if upper_bound[e] < 1e9:
CONST[e] = (lower_bound[e] + upper_bound[e]) / 2
else:
CONST[e] *= 10
_logger.debug(" Successfully generated adversarial examples " +
"on {} of {} instances.".format(
sum(upper_bound < 1e9), batch_size))
o_bestl2 = np.array(o_bestl2)
mean = np.mean(np.sqrt(o_bestl2[o_bestl2 < 1e9]))
_logger.debug(" Mean successful distortion: {:.4g}".format(mean))
# return the best solution found
o_bestl2 = np.array(o_bestl2)
return o_bestattack
def testComplexOps(self):
for dtype in self.complex_types:
self._assertOpOutputMatchesExpected(
math_ops.acosh,
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype),
expected=np.arccosh(
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.asinh,
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype),
expected=np.arcsinh(
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.atanh,
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype),
expected=np.arctanh(
np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.cosh,
np.array([1j, 2 - 3j, 3, 4 + 2j], dtype=dtype),
expected=np.cosh(np.array([1j, 2 - 3j, 3, 4 + 2j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.sinh,
np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype),
expected=np.sinh(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.exp,
np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype),
expected=np.exp(np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.expm1,
np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype),
expected=np.expm1(np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype)),
rtol=1e-6,
atol=1e-6)
self._assertOpOutputMatchesExpected(
math_ops.reciprocal,
np.array([[1, 2j, 2 + 3j]], dtype=dtype),
expected=1.0 / np.array([[1, 2j, 2 + 3j]], dtype=dtype))
self._assertOpOutputMatchesExpected(
math_ops.log,
np.array([[5j, 3 - 2j]], dtype=dtype),
expected=np.log(np.array([[5j, 3 - 2j]], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.sin,
np.array([[5j, 3 - 2j]], dtype=dtype),
expected=np.sin(np.array([[5j, 3 - 2j]], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.cos,
np.array([[5j, 3 - 2j]], dtype=dtype),
expected=np.cos(np.array([[5j, 3 - 2j]], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.log1p,
np.array([[1e-14, 1e-15j, 0.6 - 0.3j]], dtype=dtype),
expected=np.log1p(
np.array([[1e-14, 1e-15j, 0.6 - 0.3j]], dtype=dtype)),
rtol=1e-4,
atol=1e-6)
val = np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)
self._assertOpOutputMatchesExpected(
math_ops.rsqrt, val, expected=1 / np.sqrt(val))
self._assertOpOutputMatchesExpected(
math_ops.sigmoid, val, expected=1 / (1 + np.exp(-val)))
self._assertOpOutputMatchesExpected(
math_ops.sqrt, val, expected=np.sqrt(val))
self._assertOpOutputMatchesExpected(
math_ops.tanh,
np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype),
expected=np.tanh(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.tan,
np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype),
expected=np.tan(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)))
ctypes = {np.complex64: np.float32}
self._assertOpOutputMatchesExpected(
math_ops.abs,
np.array([[3 - 4j, -1j, np.inf]], dtype=dtype),
expected=np.array([[5, 1, np.inf]], dtype=ctypes[dtype]))
self._assertOpOutputMatchesExpected(
math_ops.negative,
np.array([[-1 + 2j, -3j]], dtype=dtype),
expected=np.array([[1 - 2j, 3j]], dtype=dtype))
self._assertOpOutputMatchesExpected(
math_ops.square,
np.array([[-2 - 3j, 3 + 4j, 5j]], dtype=dtype),
expected=np.array([[-2 - 3j, 3 + 4j, 5j]], dtype=dtype)**2)
self._assertOpOutputMatchesExpected(
array_ops.zeros_like,
np.array([[4j, 3 - 2j], [2, -1j]], dtype=dtype),
expected=np.array([[0, 0], [0, 0]], dtype=dtype))
self._assertOpOutputMatchesExpected(
array_ops.ones_like,
np.array([[-4j, 3 + 2j], [2, -1j]], dtype=dtype),
expected=np.array([[1, 1], [1, 1]], dtype=dtype))
self._assertOpOutputMatchesExpected(
math_ops.angle,
np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
expected=np.angle(np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype)))
self._assertOpOutputMatchesExpected(
math_ops.conj,
np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
expected=np.array([1 - 3j, -4 - 7j, 2.7, 3j], dtype=dtype))
self._assertOpOutputMatchesExpected(
math_ops.imag,
np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
expected=np.array([3, 7, 0, -3], dtype=ctypes[dtype]))
self._assertOpOutputMatchesExpected(
math_ops.real,
np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
expected=np.array([1, -4, 2.7, 0], dtype=ctypes[dtype]))
def check_loss_of_precision(self, dtype):
"""Check loss of precision in complex arc* functions"""
# Check against known-good functions
info = np.finfo(dtype)
real_dtype = dtype(0.).real.dtype
eps = info.eps
def check(x, rtol):
x = x.astype(real_dtype)
z = x.astype(dtype)
d = np.absolute(np.arcsinh(x) / np.arcsinh(z).real - 1)
assert_(np.all(d < rtol),
(np.argmax(d), x[np.argmax(d)], d.max(), 'arcsinh'))
z = (1j * x).astype(dtype)
d = np.absolute(np.arcsinh(x) / np.arcsin(z).imag - 1)
assert_(np.all(d < rtol),
(np.argmax(d), x[np.argmax(d)], d.max(), 'arcsin'))
z = x.astype(dtype)
d = np.absolute(np.arctanh(x) / np.arctanh(z).real - 1)
assert_(np.all(d < rtol),
(np.argmax(d), x[np.argmax(d)], d.max(), 'arctanh'))
z = (1j * x).astype(dtype)
d = np.absolute(np.arctanh(x) / np.arctan(z).imag - 1)
assert_(np.all(d < rtol),
(np.argmax(d), x[np.argmax(d)], d.max(), 'arctan'))
# The switchover was chosen as 1e-3; hence there can be up to
# ~eps/1e-3 of relative cancellation error before it
x_series = np.logspace(-20, -3.001, 200)
x_basic = np.logspace(-2.999, 0, 10, endpoint=False)
if dtype is np.longcomplex:
# It's not guaranteed that the system-provided arc functions
# are accurate down to a few epsilons. (Eg. on Linux 64-bit)
# So, give more leeway for long complex tests here:
check(x_series, 50 * eps)
else:
check(x_series, 2 * eps)
check(x_basic, 2 * eps / 1e-3)
# Check a few points
z = np.array([1e-5 * (1 + 1j)], dtype=dtype)
p = 9.999999999333333333e-6 + 1.000000000066666666e-5j
d = np.absolute(1 - np.arctanh(z) / p)
assert_(np.all(d < 1e-15))
p = 1.0000000000333333333e-5 + 9.999999999666666667e-6j
d = np.absolute(1 - np.arcsinh(z) / p)
assert_(np.all(d < 1e-15))
p = 9.999999999333333333e-6j + 1.000000000066666666e-5
d = np.absolute(1 - np.arctan(z) / p)
assert_(np.all(d < 1e-15))
p = 1.0000000000333333333e-5j + 9.999999999666666667e-6
d = np.absolute(1 - np.arcsin(z) / p)
assert_(np.all(d < 1e-15))
# Check continuity across switchover points
def check(func, z0, d=1):
z0 = np.asarray(z0, dtype=dtype)
zp = z0 + abs(z0) * d * eps * 2
zm = z0 - abs(z0) * d * eps * 2
assert_(np.all(zp != zm), (zp, zm))
# NB: the cancellation error at the switchover is at least eps
good = (abs(func(zp) - func(zm)) < 2 * eps)
assert_(np.all(good), (func, z0[~good]))
for func in (np.arcsinh, np.arcsinh, np.arcsin, np.arctanh, np.arctan):
pts = [
rp + 1j * ip for rp in (-1e-3, 0, 1e-3)
for ip in (-1e-3, 0, 1e-3) if rp != 0 or ip != 0
]
check(func, pts, 1)
check(func, pts, 1j)
check(func, pts, 1 + 1j)
def _fisherZ(self, pc):
return (np.arctanh(pc))
def attack_batch(self, imgs, labs):
"""
Run the attack on a batch of images and labels.
"""
def compare(x_vl, y_1h):
x_v = np.argmax(x_vl - self.CONFIDENCE * y_1h[None, :], axis=1)
y = np.argmax(y_1h)
if self.TARGETED:
return np.all(x_v == y)
else:
return np.all(x_v != y)
batch_size = self.batch_size
# convert to tanh-space
imgs = np.arctanh((imgs - 127.5) / 127.5001)
# set the lower and upper bounds accordingly
lower_bound = np.zeros(batch_size)
CONST = np.ones(batch_size) * self.initial_const
upper_bound = np.ones(batch_size) * 1e10
# the best l2, score, and image attack
o_bestl2 = [1e10] * batch_size
o_bestscore = [False] * batch_size
o_bestattack = [np.zeros(imgs[0].shape)] * batch_size
for outer_step in range(self.BINARY_SEARCH_STEPS):
# %%% print(o_bestl2)
print CONST
# completely reset adam's internal state.
self.sess.run(self.init)
batch = imgs[:batch_size]
batchlab = labs[:batch_size]
bestl2 = [1e10] * batch_size
bestscore = [False] * batch_size
# The last iteration (if we run many steps) repeat the search once.
if self.repeat == True and outer_step == self.BINARY_SEARCH_STEPS - 1:
CONST = upper_bound
# set the variables so that we don't have to send them over again
self.sess.run(
self.setup, {
self.assign_timg: batch,
self.assign_tlab: batchlab,
self.assign_const: CONST
})
prev = 1e6
for iteration in range(self.MAX_ITERATIONS):
# perform the attack
_, l, l2s, scores_nvl, nimg = self.sess.run([
self.train, self.loss, self.l2dist, self.output_nvl,
self.newimg
])
# print out the losses every 10%
if iteration % (self.MAX_ITERATIONS // 100) == 0: # %%%
print(iteration,
self.sess.run((self.loss, self.loss1, self.loss2)))
# check if we should abort search if we're getting nowhere.
if self.ABORT_EARLY and iteration % (self.MAX_ITERATIONS //
10) == 0:
if l > prev * .9999:
print 'aborting early at iteration', iteration
break
prev = l
# adjust the best result found so far
for e, (l2, sc_vl, ii) in enumerate(zip(l2s, scores_nvl,
nimg)):
if l2 < bestl2[e] and compare(sc_vl, batchlab[e]):
bestl2[e] = l2
bestscore[e] = True
if l2 < o_bestl2[e] and compare(sc_vl, batchlab[e]):
o_bestl2[e] = l2
o_bestscore[e] = True
o_bestattack[e] = ii
# adjust the constant as needed
for e in range(batch_size):
if bestscore[e]:
# success, divide const by two
upper_bound[e] = min(upper_bound[e], CONST[e])
if upper_bound[e] < 1e9:
CONST[e] = (lower_bound[e] + upper_bound[e]) / 2
else:
# failure, either multiply by 10 if no solution found yet
# or do binary search with the known upper bound
lower_bound[e] = max(lower_bound[e], CONST[e])
if upper_bound[e] < 1e9:
CONST[e] = (lower_bound[e] + upper_bound[e]) / 2
else:
CONST[e] *= 10
# return the best solution found
o_bestl2 = np.array(o_bestl2)
return o_bestattack
def CartToPartCoor3D(x, y, z):
pt, phi = CartToRad2D(x, y)
eta = np.arctanh(z / np.sqrt(z**2 + pt**2))
return pt, eta, phi
def attack_batch(self, imgs, labs):
"""
Run the attack on a batch of images and labels.
"""
def compare(x, y):
if not isinstance(x, (float, int, np.int64)):
x = np.copy(x)
if self.TARGETED:
x[y] -= self.CONFIDENCE
else:
x[y] += self.CONFIDENCE
x = np.argmax(x)
if self.TARGETED:
return x == y
else:
return x != y
batch_size = self.batch_size
# convert to tanh-space
imgs = np.arctanh((imgs - self.boxplus) / self.boxmul * 0.999999)
# set the lower and upper bounds accordingly
lower_bound = np.zeros(batch_size)
CONST = np.ones(batch_size) * self.initial_const
upper_bound = np.ones(batch_size) * 1e10
# the best l2, score, and image attack
o_bestl2 = [1e10] * batch_size
o_bestscore = [-1] * batch_size
o_bestattack = [np.zeros(imgs[0].shape)] * batch_size
for outer_step in range(self.BINARY_SEARCH_STEPS):
print(o_bestl2)
# completely reset adam's internal state.
self.sess.run(self.init)
batch = imgs[:batch_size]
batchlab = labs[:batch_size]
bestl2 = [1e10] * batch_size
bestscore = [-1] * batch_size
# The last iteration (if we run many steps) repeat the search once.
if self.repeat == True and outer_step == self.BINARY_SEARCH_STEPS - 1:
CONST = upper_bound
# set the variables so that we don't have to send them over again
self.sess.run(self.setup, {self.assign_timg: batch,
self.assign_tlab: batchlab,
self.assign_const: CONST,
self.assign_newimg: batch})
prev = np.inf
for iteration in range(self.MAX_ITERATIONS):
# perform the attack
_, l, l2s, scores, nimg = self.sess.run([self.train, self.loss,
self.l2dist, self.output,
self.newimg])
if np.all(scores >= -.0001) and np.all(scores <= 1.0001):
if np.allclose(np.sum(scores, axis=1), 1.0, atol=1e-3):
if not self.I_KNOW_WHAT_I_AM_DOING_AND_WANT_TO_OVERRIDE_THE_PRESOFTMAX_CHECK:
raise Exception(
"The output of model.predict should return the pre-softmax layer. It looks like you are returning the probability vector (post-softmax). If you are sure you want to do that, set attack.I_KNOW_WHAT_I_AM_DOING_AND_WANT_TO_OVERRIDE_THE_PRESOFTMAX_CHECK = True")
# print out the losses every 10%
if iteration % (self.MAX_ITERATIONS // 10) == 0:
print(iteration, self.sess.run((self.loss, self.loss1, self.loss2)))
# check if we should abort search if we're getting nowhere.
if self.ABORT_EARLY and iteration % (self.MAX_ITERATIONS // 10) == 0:
if l > prev * .9999:
break
prev = l
# adjust the best result found so far
for e, (l2, sc, ii) in enumerate(zip(l2s, scores, nimg)):
if l2 < bestl2[e] and compare(sc, np.argmax(batchlab[e])):
bestl2[e] = l2
bestscore[e] = np.argmax(sc)
if l2 < o_bestl2[e] and compare(sc, np.argmax(batchlab[e])):
o_bestl2[e] = l2
o_bestscore[e] = np.argmax(sc)
o_bestattack[e] = ii
# adjust the constant as needed
for e in range(batch_size):
if compare(bestscore[e], np.argmax(batchlab[e])) and bestscore[e] != -1:
# success, divide const by two
upper_bound[e] = min(upper_bound[e], CONST[e])
if upper_bound[e] < 1e9:
CONST[e] = (lower_bound[e] + upper_bound[e]) / 2
else:
# failure, either multiply by 10 if no solution found yet
# or do binary search with the known upper bound
lower_bound[e] = max(lower_bound[e], CONST[e])
if upper_bound[e] < 1e9:
CONST[e] = (lower_bound[e] + upper_bound[e]) / 2
else:
CONST[e] *= 10
# return the best solution found
o_bestl2 = np.array(o_bestl2)
return o_bestattack
def parasite_drag_fuselage(conditions, configuration, fuselage):
""" SUAVE.Methods.parasite_drag_fuselage(conditions,configuration,fuselage)
computes the parasite drag associated with a fuselage
Inputs:
Outputs:
Assumptions:
"""
# unpack inputs
form_factor = configuration.fuselage_parasite_drag_form_factor
freestream = conditions.freestream
Sref = fuselage.areas.front_projected
Swet = fuselage.areas.wetted
l_fus = fuselage.lengths.cabin
d_fus = fuselage.width
l_nose = fuselage.lengths.nose
l_tail = fuselage.lengths.tail
# conditions
Mc = freestream.mach_number
roc = freestream.density
muc = freestream.viscosity
Tc = freestream.temperature
pc = freestream.pressure
# reynolds number
V = Mc * compute_speed_of_sound(Tc, pc)
Re_fus = roc * V * l_fus / muc
# skin friction coefficient
cf_fus, k_comp, k_reyn = compressible_turbulent_flat_plate(Re_fus, Mc, Tc)
# form factor for cylindrical bodies
d_d = float(d_fus) / float(l_fus)
D = np.array([[0.0]] * len(Mc))
a = np.array([[0.0]] * len(Mc))
du_max_u = np.array([[0.0]] * len(Mc))
k_fus = np.array([[0.0]] * len(Mc))
D[Mc < 0.95] = np.sqrt(1 - (1 - Mc[Mc < 0.95]**2) * d_d**2)
a[Mc < 0.95] = 2 * (1 - Mc[Mc < 0.95]**2) * (d_d**2) * (
np.arctanh(D[Mc < 0.95]) - D[Mc < 0.95]) / (D[Mc < 0.95]**3)
du_max_u[Mc < 0.95] = a[Mc < 0.95] / ((2 - a[Mc < 0.95]) *
(1 - Mc[Mc < 0.95]**2)**0.5)
D[Mc >= 0.95] = np.sqrt(1 - d_d**2)
a[Mc >= 0.95] = 2 * (d_d**2) * (np.arctanh(D[Mc >= 0.95]) -
D[Mc >= 0.95]) / (D[Mc >= 0.95]**3)
du_max_u[Mc >= 0.95] = a[Mc >= 0.95] / ((2 - a[Mc >= 0.95]))
k_fus = (1 + form_factor * du_max_u)**2
#for i in range(len(Mc)):
#if Mc[i] < 0.95:
#D[i] = np.sqrt(1 - (1-Mc[i]**2) * d_d**2)
#a[i] = 2 * (1-Mc[i]**2) * (d_d**2) *(np.arctanh(D[i])-D[i]) / (D[i]**3)
#du_max_u[i] = a[i] / ( (2-a[i]) * (1-Mc[i]**2)**0.5 )
#else:
#D[i] = np.sqrt(1 - d_d**2)
#a[i] = 2 * (d_d**2) *(np.arctanh(D[i])-D[i]) / (D[i]**3)
#du_max_u[i] = a[i] / ( (2-a[i]) )
#k_fus[i] = (1 + cf_fus[i]*du_max_u[i])**2
# --------------------------------------------------------
# find the final result
fuselage_parasite_drag = k_fus * cf_fus * Swet / Sref
# --------------------------------------------------------
# dump data to conditions
fuselage_result = Result(
wetted_area=Swet,
reference_area=Sref,
parasite_drag_coefficient=fuselage_parasite_drag,
skin_friction_coefficient=cf_fus,
compressibility_factor=k_comp,
reynolds_factor=k_reyn,
form_factor=k_fus,
)
try:
conditions.aerodynamics.drag_breakdown.parasite[
fuselage.tag] = fuselage_result
except:
print("Drag Polar Mode fuse parasite")
return fuselage_parasite_drag
np.argmin(_convert_to_tensor(input),
axis=0 if axis is None else int(axis)).astype(
utils.numpy_dtype(output_type))))
asin = utils.copy_docstring('tf.math.asin', lambda x, name=None: np.arcsin(x))
asinh = utils.copy_docstring('tf.math.asinh',
lambda x, name=None: np.arcsinh(x))
atan = utils.copy_docstring('tf.math.atan', lambda x, name=None: np.arctan(x))
atan2 = utils.copy_docstring('tf.math.atan2',
lambda y, x, name=None: np.arctan2(y, x))
atanh = utils.copy_docstring('tf.math.atanh',
lambda x, name=None: np.arctanh(x))
bessel_i0 = utils.copy_docstring('tf.math.bessel_i0',
lambda x, name=None: scipy_special.i0(x))
bessel_i0e = utils.copy_docstring('tf.math.bessel_i0e',
lambda x, name=None: scipy_special.i0e(x))
bessel_i1 = utils.copy_docstring('tf.math.bessel_i1',
lambda x, name=None: scipy_special.i1(x))
bessel_i1e = utils.copy_docstring('tf.math.bessel_i1e',
lambda x, name=None: scipy_special.i1e(x))
betainc = utils.copy_docstring(
'tf.math.betainc',
def generate(self, x_val, **kwargs):
"""
Generate adversarial samples and return them in a Numpy array.
:param x_val:
:param y_val: If self.targeted is true, then y_val represents the target labels. If self.targeted is false,
then targets are the original class labels.
:return: A Numpy array holding the adversarial examples.
"""
# Parse and save attack-specific parameters
params_cpy = dict(kwargs)
y_val = params_cpy.pop('y_val', None)
# Assert that, if attack is targeted, y_val is provided:
assert not (self.targeted and y_val is None)
# No labels provided, use model prediction as correct class
if y_val is None:
y_val = self.sess.run(
tf.argmax(self.classifier.model(self._x), axis=1),
{self._x: x_val})
y_val = to_categorical(
y_val,
self.classifier.model.get_output_shape_at(-1)[-1])
# images to be attacked:
x_adv = x_val.copy()
# transform images to tanh space:
x_adv = np.clip(x_adv, self.clip_min, self.clip_max)
x_adv = (x_adv - self.clip_min) / (self.clip_max - self.clip_min)
x_adv = np.arctanh(((x_adv * 2) - 1) * self._tanh_smoother)
# Progress bar
progress_bar = Progbar(target=len(x_val), verbose=self.verbose)
for j, (x, target) in enumerate(zip(x_adv, y_val)):
# Assign the external inputs to the loss function:
self.sess.run(self._assign_image_tanh, {self._image_tanh: x})
self.sess.run(self._assign_target, {self._target: target})
# Initialize perturbation in tanh space:
self.sess.run(self._init_perturbation_tanh)
# Initialize binary search:
c = self.initial_const
c_lower_bound = 0
c_double = True
# Initalize placeholders for best l2 distance and attack found so far
best_l2dist = sys.float_info.max
best_adv_image = x
for _ in range(self.binary_search_steps):
attack_success = False
loss_prev = sys.float_info.max
# Initalize the optimizer:
self.sess.run(self._init_optimizer)
# Assign constant c of the loss function:
self.sess.run(self._assign_c, {self._c: np.array([c])})
for _ in range(self.max_iterations):
# perform one update of the optimizer:
_ = self.sess.run(self._minimize_loss)
# collect current loss and l2 distance:
loss, l2dist = self.sess.run([self._loss, self._l2dist])
# check whether last attack was successful:
# attack success criterion: first term of the loss function is <= 0
last_attack_success = loss[0] - l2dist <= 0
attack_success = attack_success or last_attack_success
if last_attack_success and l2dist < best_l2dist:
best_l2dist = l2dist
best_adv_image = self.sess.run(self._adv_image)
# check simple stopping criterion:
if loss[0] > loss_prev:
break
loss_prev = loss[0]
# update binary search:
if attack_success:
c_double = False
c = (c_lower_bound + c) / 2
else:
c_old = c
if c_double:
c = 2 * c
else:
c = c + (c - c_lower_bound) / 2
c_lower_bound = c_old
# Abort binary search if c exceeds upper bound:
if c > self._c_upper_bound:
break
x_adv[j] = best_adv_image
progress_bar.update(current=j,
values=[
("perturbation",
abs(np.linalg.norm(x_adv[j] - x_val[j])))
])
return x_adv
def futhark_atanh64(x):
return np.arctanh(x)
def attack_batch(self, imgs, labs):
"""
Run the attack on a batch of images and labels.
"""
def compare(x, y):
if not isinstance(x, (float, int, np.int64)):
x = np.copy(x)
if self.TARGETED:
x[y] -= self.CONFIDENCE
else:
x[y] += self.CONFIDENCE
x = np.argmax(x)
if self.TARGETED:
return x == y
else:
return x != y
batch_size = self.batch_size
# convert to tanh-space
imgs = np.arctanh((imgs - self.boxplus) / self.boxmul * 0.999999)
# set the lower and upper bounds accordingly
lower_bound = np.zeros(batch_size)
CONST = np.ones(batch_size) * self.initial_const
CONST2 = np.ones(batch_size) * self.initial_const
upper_bound = np.ones(batch_size) * 1e10
# the best l2, score, and image attack
o_bestl2 = [1e10] * batch_size
o_bestscore = [-1] * batch_size
o_bestattack = [np.zeros(imgs[0].shape)] * batch_size
for outer_step in range(self.BINARY_SEARCH_STEPS):
print(o_bestl2)
# completely reset adam's internal state.
self.sess.run(self.init)
batch = imgs[:batch_size]
batchlab = labs[:batch_size]
bestl2 = [1e10] * batch_size
bestscore = [-1] * batch_size
# The last iteration (if we run many steps) repeat the search once.
if self.repeat == True and outer_step == self.BINARY_SEARCH_STEPS - 1:
CONST = upper_bound
# set the variables so that we don't have to send them over again
self.sess.run(
self.setup, {
self.assign_timg: batch,
self.assign_tlab: batchlab,
self.assign_const: CONST,
self.assign_const2: CONST2 * 1000
})
prev = 1e6
for iteration in range(self.MAX_ITERATIONS):
# perform the attack
_, l, l2s, scores, det, nimg = self.sess.run([
self.train, self.loss, self.l2dist, self.output,
self.detected, self.newimg
])
# print out the losses every 10%
if iteration % (self.MAX_ITERATIONS // 10) == 0:
print(
iteration,
self.sess.run(
(self.loss, self.loss1, self.loss2, self.loss3)))
# check if we should abort search if we're getting nowhere.
if self.ABORT_EARLY and iteration % (self.MAX_ITERATIONS //
10) == 0:
if l > prev * .9999:
break
prev = l
#print('detection rate',det)
# adjust the best result found so far
if type(det) == np.float32:
det = [0] * batch_size
for e, (l2, sc, ii,
detected) in enumerate(zip(l2s, scores, nimg, det)):
#print('have scores',sc)
if l2 < bestl2[e] and all(
[compare(scc, np.argmax(batchlab[e]))
for scc in sc]) and detected == 0:
bestl2[e] = l2
bestscore[e] = np.argmax(sc)
if l2 < o_bestl2[e] and all(
[compare(scc, np.argmax(batchlab[e]))
for scc in sc]) and detected == 0:
o_bestl2[e] = l2
o_bestscore[e] = np.argmax(sc)
o_bestattack[e] = ii
# adjust the constant as needed
for e in range(batch_size):
if compare(bestscore[e], np.argmax(
batchlab[e])) and bestscore[e] != -1:
# success, divide const by two
upper_bound[e] = min(upper_bound[e], CONST[e])
if upper_bound[e] < 1e9:
CONST[e] = (lower_bound[e] + upper_bound[e]) / 2
else:
# failure, either multiply by 10 if no solution found yet
# or do binary search with the known upper bound
lower_bound[e] = max(lower_bound[e], CONST[e])
if upper_bound[e] < 1e9:
CONST[e] = (lower_bound[e] + upper_bound[e]) / 2
else:
CONST[e] *= 10
# return the best solution found
o_bestl2 = np.array(o_bestl2)
return o_bestattack
def gamma_approx():
gamma_approx = 1
for k in range(2, 20):
gamma_approx += 1 / k + np.log(1 - 1 / k)
gamma_approx += 38 * np.arctanh(1 / 39) - 1 + euler_maclaurin_terms(20)
return gamma_approx
