def _compute_weights(source_series, parcel_series, identities, inverse):
cplv_array = plv(source_series, parcel_series, identities)
"""Get weights and flip. This could be the output."""
weights = scipy.sign(scipy.real(cplv_array)) * scipy.real(cplv_array)**2
"""Create weighted inverse operator and normalize the norm of weighted inv op
to match original inv op's norm."""
"""Multiply sensor dimension in inverseOperator by weight. This one would be
the un-normalized operator."""
weighted_inv = scipy.einsum('ij,i->ij', inverse, weights)
n_parcels = max(identities) + 1
"""Initialize norm normalized weights. Maybe not necessary."""
weights_normalized = scipy.zeros(len(weights))
for parcel in range(n_parcels): # Normalize parcel level norms.
# Index sources belonging to parcel
ii = [i for i, source in enumerate(identities) if source == parcel]
# Normalize per parcel.
weights_normalized[ii] = weights[ii] * (norm(inverse[ii]) /
norm(weighted_inv[ii]))
"""Parcel level normalized operator."""
weighted_inv = scipy.einsum('ij,i->ij', inverse, weights_normalized)
"""Operator level normalized operator. If there are sources not in any
parcel weightedInvOp gets Nan values due to normalizations."""
weighted_inv *= norm(inverse) / norm(scipy.nan_to_num(weighted_inv))
weighted_inv = scipy.nan_to_num(weighted_inv)
return weighted_inv
def fixfermisigns(Lx,Ly,shift,q,H,O,ori):
fs=[]
if ori=='trans':
fs=transfermisigns(Lx,Ly,shift,q)
elif ori=='long':
fs=longfermisigns(Lx,Ly,shift,q)
H=sc.einsum('i,jik,k->jik',fs,H,fs)
O=sc.einsum('i,jik,k->jik',fs,O,fs)
return H,O
def fixfermisigns(Lx, Ly, shift, q, H, O, ori):
fs = []
if ori == 'trans':
fs = transfermisigns(Lx, Ly, shift, q)
elif ori == 'long':
fs = longfermisigns(Lx, Ly, shift, q)
H = sc.einsum('i,jik,k->jik', fs, H, fs)
O = sc.einsum('i,jik,k->jik', fs, O, fs)
return H, O
def process(self, G, verbose=False):
r"""
Fit genotypes one-by-one.
Parameters
----------
G : (`N`, `S`) ndarray
genotype vector for `N` individuals and `S` variants.
verbose : bool
verbose flag.
Returns
-------
pv : ndarray
P values
beta : ndarray
variant effect szies
"""
import scipy as sp
import scipy.stats as st
t0 = time()
# precompute some stuff
if self.Ki_dot is None:
KiG = G
else:
KiG = self.Ki_dot(G)
GKiy = sp.dot(G.T, self.Kiy[:, 0])
GKiG = sp.einsum("ij,ij->j", G, KiG)
FKiG = sp.dot(self.F.T, KiG)
# Let us denote the inverse of Areml as
# Ainv = [[A0i + m mt / n, m], [mT, n]]
A0iFKiG = sp.dot(self.A0i, FKiG)
n = 1.0 / (GKiG - sp.einsum("ij,ij->j", FKiG, A0iFKiG))
M = -n * A0iFKiG
self.beta_F = self.beta_F0 + M * sp.dot(M.T, self.FKiy[:, 0]) / n
self.beta_F += M * GKiy
self.beta_g = sp.einsum("is,i->s", M, self.FKiy[:, 0])
self.beta_g += n * GKiy
# sigma
s2 = self.yKiy - sp.einsum("i,is->s", self.FKiy[:, 0], self.beta_F)
s2 -= GKiy * self.beta_g
s2 /= self.df
# dlml and pvs
self.lrt = -self.df * sp.log(s2 / self.s20)
self.pv = st.chi2(1).sf(self.lrt)
t1 = time()
if verbose:
print("Tested for %d variants in %.2f s" % (G.shape[1], t1 - t0))
def GetEigSys(filename,gsfile=None,Nsamp=1,channel=None,wavefile=None,q=None):
if type(filename)==str:
filename=[filename]
hfile=h5py.File(filename[0],'r')
attr=GetAttr(filename[0])
if channel==None:
channel=attr['channel']
dpath,args=GetStat(filename,Nsamp)
dat=sc.array(hfile["/rank-1/data-0"])
hfile.close()
N=int(sc.shape(dat)[0]/2)
L=attr['L']
shift=None
if 'phasex' in attr.keys():
shift=[attr['phasex']/2.0,attr['phasey']/2.0]
else:
shift=[attr['phase_shift_x']/2.0,attr['phase_shift_y']/2.0]
H=sc.zeros([Nsamp,N,N],complex)
O=sc.zeros([Nsamp,N,N],complex)
E=sc.zeros([Nsamp,N])
V=sc.zeros([Nsamp,N,N],complex)
for sample,b in enumerate(args):
for d in b:
hfile=h5py.File(dpath[d][0],'r')
dat=hfile[dpath[d][1]]
H[sample,:,:]+=dat[0:N,0:2*N:2]+1j*dat[0:N,1:2*N:2]
O[sample,:,:]+=dat[N:2*N,0:2*N:2]+1j*dat[N:2*N,1:2*N:2]
hfile.close()
H[sample,:,:]=0.5*(H[sample,:,:]+sc.conj(H[sample,:,:].T))/len(b)
O[sample,:,:]=0.5*(O[sample,:,:]+sc.conj(O[sample,:,:].T))/len(b)
if channel=='groundstate':
return H
fs=None
refstate=sc.zeros(2*L*L)
refstate[0::2]=1
if wavefile==None:
fs=GetFermiSigns(filename[0],refstate,channel=channel)
else:
fs=GetFermiSigns(wavefile,refstate,channel=channel)
for s in range(sc.shape(H)[0]):
H[s,:,:]=sc.dot(sc.diag(fs),sc.dot(H[s,:,:],sc.diag(fs)))
O[s,:,:]=sc.dot(sc.diag(fs),sc.dot(O[s,:,:],sc.diag(fs)))
ren=sc.ones(Nsamp)
if gsfile!=None:
ren=RenormalizeFactor(filename,gsfile,Nsamp=1,channel=channel,O=O,q=q)
print('{0} pair of (H,O) matrices loaded, now diagonalize'.format(sc.shape(H)[0]))
H=sc.einsum('ijk,i->ijk',H,ren)
O=sc.einsum('ijk,i->ijk',O,ren)
for s in range(sc.shape(H)[0]):
E[s,:],V[s,:,:]=vln.geneigh(sc.squeeze(H[s,:,:]),sc.squeeze(O[s,:,:]))
print('diagonalization finished')
return H,O,E,V
def sqwtransamp(V,O,Lx,Ly,q,shift,phi,neel,r=sc.zeros((1,2)),rp=sc.zeros((1,2))):
"""
Returns Sq[sample,r,rp,n]=<q,r|q,n><q,n|q,rp>
"""
sqn=sc.zeros(sc.shape(V)[0:2],complex)
kx,ky=fermisea(Lx,Ly,shift)
pkrp=sc.zeros((sc.shape(V)[1],sc.shape(rp)[0]),complex)
pkr=sc.zeros((sc.shape(V)[1],sc.shape(r)[0]),complex)
pkrp[0:len(kx),:]=phiktrans(kx,ky,q[0],q[1],[phi,neel],rp)
pkr[0:len(ky),:]=phiktrans(kx,ky,q[0],q[1],[phi,neel],r)
OV=sc.einsum('ijk,ikl->ijl',O,V)
rhs=sc.einsum('ijk,jl->ikl',sc.conj(OV),pkrp)
lhs=sc.einsum('ij,kil->kjl',sc.conj(pkr),OV)
sqn=sc.einsum('ijk,ikl->ijlk',lhs,rhs)
return sqn
def co_occurrence(mats, mode='dot', verbose=False):
import numpy as np
# feature co_occurrence within one multi-dimensional features
if len(mats) == 1:
x = mats[0]
if mode == 'dot':
cont = x.T.dot(x) # equivalent of x' * y
elif mode == 'corr':
cont = np.corrcoef(x, rowvar=0)
elif mode == 'sparse':
x = np.sparse.csr_matrix(x)
cont = x.T.dot(x)
cont = cont.todense()
else:
raise NameError('mode does not exist')
# feature co_occurrence across two multidimensional features
if len(mats) == 2:
x, y = mats[:]
if mode == 'dot':
cont = x.T.dot(y)
elif mode == 'corr':
x = (x - np.mean(x, axis=0)) / np.std(x, axis=0)
y = (y - np.mean(y, axis=0)) / np.std(y, axis=0)
cont = x.T.dot(y)
elif mode == 'sparse':
x = np.sparse.csr_matrix(x)
y = np.sparse.csr_matrix(y)
cont = x.T.dot(y)
cont = cont.todense()
else:
raise NameError('mode does not exist')
# feature co_occurrence across three multidimensional features
if len(mats) == 3:
x, y, z = mats[:]
if mode == 'dot':
cont = einsum('ni,nj,nk', x, y, z)
elif mode == 'corr':
x = (x - np.mean(x, axis=0)) / np.std(x, axis=0)
y = (y - np.mean(y, axis=0)) / np.std(y, axis=0)
z = (z - np.mean(z, axis=0)) / np.std(z, axis=0)
cont = einsum('ni,nj,nk', x, y, z)
else:
raise NameError('mode does not exist')
return np.array(cont) # we don't want returns of type matrix
def co_occurrence(mats, mode='dot', verbose=False):
import numpy as np
# feature co_occurrence within one multi-dimensional features
if len(mats) == 1:
x = mats[0]
if mode == 'dot':
cont = x.T.dot(x) # equivalent of x' * y
elif mode == 'corr':
cont = np.corrcoef(x, rowvar=0)
elif mode == 'sparse':
x = np.sparse.csr_matrix(x)
cont = x.T.dot(x)
cont = cont.todense()
else:
raise NameError('mode does not exist')
# feature co_occurrence across two multidimensional features
if len(mats) == 2:
x, y = mats[:]
if mode == 'dot':
cont = x.T.dot(y)
elif mode == 'corr':
x = (x - np.mean(x, axis=0)) / np.std(x, axis=0)
y = (y - np.mean(y, axis=0)) / np.std(y, axis=0)
cont = x.T.dot(y)
elif mode == 'sparse':
x = np.sparse.csr_matrix(x)
y = np.sparse.csr_matrix(y)
cont = x.T.dot(y)
cont = cont.todense()
else:
raise NameError('mode does not exist')
# feature co_occurrence across three multidimensional features
if len(mats) == 3:
x, y, z = mats[:]
if mode == 'dot':
cont = einsum('ni,nj,nk', x, y, z)
elif mode == 'corr':
x = (x - np.mean(x, axis=0)) / np.std(x, axis=0)
y = (y - np.mean(y, axis=0)) / np.std(y, axis=0)
z = (z - np.mean(z, axis=0)) / np.std(z, axis=0)
cont = einsum('ni,nj,nk', x, y, z)
else:
raise NameError('mode does not exist')
return np.array(cont) # we don't want returns of type matrix
def blocks():
bs = 10000
for i in range(0, n, bs):
Y = X[i:i + bs].todense()
Y = sp.einsum('ij,ik->ijk', Y, Y).reshape(Y.shape[0], -1)
Y = Y @ M
yield Y
def grad_f1(self, a):
"Define the gradient for each convex inequality."
# Initialize the output vector
out = sp.zeros((self.M, self.Na))
# Preliminary calculation
_xx = sp.einsum('mi,mj->mij', self.xarray, self.xarray)
# Compute the four terms
_Da = sp.tensordot(self.D, a, axes=[(0,), (0,)])
_DDa = sp.tensordot(self.D, _Da, axes=[(1,), (0,)])
xxDDa = sp.tensordot(_xx.reshape(self.M, self.ndim**2),
_DDa.reshape(self.Na, self.ndim**2),
axes=[(-1,), (-1,)])
_BDa = sp.dot(self.B, _Da)
xBDa = sp.inner(self.xarray, _BDa)
_Ba = sp.dot(a, self.B)
_DBa = sp.dot(_Ba, self.D)
xDBa = sp.tensordot(self.xarray,
_DBa, axes=[(-1,), (-1,)])
BBa = sp.dot(self.B, _Ba)
# compute the gradient by summing the four terms
out[:, :] = 2.0 * (xxDDa + xBDa + xDBa + BBa)
return out
def phiktrans(kx,ky,qx,qy,p,r=sc.zeros((1,2))):
"""
Returns phi[k,r] such that |q,r>=sum_k phi[k,r]|q,k>
"""
kqx=kx-qx
kqy=ky-qy
pk=sc.zeros((sc.shape(kx)[0],sc.shape(r)[0]),complex)
pke=sc.conj(uk(kx,ky,1,1,p))*uk(kqx,kqy,-1,-1,p)+\
sc.conj(vk(kx,ky,1,1,p))*vk(kqx,kqy,-1,-1,p)
pko=sc.conj(vk(kx,ky,1,1,p))*uk(kqx,kqy,-1,-1,p)+\
sc.conj(uk(kx,ky,1,1,p))*vk(kqx,kqy,-1,-1,p)
even=1-sc.mod(r[:,0]+r[:,1],2)
odd=sc.mod(r[:,0]+r[:,1],2)
ph=sc.exp(-2j*sc.pi*(sc.einsum('i,j->ij',kx,r[:,0])+sc.einsum('i,j->ij',ky,r[:,1])))
pk=sc.einsum('ij,j,i->ij',ph,even,pke)+sc.einsum('ij,j,i->ij',ph,odd,pko)
return pk
def barycoords(tri, faces, points):
# Because python and matlab can have different point order within triangles, column order is not guaranteed
# Faces should be in format triangle indices, points in npts x dim array
X = tri.transform[faces, :-1]
Y = points - tri.transform[faces, -1]
b = sp.einsum('ijk,ik->ij', X, Y)
return sp.c_[b, 1 - b.sum(axis=1)]
def pc_abd(a, b, d, params, model=1):
warnings.filterwarnings('ignore')
if not isinstance(a, collections.Iterable):
a = np.array([a])
elif type(a) != np.ndarray:
a = np.array(a)
if not isinstance(b, collections.Iterable):
b = np.array([b])
elif type(b) != np.ndarray:
b = np.array(b)
if not isinstance(d, collections.Iterable):
d = np.array([d])
elif type(d) != np.ndarray:
d = np.array(d)
c_ab_prob, val = pc_ab(a, b, params, model)
d_c_prob, _ = pd_c(val, params, model)
d_c_prob = d_c_prob[d, :].T
prob = einsum('ijk,il->ijkl',
c_ab_prob,
d_c_prob,
optimize='optimal',
dtype=np.float64)
prob /= prob.sum(axis=0)
prob[np.where(prob < 0)] = 0
prob[np.isnan(prob)] = 0
return (prob, val)
def loss( self, y, X, B ):
"""Residual in error"""
N = len( y )
tot = 0
for (x_i, y_i) in zip( X, y ):
tot += (einsum( "ijk,i,j,k", B, x_i, x_i, x_i) - y_i)**2
return tot/N
def phiktrans(kx, ky, qx, qy, p, r=sc.zeros((1, 2))):
"""
Returns phi[k,r] such that |q,r>=sum_k phi[k,r]|q,k>
"""
kqx = kx - qx
kqy = ky - qy
pk = sc.zeros((sc.shape(kx)[0], sc.shape(r)[0]), complex)
pke=sc.conj(uk(kx,ky,1,1,p))*uk(kqx,kqy,-1,-1,p)+\
sc.conj(vk(kx,ky,1,1,p))*vk(kqx,kqy,-1,-1,p)
pko=sc.conj(vk(kx,ky,1,1,p))*uk(kqx,kqy,-1,-1,p)+\
sc.conj(uk(kx,ky,1,1,p))*vk(kqx,kqy,-1,-1,p)
even = 1 - sc.mod(r[:, 0] + r[:, 1], 2)
odd = sc.mod(r[:, 0] + r[:, 1], 2)
ph = sc.exp(-2j * sc.pi * (sc.einsum('i,j->ij', kx, r[:, 0]) +
sc.einsum('i,j->ij', ky, r[:, 1])))
pk = sc.einsum('ij,j,i->ij', ph, even, pke) + sc.einsum(
'ij,j,i->ij', ph, odd, pko)
return pk
def solve_with_moments(M1, M2, M3, K):
W, Wt = get_whitener(M2, K)
M3_ = sc.einsum('ijk,ia,jb,kc->abc', M3, W, W, W)
#print "M3", M3
pi_, M_, _, _ = candecomp(M3_, K)
#print "mu", M_
mu = Wt.dot(M_.dot(np.diag(pi_)))
return mu
def solve_with_moments(M1, M2, M3, K):
W, Wt = get_whitener( M2, K )
M3_ = sc.einsum( 'ijk,ia,jb,kc->abc', M3, W, W, W )
#print "M3", M3
pi_, M_, _, _ = candecomp(M3_, K)
#print "mu", M_
mu = Wt.dot(M_.dot(np.diag(pi_)))
return mu
def pc(params, model=1):
prob, val = pc_ab(np.arange(params['amin'], params['amax'] + 1),
np.arange(params['bmin'], params['bmax'] + 1), params,
model)
prob = einsum('ijk->i', prob) / \
(params['amax'] - params['amin'] + 1) / \
(params['bmax'] - params['bmin'] + 1)
prob[np.where(prob < 0)] = 0
prob[np.isnan(prob)] = 0
return (prob, val)
def RenormalizeFactor(excfile,gsfile,channel=None,Nsamp=1,O=None,q=None):
if not type(excfile)==list:
excfile=[excfile]
if not type(gsfile)==list:
gsfile=[gsfile]
exat=GetAttr(excfile[0])
gsat=GetAttr(gsfile[0])
L=exat['L']
if q==None:
q=sc.array([exat['qx'],exat['qy']])
if 'phasex' in exat.keys():
shift=sc.array([exat['phasex']/2.0,exat['phasey']/2.0])
else:
shift=sc.array([exat['phase_shift_x']/2.0,exat['phase_shift_y']/2.0])
phi=exat['phi']
neel=exat['neel']
qx,qy,Sq=GetSq(gsfile)
kx,ky=sf.fermisea(L,L,shift)
qidx=ml.find((qx==q[0])*(qy==q[1]))
if O==None:
_,O,_,_=GetEigSys(excfile,Nsamp)
pk=None
sqq=None
if channel==None:
channel=exat['channel']
if channel=='trans':
pk=sc.squeeze(sf.phiktrans(kx,ky,q[0]/L,q[1]/L,[phi,neel]))
sqq=sc.real(0.5*(Sq[0,1,qidx]+Sq[0,2,qidx]))
elif channel=='long':
pkup=sc.squeeze(sf.phiklong(kx,ky,q[0]/L,q[1]/L,1,[phi,neel]))
pkdo=sc.squeeze(sf.phiklong(kx,ky,q[0]/L,q[1]/L,-1,[phi,neel]))
if (q[0]/L==0.5 and q[1]/L==0.5) or (q[0]/L==0 and q[1]/L==0):
pk=sc.zeros(2*sc.shape(pkup)[0]+1,complex)
else:
pk=sc.zeros(2*sc.shape(pkup)[0],complex)
pk[0:2*sc.shape(pkup)[0]:2]=pkup
pk[1:2*sc.shape(pkdo)[0]:2]=pkdo
if (qx[0]/L==0.5 and q[1]/L==0.5) or (q[0]/L==0 and q[1]/L==0):
if neel==0:
pk[-1]=0
else:
pk[-1]=sum(neel/sf.omega(kx,ky,[phi,neel]))
sqq=Sq[0,0,qidx]
else:
raise(InputFileError('In file \''+excfile+'\', channel=\''+str(channel)+'\'. Should be \'trans\' or \'long\''))
sqe=sc.einsum('i,jik,k->j',sc.conj(pk),O,pk)
out=sc.zeros(Nsamp)
for n in range(Nsamp):
if abs(sqq)<1e-6 or abs(sqe[n])<1e-6:
warnings.warn('Probably ill-defined renormalization, returns 1 for sample {0} out of {1}'.format(n,Nsamp),UserWarning)
out[n]=1
else:
out[n]=sc.real(sqq/sqe[n])
return out
def process(self, G, verbose=False):
r"""
Fit genotypes one-by-one.
Parameters
----------
G : (`N`, `S`) ndarray
genotype vector for `N` individuals and `S` variants.
verbose : bool
verbose flag.
"""
t0 = time.time()
# precompute some stuff
if self.cov == None: KiG = G
else: KiG = self.cov.solve(G)
GKiy = sp.dot(G.T, self.Kiy[:, 0])
GKiG = sp.einsum('ij,ij->j', G, KiG)
FKiG = sp.dot(self.F.T, KiG)
# Let us denote the inverse of Areml as
# Ainv = [[A0i + m mt / n, m], [mT, n]]
A0iFKiG = sp.dot(self.A0i, FKiG)
n = 1. / (GKiG - sp.einsum('ij,ij->j', FKiG, A0iFKiG))
M = -n * A0iFKiG
self.beta_F = self.beta_F0 + M * sp.dot(M.T, self.FKiy[:, 0]) / n
self.beta_F += M * GKiy
self.beta_g = sp.einsum('is,i->s', M, self.FKiy[:, 0])
self.beta_g += n * GKiy
# sigma
s2 = self.yKiy - sp.einsum('i,is->s', self.FKiy[:, 0], self.beta_F)
s2 -= GKiy * self.beta_g
s2 /= self.df
#dlml and pvs
self.lrt = -self.df * sp.log(s2 / self.s20)
self.pv = st.chi2(1).sf(self.lrt)
t1 = time.time()
if verbose:
print('Tested for %d variants in %.2f s' % (G.shape[1], t1 - t0))
def sqwlongamp(V, O, Lx, Ly, q, shift, phi, neel):
sqn = sc.zeros(sc.shape(V)[0:2], complex)
kx, ky = fermisea(Lx, Ly, shift)
pkup = phiklong(kx, ky, q[0], q[1], 1, [phi, neel])
pkdo = phiklong(kx, ky, q[0], q[1], -1, [phi, neel])
pk = sc.zeros(sc.shape(V)[1], complex)
pk[0:2 * len(pkup):2] = pkup
pk[1:2 * len(pkup):2] = pkdo
if (abs(q[0])+abs(q[1]))<1e-6 or\
(abs(q[0]-0.5)+abs(q[1]-0.5))<1e-6:
if neel != 0:
pk[-1] = sc.sum(neel / omega(kx, ky, [phi, neel]))
sqn = abs(sc.einsum('ijk,ijl,l->ik', sc.conj(V), O, pk))**2
return sqn
def test_mtmm_scan_pv_beta():
import scipy as sp
import scipy.linalg as la
from limix_core.gp import GP2KronSum
from limix_core.covar import FreeFormCov
N = 200
P = 4
M = 2
K = 2
S = 10
Y, F, G, B0, Cg0, Cn0 = _generate_data(N, P, K, S)
A = sp.eye(P)
Asnp = sp.rand(P, M)
# compute eigenvalue decomp of RRM
R = sp.dot(G, G.T)
R /= R.diagonal().mean()
R += 1e-4 * sp.eye(R.shape[0])
Sr, Ur = la.eigh(R)
# fit null model
Cg = FreeFormCov(Y.shape[1])
Cn = FreeFormCov(Y.shape[1])
gp = GP2KronSum(Y=Y, S_R=Sr, U_R=Ur, Cg=Cg, Cn=Cn, F=F, A=sp.eye(P))
gp.covar.Cg.setCovariance(0.5 * sp.cov(Y.T))
gp.covar.Cn.setCovariance(0.5 * sp.cov(Y.T))
gp.optimize(factr=10)
# run MTLMM
from limix_lmm import MTLMM
mtlmm = MTLMM(Y, F=F, A=A, Asnp=Asnp, covar=gp.covar)
pv, B = mtlmm.process(G)
# run standard LMMcore
from limix_lmm.lmm_core import LMMCore
y = sp.reshape(Y, [Y.size, 1], order="F")
covs = sp.kron(A, F)
Aext = sp.kron(Asnp, sp.ones((G.shape[0], 1)))
Gext = sp.kron(sp.ones((Asnp.shape[0], 1)), G)
Wext = sp.einsum("ip,in->inp", Aext, Gext).reshape(Aext.shape[0], -1)
stlmm = LMMCore(y, covs, Ki_dot=gp.covar.solve)
stlmm.process(Wext, step=Asnp.shape[1])
pv0 = stlmm.getPv()
B0 = stlmm.getBetaSNP()
assert_allclose(pv0, pv, rtol=1e-06, atol=1e-06)
assert_allclose(B0, B, rtol=1e-06, atol=1e-06)
def sqwlongamp(V,O,Lx,Ly,q,shift,phi,neel):
sqn=sc.zeros(sc.shape(V)[0:2],complex)
kx,ky=fermisea(Lx,Ly,shift)
pkup=phiklong(kx,ky,q[0],q[1],1,[phi,neel])
pkdo=phiklong(kx,ky,q[0],q[1],-1,[phi,neel])
pk=sc.zeros(sc.shape(V)[1],complex)
pk[0:2*len(pkup):2]=pkup
pk[1:2*len(pkup):2]=pkdo
if (abs(q[0])+abs(q[1]))<1e-6 or\
(abs(q[0]-0.5)+abs(q[1]-0.5))<1e-6:
if neel!=0:
pk[-1]=sc.sum(neel/omega(kx,ky,[phi,neel]))
sqn=abs(sc.einsum('ijk,ijl,l->ik',sc.conj(V),O,pk))**2
return sqn
def sqwtransamp(V,
O,
Lx,
Ly,
q,
shift,
phi,
neel,
r=sc.zeros((1, 2)),
rp=sc.zeros((1, 2))):
"""
Returns Sq[sample,r,rp,n]=<q,r|q,n><q,n|q,rp>
"""
sqn = sc.zeros(sc.shape(V)[0:2], complex)
kx, ky = fermisea(Lx, Ly, shift)
pkrp = sc.zeros((sc.shape(V)[1], sc.shape(rp)[0]), complex)
pkr = sc.zeros((sc.shape(V)[1], sc.shape(r)[0]), complex)
pkrp[0:len(kx), :] = phiktrans(kx, ky, q[0], q[1], [phi, neel], rp)
pkr[0:len(ky), :] = phiktrans(kx, ky, q[0], q[1], [phi, neel], r)
OV = sc.einsum('ijk,ikl->ijl', O, V)
rhs = sc.einsum('ijk,jl->ikl', sc.conj(OV), pkrp)
lhs = sc.einsum('ij,kil->kjl', sc.conj(pkr), OV)
sqn = sc.einsum('ijk,ikl->ijlk', lhs, rhs)
return sqn
def rotate_wind(U,V,alpha):
alpha = sp.array(alpha)*sp.pi/180
alpha = alpha.flatten()
R = sp.array([[sp.cos(alpha), -sp.sin(alpha)], [sp.sin(alpha), sp.cos(alpha)] ])
shpe = U.shape
origwind = sp.array((U.flatten(), V.flatten()))
if len(R.shape)==2:
rotwind = dot(R, origwind) # for constant rotation angle
else:
# for rotation angle given as array with same dimensions as U and V:
# k-loop with rotwind(k) = dot(R(i,j,k), origwind(j,k)) (einstein summation indices)
rotwind = sp.einsum("ijk,ik -> jk", R, origwind)
Urot ,Vrot = rotwind[0,:], rotwind[1,:]
Urot = Urot.reshape(shpe)
Vrot = Vrot.reshape(shpe)
return Urot, Vrot
def solve_with_moments(m1, M2, M3, K):
"""
Whiten and unwhiten appropriately
"""
assert symmetric_skew(M2) < 1e-2
assert symmetric_skew(M3) < 1e-2
W, Wt = get_whitener( M2, K )
M3_ = sc.einsum( 'ijk,ia,jb,kc->abc', M3, W, W, W )
#print "M3", M3
pi_, M_, _, _ = candecomp(M3_, K)
#print "mu", M_
mu = Wt.dot(M_.dot(diag(pi_)))
return mu
def recover_B( k, y, X, iters = 50, alpha0 = 0.1, reg = 0, B20B30 = (None,None), B2B3 = (None, None) ):
"""Recover the mixture weights B"""
B20, B30 = B20B30
B2, B3 = B2B3
# Use convex optimisation to recover B2 and B3
B2_ = PhaseRecovery().solve( y**2, X, B0 = B20, alpha = "1/sqrt(T)", alpha0 = alpha0, iters = iters, reg = reg, verbose = False )
print norm( B20 - B2 ), norm( B2_ - B2 )
B3_ = TensorRecovery().solve( y**3, X, B0 = B30, alpha = "1/sqrt(T)", alpha0 = alpha0, iters = iters, reg = reg, verbose = False )
print norm( B30 - B3 ), norm( B3_ - B3 )
B3_ = lambda theta: sc.einsum( 'abj,j->ab', B3, theta )
B = recover_M3( k, B2, B2, B3_ )
return B, B2, B3
def gaussians(x,x0,A,sig):
#if sc.amax(abs(sc.imag(A)))/sc.amax(abs(sc.real(A)))>0.01:
# warnings.warn(\
#'Gaussian amplitude has a sizable imaginary part\(max(|Im|)/max(|Re|)={0}, mean(abs(A))={1}).'\
# .format(sc.amax(abs(sc.imag(A)))/sc.amax(abs(sc.real(A))), sc.mean(abs(A))))
x0=sc.atleast_1d(x0)
A=sc.atleast_1d(A)
sig=sc.atleast_1d(sig)
amp=A*sc.sqrt(1/2.0/sc.pi)/sig
[X,X0]=sc.meshgrid(x,x0)
gg=None
#if len(sc.shape(amp))==1:
# gg=sc.einsum('i,ij',amp,sc.exp(-0.5*(X-X0)**2/sc.tile(sig**2,(sc.shape(x)[0],1)).T))
#elif len(sc.shape(amp))==2:
# gg=sc.einsum('ij,jk',amp,sc.exp(-0.5*(X-X0)**2/sc.tile(sig**2,(sc.shape(x)[0],1)).T))
gg=sc.einsum('...i,...ij->...j',amp,sc.exp(-0.5*(X-X0)**2/sc.tile(sig**2,(sc.shape(x)[0],1)).T))
return gg
def rotate_vectorfield(U,V,alpha):
'''rotate wind vectors clockwise. alpha may be a scalar or an array
alpha is in degrees
returns u,v '''
alpha = sp.array(alpha)*sp.pi/180
alpha = alpha.flatten()
R = sp.array([[sp.cos(alpha), -sp.sin(alpha)], [sp.sin(alpha), sp.cos(alpha)] ])
shpe = U.shape
origwind = sp.array((U.flatten(), V.flatten()))
if len(R.shape)==2:
rotwind = dot(R, origwind) # for constant rotation angle
else:
# for rotation angle given as array with same dimensions as U and V:
# k-loop with rotwind(k) = dot(R(i,j,k), origwind(j,k)) (einstein summation indices)
rotwind = sp.einsum("ijk,ik -> jk", R, origwind) # einstein summation indices
Urot ,Vrot = rotwind[0,:], rotwind[1,:]
Urot = Urot.reshape(shpe)
Vrot = Vrot.reshape(shpe)
return Urot, Vrot
def rotate_vectorfield(U,V,alpha):
'''rotate wind vectors clockwise. alpha may be a scalar or an array
alpha is in degrees
returns u,v '''
alpha = sp.array(alpha)*sp.pi/180
alpha = alpha.flatten()
R = sp.array([[sp.cos(alpha), -sp.sin(alpha)], [sp.sin(alpha), sp.cos(alpha)] ])
shpe = U.shape
origwind = sp.array((U.flatten(), V.flatten()))
if len(R.shape)==2:
rotwind = dot(R, origwind) # for constant rotation angle
else:
# for rotation angle given as array with same dimensions as U and V:
# k-loop with rotwind(k) = dot(R(i,j,k), origwind(j,k)) (einstein summation indices)
rotwind = sp.einsum("ijk,ik -> jk", R, origwind) # einstein summation indices
Urot ,Vrot = rotwind[0,:], rotwind[1,:]
Urot = Urot.reshape(shpe)
Vrot = Vrot.reshape(shpe)
return Urot, Vrot
def gaussians(x, x0, A, sig):
#if sc.amax(abs(sc.imag(A)))/sc.amax(abs(sc.real(A)))>0.01:
# warnings.warn(\
#'Gaussian amplitude has a sizable imaginary part\(max(|Im|)/max(|Re|)={0}, mean(abs(A))={1}).'\
# .format(sc.amax(abs(sc.imag(A)))/sc.amax(abs(sc.real(A))), sc.mean(abs(A))))
x0 = sc.atleast_1d(x0)
A = sc.atleast_1d(A)
sig = sc.atleast_1d(sig)
amp = A * sc.sqrt(1 / 2.0 / sc.pi) / sig
[X, X0] = sc.meshgrid(x, x0)
gg = None
#if len(sc.shape(amp))==1:
# gg=sc.einsum('i,ij',amp,sc.exp(-0.5*(X-X0)**2/sc.tile(sig**2,(sc.shape(x)[0],1)).T))
#elif len(sc.shape(amp))==2:
# gg=sc.einsum('ij,jk',amp,sc.exp(-0.5*(X-X0)**2/sc.tile(sig**2,(sc.shape(x)[0],1)).T))
gg = sc.einsum(
'...i,...ij->...j', amp,
sc.exp(-0.5 * (X - X0)**2 / sc.tile(sig**2, (sc.shape(x)[0], 1)).T))
return gg
def solve_mixture_model(model, data):
"""
Whiten and unwhiten appropriately
"""
d = model["d"]
# Get moments
moments = model.empirical_moments(data, model.observed_monomials(3))
M2 = zeros((d, d))
M3 = zeros((d, d, d))
for i in xrange(d):
for j in xrange(d):
xij = sp.sympify('x%d * x%d' % (i + 1, j + 1))
M2[i, j] = moments[xij]
for k in xrange(d):
xijk = sp.sympify('x%d * x%d * x%d' % (i + 1, j + 1, k + 1))
M3[i, j, k] = moments[xijk]
k = model["k"]
# Symmetrize
M2, M3 = symmetrize(M2), symmetrize(M3)
assert symmetric_skew(M2) < 1e-2
assert symmetric_skew(M3) < 1e-2
# Whiten
W, Wt = get_whitener(M2, k)
M3_ = einsum('ijk,ia,jb,kc->abc', M3, W, W, W)
pi_, M_, _, _ = candecomp(M3_, k)
# Unwhiten M
M_ = Wt.dot(M_.dot(diag(pi_)))
pi_ = 1. / pi_**2
# "Project" onto simplex
pi_ = make_distribution(abs(pi_))
M_ = array([make_distribution(col) for col in M_.T]).T
return pi_, M_
def solve_mixture_model(model, data):
"""
Whiten and unwhiten appropriately
"""
d = model["d"]
# Get moments
moments = model.empirical_moments(data, model.observed_monomials(3))
M2 = zeros((d, d))
M3 = zeros((d, d, d))
for i in xrange(d):
for j in xrange(d):
xij = sp.sympify('x%d * x%d' %(i+1, j+1))
M2[i,j] = moments[xij]
for k in xrange(d):
xijk = sp.sympify('x%d * x%d * x%d' % (i+1, j+1, k+1))
M3[i,j,k] = moments[xijk]
k = model["k"]
# Symmetrize
M2, M3 = symmetrize(M2), symmetrize(M3)
assert symmetric_skew(M2) < 1e-2
assert symmetric_skew(M3) < 1e-2
# Whiten
W, Wt = get_whitener(M2, k)
M3_ = einsum('ijk,ia,jb,kc->abc', M3, W, W, W)
pi_, M_, _, _ = candecomp(M3_, k)
# Unwhiten M
M_ = Wt.dot(M_.dot(diag(pi_)))
pi_ = 1./pi_**2
# "Project" onto simplex
pi_ = make_distribution(abs(pi_))
M_ = array([make_distribution(col) for col in M_.T]).T
return pi_, M_
def test_solve_exact(samples = 1e4, iters = 1e2):
K = 2
d = 3
N = samples
pi = array( [0.5, 0.5] )
B = eye( d, K )
B3 = sum( [ pi[i] * tensorify(B.T[i], B.T[i], B.T[i] ) for i in xrange(K) ] )
X = sc.randn( N, d )
y = array( [ einsum( "ijk,i,j,k", B3, x, x, x ) for x in X ] )
algo = TensorRecovery()
B3_ = algo.solve( y, X, iters = iters, alpha0 = 0.01, eps=1e-5, reg = 1e-3 )
_, _, V = HOSVD( B3 )
print algo.loss( y, X, B3 ), V
__, _, V_ = HOSVD( B3_ )
print algo.loss( y, X, B3_ ), V_
print norm(B3 - B3_)/norm(B3)
assert norm(B3 - B3_)/norm(B3) < 1e-1
def RenormalizeFactor(excfile, gsfile, channel=None, Nsamp=1, O=None, q=None):
if not type(excfile) == list:
excfile = [excfile]
if not type(gsfile) == list:
gsfile = [gsfile]
exat = GetAttr(excfile[0])
gsat = GetAttr(gsfile[0])
L = exat['L']
if q == None:
q = sc.array([exat['qx'], exat['qy']])
if 'phasex' in exat.keys():
shift = sc.array([exat['phasex'] / 2.0, exat['phasey'] / 2.0])
else:
shift = sc.array(
[exat['phase_shift_x'] / 2.0, exat['phase_shift_y'] / 2.0])
phi = exat['phi']
neel = exat['neel']
qx, qy, Sq = GetSq(gsfile)
kx, ky = sf.fermisea(L, L, shift)
qidx = ml.find((qx == q[0]) * (qy == q[1]))
if O == None:
_, O, _, _ = GetEigSys(excfile, Nsamp)
pk = None
sqq = None
if channel == None:
channel = exat['channel']
if channel == 'trans':
pk = sc.squeeze(sf.phiktrans(kx, ky, q[0] / L, q[1] / L, [phi, neel]))
sqq = sc.real(0.5 * (Sq[0, 1, qidx] + Sq[0, 2, qidx]))
elif channel == 'long':
pkup = sc.squeeze(
sf.phiklong(kx, ky, q[0] / L, q[1] / L, 1, [phi, neel]))
pkdo = sc.squeeze(
sf.phiklong(kx, ky, q[0] / L, q[1] / L, -1, [phi, neel]))
if (q[0] / L == 0.5 and q[1] / L == 0.5) or (q[0] / L == 0
and q[1] / L == 0):
pk = sc.zeros(2 * sc.shape(pkup)[0] + 1, complex)
else:
pk = sc.zeros(2 * sc.shape(pkup)[0], complex)
pk[0:2 * sc.shape(pkup)[0]:2] = pkup
pk[1:2 * sc.shape(pkdo)[0]:2] = pkdo
if (qx[0] / L == 0.5 and q[1] / L == 0.5) or (q[0] / L == 0
and q[1] / L == 0):
if neel == 0:
pk[-1] = 0
else:
pk[-1] = sum(neel / sf.omega(kx, ky, [phi, neel]))
sqq = Sq[0, 0, qidx]
else:
raise (InputFileError('In file \'' + excfile + '\', channel=\'' +
str(channel) +
'\'. Should be \'trans\' or \'long\''))
sqe = sc.einsum('i,jik,k->j', sc.conj(pk), O, pk)
out = sc.zeros(Nsamp)
for n in range(Nsamp):
if abs(sqq) < 1e-6 or abs(sqe[n]) < 1e-6:
warnings.warn(
'Probably ill-defined renormalization, returns 1 for sample {0} out of {1}'
.format(n, Nsamp), UserWarning)
out[n] = 1
else:
out[n] = sc.real(sqq / sqe[n])
return out
def GetEigSys(filename,
gsfile=None,
Nsamp=1,
channel=None,
wavefile=None,
q=None):
if type(filename) == str:
filename = [filename]
hfile = h5py.File(filename[0], 'r')
attr = GetAttr(filename[0])
if channel == None:
channel = attr['channel']
dpath, args = GetStat(filename, Nsamp)
dat = sc.array(hfile["/rank-1/data-0"])
hfile.close()
N = int(sc.shape(dat)[0] / 2)
L = attr['L']
shift = None
if 'phasex' in attr.keys():
shift = [attr['phasex'] / 2.0, attr['phasey'] / 2.0]
else:
shift = [attr['phase_shift_x'] / 2.0, attr['phase_shift_y'] / 2.0]
H = sc.zeros([Nsamp, N, N], complex)
O = sc.zeros([Nsamp, N, N], complex)
E = sc.zeros([Nsamp, N])
V = sc.zeros([Nsamp, N, N], complex)
for sample, b in enumerate(args):
for d in b:
hfile = h5py.File(dpath[d][0], 'r')
dat = hfile[dpath[d][1]]
H[sample, :, :] += dat[0:N, 0:2 * N:2] + 1j * dat[0:N, 1:2 * N:2]
O[sample, :, :] += dat[N:2 * N,
0:2 * N:2] + 1j * dat[N:2 * N, 1:2 * N:2]
hfile.close()
H[sample, :, :] = 0.5 * (H[sample, :, :] +
sc.conj(H[sample, :, :].T)) / len(b)
O[sample, :, :] = 0.5 * (O[sample, :, :] +
sc.conj(O[sample, :, :].T)) / len(b)
if channel == 'groundstate':
return H
fs = None
refstate = sc.zeros(2 * L * L)
refstate[0::2] = 1
if wavefile == None:
fs = GetFermiSigns(filename[0], refstate, channel=channel)
else:
fs = GetFermiSigns(wavefile, refstate, channel=channel)
for s in range(sc.shape(H)[0]):
H[s, :, :] = sc.dot(sc.diag(fs), sc.dot(H[s, :, :], sc.diag(fs)))
O[s, :, :] = sc.dot(sc.diag(fs), sc.dot(O[s, :, :], sc.diag(fs)))
ren = sc.ones(Nsamp)
if gsfile != None:
ren = RenormalizeFactor(filename,
gsfile,
Nsamp=1,
channel=channel,
O=O,
q=q)
print('{0} pair of (H,O) matrices loaded, now diagonalize'.format(
sc.shape(H)[0]))
H = sc.einsum('ijk,i->ijk', H, ren)
O = sc.einsum('ijk,i->ijk', O, ren)
for s in range(sc.shape(H)[0]):
E[s, :], V[s, :, :] = vln.geneigh(sc.squeeze(H[s, :, :]),
sc.squeeze(O[s, :, :]))
print('diagonalization finished')
return H, O, E, V
# H = 0.5 * trace_term - quadratic_term
# expected_H = -0.5 * trace_term
# average_H = quadratic_term
# 1. trace_term = tr(Ki_dKp_Ki_dKp)
rankCr = 2
T = []
KiT = []
for i in range(3):
_S, _U = la.eigh(_Cr.K_grad_i(i))
Cr_h = _U * sp.sqrt(_S)
T.append(
sp.reshape(sp.kron(Cr_h, Xr), (N, P, rankCr * S), order='F'))
KiT.append(gp.covar.solve_t(T[i]))
Ht = sp.zeros((3, 3))
for i in range(3):
Ht[i, i] = (sp.einsum('qpn,qpm->nm', T[i], KiT[i])**2).sum()
for j in range(0, i):
Ht[i, j] = (sp.einsum('qpn,qpm->nm', T[i], KiT[j])**2).sum()
Ht[j, i] = Ht[i, j]
if Y.shape[0] <= 1000:
Ki = la.inv(
sp.kron(mvset.gp[type].covar.Cn.K(), sp.eye(Y.shape[0])))
XrXr = sp.dot(Xr, Xr.T)
KidK = [
sp.dot(Ki, sp.kron(_Cr.K_grad_i(i), XrXr)) for i in range(3)
]
Ht0 = sp.zeros((3, 3))
for i in range(3):
for j in range(3):
Ht0[i, j] = sp.trace(sp.dot(KidK[i], KidK[j]))
pdb.set_trace()
def transspinonoverlap(O,V,Lx,Ly,q,shift,phi,neel,r):
kx,ky=fermisea(Lx,Ly,shift)
pkr=phiktrans(kx,ky,q[0],q[1],[phi,neel],r)
ork=sc.einsum('ij,kil->kjl',sc.conj(pkr),O)
return sc.einsum('kil,klm->kim',ork,V)
if 0:
# H = 0.5 * trace_term - quadratic_term
# expected_H = -0.5 * trace_term
# average_H = quadratic_term
# 1. trace_term = tr(Ki_dKp_Ki_dKp)
rankCr = 2
T = []
KiT = []
for i in range(3):
_S, _U = la.eigh(_Cr.K_grad_i(i))
Cr_h = _U*sp.sqrt(_S)
T.append(sp.reshape(sp.kron(Cr_h, Xr), (N,P,rankCr*S), order='F'))
KiT.append(gp.covar.solve_t(T[i]))
Ht = sp.zeros((3,3))
for i in range(3):
Ht[i,i] = (sp.einsum('qpn,qpm->nm', T[i], KiT[i])**2).sum()
for j in range(0,i):
Ht[i,j] = (sp.einsum('qpn,qpm->nm', T[i], KiT[j])**2).sum()
Ht[j,i] = Ht[i,j]
if Y.shape[0]<=1000:
Ki = la.inv(sp.kron(mvset.gp[type].covar.Cn.K(), sp.eye(Y.shape[0])))
XrXr = sp.dot(Xr, Xr.T)
KidK = [sp.dot(Ki, sp.kron(_Cr.K_grad_i(i), XrXr)) for i in range(3)]
Ht0 = sp.zeros((3,3))
for i in range(3):
for j in range(3):
Ht0[i,j] = sp.trace(sp.dot(KidK[i], KidK[j]))
pdb.set_trace()
if 0:
# 2. quadratic_term = y_Ki_dKp_Ki_dKq_Ki_y
def transspinonoverlap(O, V, Lx, Ly, q, shift, phi, neel, r):
kx, ky = fermisea(Lx, Ly, shift)
pkr = phiktrans(kx, ky, q[0], q[1], [phi, neel], r)
ork = sc.einsum('ij,kil->kjl', sc.conj(pkr), O)
return sc.einsum('kil,klm->kim', ork, V)
def get_yKiy(self):
return sp.einsum("ip,ip->", self.LY, self._Y)
def _pdf_single(self, x):
distance = self.X_arr - x
cov_distance = sp.einsum("ij,ijk,ik->i",
distance, self.inv_covs, distance)
return sp.average(sp.exp(-.5 * cov_distance) / self.normalization,
weights=self.w)