def crossvalidate(X,Y, f=5, trainfun=train_ncc):
'''
Test generalization performance of a linear classifier by crossvalidation
Definition: crossvalidate(X,Y, f=5, trainfun=train_ncc)
Input: X - DxN array of N data points with D features
Y - 1D array of length N of class labels
f - number of cross-validation folds
trainfun - function for linear classification training
Output: acc_train - (f,) array of accuracies in test train folds
acc_test - (f,) array of accuracies in each test fold
'''
N = f*(X.shape[-1]/f)
idx = sp.reshape(sp.arange(N),(f,N/f))
acc_train = sp.zeros((f))
acc_test = sp.zeros((f))
for ifold in sp.arange(f):
testidx = sp.zeros((f),dtype=bool)
testidx[ifold] = 1
test = idx[testidx,:].flatten()
train = idx[~testidx,:].flatten()
w,b = trainfun(X[:,train],Y[train])
acc_train[ifold] = sp.sum(sp.sign(w.dot(X[:,train])-b)==Y[train])/sp.double(train.shape[0])
acc_test[ifold] = sp.sum(sp.sign(w.dot(X[:,test])-b)==Y[test])/sp.double(test.shape[0])
# pdb.set_trace()
return acc_train,acc_test
def _non_dominated_front_arr(iterable, key=lambda x: x, allowequality=True):
"""Return a subset of items from iterable which are not dominated by any
other item in iterable.
Faster version, based on boolean matrix manipulations.
"""
items = list(iterable)
fits = map(key, items)
l = len(items)
x = array(fits)
a = tile(x, (l, 1, 1))
b = a.transpose((1, 0, 2))
if allowequality:
ndom = sum(a <= b, axis=2)
else:
ndom = sum(a < b, axis=2)
ndom = array(ndom, dtype=bool)
res = set()
for ii in range(l):
res.add(ii)
for ij in list(res):
if ii == ij:
continue
if not ndom[ij, ii]:
res.remove(ii)
break
elif not ndom[ii, ij]:
res.remove(ij)
return set(map(lambda i: items[i], res))
def integrator_solve(df):
cum_vec = np.array(np.cumsum(df['ct']))
binheaders = utils.get_column_headers(df)
n_bins = 1000
n_batches = len(binheaders)
f_binned = sp.zeros((n_batches,n_bins))
bins = np.linspace(cum_vec[-1]/1000-1,cum_vec[-1]-1,1000,dtype=int)
for i in range(n_bins):
for j in range(n_batches):
batch_name = binheaders[j]
f_binned[j,i] = scipy.integrate.quad(integrand_1,bins[i],bins[i+1])[0]
f_reg = scipy.ndimage.gaussian_filter1d(f_binned,0.04*n_bins,axis=0)
f_reg = f_reg/f_reg.sum()
# compute marginal probabilities
p_b = sp.sum(f_reg,axis=1)
p_s = sp.sum(f_reg,axis=0)
# finally sum to compute the MI
MI = 0
for j in range(n_batches):
for i in range(n_bins):
if f_reg[i,j] != 0:
MI = MI + f_reg[i,j]*sp.log2(f_reg[i,j]/(p_b[i]*p_s[j]))
return MI
def generate_proc_sim(input_file, weightfile, output_file,
meansub=False, degrade=False):
r"""make the maps with various combinations of beam conv/meansub"""
print "%s -> %s (beam, etc.)" % (input_file, output_file)
simmap = algebra.make_vect(algebra.load(input_file))
if degrade:
print "performing common resolution convolution"
beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792,
0.281176247549, 0.270856788455, 0.26745856078,
0.258910010848, 0.249188429031])
freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905],
dtype=float)
freq_data *= 1.0e6
beam_diff = sp.sqrt(max(1.1 * beam_data) ** 2 - (beam_data) ** 2)
common_resolution = beam.GaussianBeam(beam_diff, freq_data)
# Convolve to a common resolution.
simmap = common_resolution.apply(simmap)
if meansub:
print "performing mean subtraction"
noise_inv = algebra.make_vect(algebra.load(weightfile))
means = sp.sum(sp.sum(noise_inv * simmap, -1), -1)
means /= sp.sum(sp.sum(noise_inv, -1), -1)
means.shape += (1, 1)
simmap -= means
# the weights will be zero in some places
simmap[noise_inv < 1.e-20] = 0.
# extra sanity?
simmap[np.isinf(simmap)] = 0.
simmap[np.isnan(simmap)] = 0.
print "saving to" + output_file
algebra.save(output_file, simmap)
def _do_one_inner_iteration(self, inv_val):
r"""
Determine which throats are invaded at a given applied capillary
pressure.
"""
# Generate a tlist containing boolean values for throat state
Tinvaded = self['throat.entry_pressure'] <= inv_val
# Find all pores that can be invaded at specified pressure
[pclusters, tclusters] = self._net.find_clusters2(mask=Tinvaded,
t_labels=True)
if self._AL:
# Identify clusters connected to invasion sites
inv_clusters = sp.unique(pclusters[self['pore.inlets']])
else:
# All clusters are invasion sites
inv_clusters = pclusters
inv_clusters = inv_clusters[inv_clusters >= 0]
# Find pores on the invading clusters
pmask = np.in1d(pclusters, inv_clusters)
# Store current applied pressure in newly invaded pores
pinds = (self['pore.inv_Pc'] == sp.inf) * (pmask)
self['pore.inv_Pc'][pinds] = inv_val
# Find throats on the invading clusters
tmask = np.in1d(tclusters, inv_clusters)
# Store current applied pressure in newly invaded throats
tinds = (self['throat.inv_Pc'] == sp.inf) * (tmask)
self['throat.inv_Pc'][tinds] = inv_val
# Store total network saturation
tsat = sp.sum(self._net['throat.volume'][self['throat.inv_Pc'] <= inv_val])
psat = sp.sum(self._net['pore.volume'][self['pore.inv_Pc'] <= inv_val])
total = sp.sum(self._net['throat.volume']) + sp.sum(self._net['pore.volume'])
self['pore.inv_sat'][pinds] = (tsat + psat)/total
self['throat.inv_sat'][tinds] = (tsat + psat)/total
def weightedUtest(g1, w1, g2, w2):
""" Determines the confidence level of the assertion:
'The values of g2 are higher than those of g1'.
(adapted from the scipy.stats version)
Twist: here the elements of each group have associated weights,
corresponding to how often they are present (i.e. two identical entries with
weight w are equivalent to one entry with weight 2w).
Reference: "Studies in Continuous Black-box Optimization", Schaul, 2011 [appendix B].
TODO: make more efficient for large sets.
"""
from scipy.stats.distributions import norm
import numpy
n1 = sum(w1)
n2 = sum(w2)
u1 = 0.
for x1, wx1 in zip(g1, w1):
for x2, wx2 in zip(g2, w2):
if x1 == x2:
u1 += 0.5 * wx1 * wx2
elif x1 > x2:
u1 += wx1 * wx2
mu = n1 * n2 / 2.
sigu = numpy.sqrt(n1 * n2 * (n1 + n2 + 1) / 12.)
z = (u1 - mu) / sigu
conf = norm.cdf(z)
return conf
def determine_sign_of_emat(emat,wt_seq):
"""determine what the correct sign is for an energy matrix. We will
use the assumption that the wild type sequence must be better
binding than a random sequence.
INPUTS:
emat: energy matrix
wt_seq: wild type sequence of energy matrix
OUTPUT:
emat: energy matrix with correct sign
"""
n_rand = 1000 # number of random sequences to check
e_rand = sp.zeros(n_rand)
# convert sequence to matrix
seq_mat = seq2mat(wt_seq)
e_wt = sp.sum(emat*seq_mat)
for i in range(n_rand):
seq_rand = sp.zeros((4,len(wt_seq)))
for j in range(len(wt_seq)):
seq_rand[sp.random.randint(4),j] = 1
e_rand[i] = sp.sum(emat*seq_rand)
if e_wt < sp.mean(e_rand):
return emat
else:
return -emat
def get_loss_grad(self,w_vector,*args): X=args[0] Y=args[1] Gobs=args[2] reg_type = args[3] reg_lambda = args[4] wfull = scipy.reshape(w_vector,((shape(X)[1]+1),shape(Y)[1])) B = self.get_energy(X,wfull[:-1,:],wfull[-1,:]) G_pred = scipy.hstack(((exp(B).transpose()*X),scipy.sum(exp(B).transpose(),axis=1))) # Cross entropy: vv = scipy.sum(np.multiply(B,Y),axis=1) # Calculate and subtract the entropy of Y to get kl-divergence Ypl = log(Y) Ypl[Y==0]=0 Ypl = np.multiply(Ypl,Y) vv = vv-scipy.sum(Ypl,axis=1) # Get the mean of the kl-divergence V = sum(vv)/float(shape(X)[0]) G = np.array(Gobs-G_pred).transpose() G = np.reshape(G,size(G))/float(shape(X)[0]) V_reg, G_reg = self.get_regularization_loss_grad(w_vector,X,Y,reg_type,reg_lambda) V += V_reg G += G_reg if self.verbose: print -V, return -V, -np.array(G)
def LML(self,params=None,*kw_args):
"""
calculate LML
"""
if params is not None:
self.setParams(params)
self._update_cache()
start = TIME.time()
#1. const term
lml = self.N*self.P*SP.log(2*SP.pi)
#2. logdet term
lml += SP.sum(SP.log(self.cache['Sc2']))*self.N
lml += 2*SP.log(SP.diag(self.cache['cholB'])).sum()
#3. quatratic term
lml += SP.sum(self.cache['LY']*self.cache['LY'])
lml -= SP.sum(self.cache['WLY']*self.cache['BiWLY'])
lml *= 0.5
smartSum(self.time,'lml',TIME.time()-start)
smartSum(self.count,'lml',1)
return lml
def sgr_sim_density(stripes, params):
SGA = fi.read_data("../sgrnorth_paper/stream_gen_analysis.txt", ",")
low_lam, high_lam = 200.0, 314.8 # from Sgr stripes: (200.025901082, 314.707561185)
size = (high_lam - low_lam) / 15.0
N_stars = [19386, 18734, 11096, 17044, 18736, 15409, 12519, 12248, 8853, 7328,
5479, 4450, 3486, 2425, 971, 9511, 16119, 16603]
south = [79, 82, 86]
out = []
for i in range(15):
print "# - Lambda wedge {0}:".format(i)
num = str(i)
name = "./sim_streams/lamsim_out_"+num+".txt"
data = fi.read_data(name, echo=1)
total = len(data[:,0])
other = sc.sum(data[:,3])
length = 7.653333333 # bin size
out.append([(low_lam+(size*i)+(size*0.5)), total-other, total, 0.0, total/length])
for i in range(15,18):
print "# - Southern wedge {0}:".format(south[i-15])
data = fi.read_data("./sim_streams/sim_stripe_"+str(south[i-15])+".txt")
total = len(data[:,0])
other = sc.sum(data[:,3])
length = SGA[i,3]
mu, r, theta, phi, wedge = eval(params[i][4]), eval(params[i][5]), \
eval(params[i][6]), eval(params[i][7]), stripes[i]
x,y,z = coor.stream2xyz(0.0, 0.0, 0.0, mu, r, theta, phi, wedge)
Xs, Ys, Zs, lam, beta, r = sl.law_xyz2sgr_sun(x,y,z)
out.append([lam, total-other, total, 0.0, total/length])
fi.write_data(sc.array(out), "sgrNorth_density_sim.txt", ",", " lam(mid), count, SDSS Deff, SDSS/FTO Deff, per degree")
def computeOpenMaxProbability(openmax_fc8, openmax_score_u):
""" Convert the scores in probability value using openmax
Input:
---------------
openmax_fc8 : modified FC8 layer from Weibull based computation
openmax_score_u : degree
Output:
---------------
modified_scores : probability values modified using OpenMax framework,
by incorporating degree of uncertainity/openness for a given class
"""
prob_scores, prob_unknowns = [], []
for channel in range(NCHANNELS):
channel_scores, channel_unknowns = [], []
for category in range(NCLASSES):
channel_scores += [sp.exp(openmax_fc8[channel, category])]
total_denominator = sp.sum(sp.exp(openmax_fc8[channel, :])) + sp.exp(sp.sum(openmax_score_u[channel, :]))
prob_scores += [channel_scores/total_denominator ]
prob_unknowns += [sp.exp(sp.sum(openmax_score_u[channel, :]))/total_denominator]
prob_scores = sp.asarray(prob_scores)
prob_unknowns = sp.asarray(prob_unknowns)
scores = sp.mean(prob_scores, axis = 0)
unknowns = sp.mean(prob_unknowns, axis=0)
modified_scores = scores.tolist() + [unknowns]
assert len(modified_scores) == 1001
return modified_scores
def _maximum_likelihood(self, X):
n_samples, n_features = X.shape if X.ndim > 1 else (1, X.shape[0])
n_components = self.n_components
# Predict mean
mu = X.mean(axis=0)
# Predict covariance
cov = sp.cov(X, rowvar=0)
eigvals, eigvecs = self._eig_decomposition(cov)
sigma2 = ((sp.sum(cov.diagonal()) - sp.sum(eigvals.sum())) /
(n_features - n_components)) # FIXME: M < D?
weight = sp.dot(eigvecs, sp.diag(sp.sqrt(eigvals - sigma2)))
M = sp.dot(weight.T, weight) + sigma2 * sp.eye(n_components)
inv_M = spla.inv(M)
self.eigvals = eigvals
self.eigvecs = eigvecs
self.predict_mean = mu
self.predict_cov = sp.dot(weight, weight.T) + sigma2 * sp.eye(n_features)
self.latent_mean = sp.transpose(sp.dot(inv_M, sp.dot(weight.T, X.T - mu[:, sp.newaxis])))
self.latent_cov = sigma2 * inv_M
self.sigma2 = sigma2 # FIXME!
self.weight = weight
self.inv_M = inv_M
return self.latent_mean
def tfidf(termFrequency): """ The student must code this. """ gf = sp.sum(termFrequency,axis=1).astype(float) p = (termFrequency.T/gf).T g = sp.sum(p*sp.log(p+1)/sp.log(len(p[0,:])),axis=1) + 1 a = (sp.log(termFrequency + 1).T*g).T return a
def __init__(self, grid, fArray, zDrawsSorted):
assert(len(grid) == len(fArray))
(self.grid, self.fArray) = (grid, fArray)
self.zDraws = zDraws
self.slopes = scipy.zeros(len(grid) - 1)
self.dx = grid[1] - grid[0]
for i in range(len(grid) - 1):
self.slopes[i] = (fArray[i+1] - fArray[i]) / self.dx
# set up sums
self.cellSums = scipy.zeros(len(grid) + 1)
self.boundaryIndices = [len(zDraws)] * len(grid)
for (i, x) in enumerate(grid):
indices = scipy.nonzero(self.zDraws >= x)[0]
if (len(indices) > 0):
self.boundaryIndices[i] = indices[0]
self.cellSums[0] = scipy.sum(self.zDraws[0:self.boundaryIndices[0]])
for i in range(1, len(self.cellSums)-1):
self.cellSums[i] = scipy.sum(self.zDraws[self.boundaryIndices[i-1] : self.boundaryIndices[i]])
self.cellSums[-1] = scipy.sum(self.zDraws[self.boundaryIndices[-1] : ])
diff = scipy.sum(self.zDraws) - scipy.sum(self.cellSums)
print("diff: %f" % diff)
for i in range(len(grid)):
if (self.boundaryIndices[i] < len(self.zDraws)):
print("grid point %f, boundary %f" % (self.grid[i], self.zDraws[self.boundaryIndices[i]]))
else:
print("grid point %f, no draws to right" % self.grid[i])
def execute(self):
self.power_mat, self.thermal_expectation = self.full_calculation()
n_chan = self.power_mat.shape[1]
n_freq = self.power_mat.shape[0]
# Calculate the the mean channel correlations at low frequencies.
low_f_mat = sp.mean(self.power_mat[1:4 * n_chan + 1,:,:], 0).real
# Factorize it into preinciple components.
e, v = linalg.eigh(low_f_mat)
self.low_f_mode_values = e
# Make sure the eigenvalues are sorted.
if sp.any(sp.diff(e) < 0):
raise RuntimeError("Eigenvalues not sorted.")
self.low_f_modes = v
# Now subtract out the noisiest channel modes and see what is left.
n_modes_subtract = 10
mode_subtracted_power_mat = sp.copy(self.power_mat.real)
mode_subtracted_auto_power = sp.empty((n_modes_subtract, n_freq))
for ii in range(n_modes_subtract):
mode = v[:,-ii]
amp = sp.sum(mode[:,None] * mode_subtracted_power_mat, 1)
amp = sp.sum(amp * mode, 1)
to_subtract = amp[:,None,None] * mode[:,None] * mode
mode_subtracted_power_mat -= to_subtract
auto_power = mode_subtracted_power_mat.view()
auto_power.shape = (n_freq, n_chan**2)
auto_power = auto_power[:,::n_chan + 1]
mode_subtracted_auto_power[ii,:] = sp.mean(auto_power, -1)
self.subtracted_auto_power = mode_subtracted_auto_power
def updateB(gam,obs,M): N = gam.shape[1] s = sp.sum(gam,0) obs_gam = sp.zeros((N,M)) for i in xrange(M): obs_gam[:,i] = sp.sum(gam[obs==i,:],0)/s return(obs_gam)
def test_set_boundary_conditions_bctypes(self):
self.alg.setup(invading_phase=self.water,
defending_phase=self.air,
trapping=True)
Ps = sp.random.randint(0, self.net.Np, 10)
self.alg.set_boundary_conditions(pores=Ps, bc_type='inlets')
assert sp.sum(self.alg['pore.inlets']) == sp.size(sp.unique(Ps))
self.alg['pore.inlets'] = False
self.alg.set_boundary_conditions(pores=Ps, bc_type='outlets')
assert sp.sum(self.alg['pore.outlets']) == sp.size(sp.unique(Ps))
self.alg['pore.outlets'] = False
self.alg.set_boundary_conditions(pores=Ps, bc_type='residual')
assert sp.sum(self.alg['pore.residual']) == sp.size(sp.unique(Ps))
self.alg['pore.residual'] = False
flag = False
try:
self.alg.set_boundary_conditions(pores=Ps, bc_type='bad_type')
except:
flag = True
assert flag
flag = False
try:
self.alg.set_boundary_conditions(bc_type=None, mode='bad_type')
except:
flag = True
assert flag
def test_linear_solvers():
pn = OpenPNM.Network.Cubic([1, 40, 30], spacing=0.0001)
geom = OpenPNM.Geometry.Toray090(network=pn,
pores=pn.pores(),
throats=pn.throats())
air = OpenPNM.Phases.Air(network=pn)
phys_air = OpenPNM.Physics.Standard(network=pn,
phase=air,
pores=pn.pores(),
throats=pn.throats())
BC1_pores = pn.pores(labels=['left'])
BC2_pores = pn.pores(labels=['right'])
alg_1 = OpenPNM.Algorithms.FickianDiffusion(network=pn, phase=air)
alg_1.set_boundary_conditions(bctype='Dirichlet',
bcvalue=1,
pores=BC1_pores)
alg_1.set_boundary_conditions(bctype='Dirichlet',
bcvalue=0,
pores=BC2_pores)
alg_1.run(iterative_solver='gmres')
alg_2 = OpenPNM.Algorithms.FickianDiffusion(network=pn, phase=air)
alg_2.set_boundary_conditions(bctype='Neumann',
bcvalue=-1e-11,
pores=BC1_pores)
alg_2.set_boundary_conditions(bctype='Dirichlet',
bcvalue=0,
pores=BC2_pores)
alg_2.run(iterative_solver='cg')
alg_3 = OpenPNM.Algorithms.FickianDiffusion(network=pn, phase=air)
alg_3.set_boundary_conditions(bctype='Neumann_group',
bcvalue=-3e-10,
pores=BC1_pores)
alg_3.set_boundary_conditions(bctype='Dirichlet',
bcvalue=0,
pores=BC2_pores)
alg_3.run()
alg_4 = OpenPNM.Algorithms.FickianDiffusion(network=pn, phase=air)
alg_4.set_boundary_conditions(bctype='Neumann_group',
bcvalue=-3e-10,
pores=BC1_pores)
alg_4.set_boundary_conditions(bctype='Dirichlet',
bcvalue=0,
pores=BC2_pores)
alg_4.setup()
alg_4.solve()
assert round(sp.absolute(alg_1.rate(BC1_pores))[0], 16) == round(sp.absolute(alg_1.rate(BC2_pores))[0], 16)
assert round(sp.absolute(alg_2.rate(BC2_pores))[0], 16) == round(sp.absolute(sp.unique(alg_2['pore.'+air.name+'_bcval_Neumann']))[0]*len(BC1_pores), 16)
assert round(sp.absolute(alg_3.rate(BC2_pores))[0], 16) == round(sp.absolute(sp.unique(alg_3['pore.'+air.name+'_bcval_Neumann_group']))[0], 16)
assert round(sp.absolute(alg_4.rate(BC2_pores))[0], 16) == round(sp.absolute(sp.unique(alg_4['pore.'+air.name+'_bcval_Neumann_group']))[0], 16)
assert round(sp.absolute(sp.sum(alg_1.rate(BC1_pores,mode='single'))),16) == round(sp.absolute(alg_1.rate(BC1_pores))[0],16)
assert round(sp.absolute(sp.sum(alg_2.rate(BC2_pores,mode='single'))),16) == round(sp.absolute(alg_2.rate(BC2_pores))[0],16)
assert round(sp.absolute(sp.sum(alg_3.rate(BC2_pores,mode='single'))),16) == round(sp.absolute(alg_3.rate(BC2_pores))[0],16)
assert round(sp.absolute(sp.sum(alg_4.rate(BC2_pores,mode='single'))),16) == round(sp.absolute(alg_4.rate(BC2_pores))[0],16)
def cosine_alt(h1, h2): # 17 us @array, 42 us @list \w 100 bins
"""
Alternative implementation of the cosine distance measure.
@note under development.
"""
h1, h2 = __prepare_histogram(h1, h2)
return -1 * float(scipy.sum(h1 * h2)) / (scipy.sum(scipy.power(h1, 2)) * scipy.sum(scipy.power(h2, 2)))
def quality(func, mesh, interpolator='nn', n=33):
"""Compute a quality factor (the quantity r**2 from TOMS792).
interpolator must be in ('linear', 'nn').
"""
fz = func(mesh.x, mesh.y)
tri = Triangulation(mesh.x, mesh.y)
intp = getattr(tri, interpolator+'_extrapolator')(fz, bbox=(0.,1.,0.,1.))
Y, X = sp.mgrid[0:1:complex(0,n),0:1:complex(0,n)]
Z = func(X, Y)
iz = intp[0:1:complex(0,n),0:1:complex(0,n)]
#nans = sp.isnan(iz)
#numgood = n*n - sp.sum(sp.array(nans.flat, sp.int32))
numgood = n*n
SE = (Z - iz)**2
SSE = sp.sum(SE.flat)
meanZ = sp.sum(Z.flat) / numgood
SM = (Z - meanZ)**2
SSM = sp.sum(SM.flat)
r2 = 1.0 - SSE/SSM
print func.func_name, r2, SSE, SSM, numgood
return r2
def _do_outer_iteration_stage(self):
#Generate curve from points
for inv_val in self._inv_points:
#Apply one applied pressure and determine invaded pores
logger.info('Applying capillary pressure: '+str(inv_val))
self._do_one_inner_iteration(inv_val)
#Store results using networks' get/set method
self['pore.inv_Pc'] = self._p_inv
self['throat.inv_Pc'] = self._t_inv
#Find invasion sequence values (to correspond with IP algorithm)
self._p_seq = sp.searchsorted(sp.unique(self._p_inv),self._p_inv)
self._t_seq = sp.searchsorted(sp.unique(self._t_inv),self._t_inv)
self['pore.inv_seq'] = self._p_seq
self['throat.inv_seq'] = self._t_seq
#Calculate Saturations
v_total = sp.sum(self._net['pore.volume'])+sp.sum(self._net['throat.volume'])
sat = 0.
self['pore.inv_sat'] = 1.
self['throat.inv_sat'] = 1.
for i in range(self._npts):
inv_pores = sp.where(self._p_seq==i)[0]
inv_throats = sp.where(self._t_seq==i)[0]
new_sat = (sum(self._net['pore.volume'][inv_pores])+sum(self._net['throat.volume'][inv_throats]))/v_total
sat += new_sat
self['pore.inv_sat'][inv_pores] = sat
self['throat.inv_sat'][inv_throats] = sat
def get_fluid_image(self, size=None, saturation=None):
r"""
Returns a binary image of the invading fluid configuration
Parameters
----------
size : scalar
The size of invaded pores, so these and all larger pores will be
filled, if they are accessible.
saturation : scalar
The fractional filling of the pore space to return. The size of
the invaded pores are adjusted by trial and error until this
value is reached. If size is sent then saturation is ignored.
"""
if size is not None:
im = self._iminv >= size
else:
Vp = sp.sum(self.image)
for r in sp.unique(self._iminv):
im = self._iminv >= r
if sp.sum(im)/Vp >= saturation:
break
return im
def decode(file_name):
border.rotate(file_name)
image = Image.open("temp.png")
q = border.find("temp.png")
ind = sp.argmin(sp.sum(q, 1), 0)
up_left = q[ind, 0] + 2
up_top = q[ind, 1] + 2
d_right = q[ind+1, 0] - 3
d_bottom = q[ind-1, 1] - 3
box = (up_left, up_top, d_right, d_bottom)
region = image.crop(box)
h_sum = sp.sum(region, 0)
m = argrelmax(sp.correlate(h_sum, h_sum, 'same'))
s = sp.average(sp.diff(m))
m = int(round(d_right - up_left)/s)
if m % 3 != 0:
m += 3 - m % 3
n = int(round(d_bottom - up_top)/s)
if n % 4 != 0:
n += 4 - n % 4
s = int(round(s))+1
region = region.resize((s*m, s*n), PIL.Image.ANTIALIAS)
region.save("0.png")
pix = region.load()
matrix = mix.off(rec.matrix(pix, s, m, n))
str2 = hamming.decode(array_to_str(matrix))
return hamming.bin_to_str(str2)
def _msge_with_gradient_underdetermined(self, data, delta, xvschema, skipstep):
""" Calculate the mean squared generalization error and it's gradient for underdetermined equation system.
"""
(l, m, t) = data.shape
d = None
j, k = 0, 0
nt = sp.ceil(t / skipstep)
for s in range(0, t, skipstep):
trainset, testset = xvschema(s, t)
(a, b) = self._construct_eqns(sp.atleast_3d(data[:, :, trainset]))
(c, d) = self._construct_eqns(sp.atleast_3d(data[:, :, testset]))
e = sp.linalg.inv(sp.eye(a.shape[0]) * delta ** 2 + a.dot(a.transpose()))
cc = c.transpose().dot(c)
be = b.transpose().dot(e)
bee = be.dot(e)
bea = be.dot(a)
beea = bee.dot(a)
beacc = bea.dot(cc)
dc = d.transpose().dot(c)
j += sp.sum(beacc * bea - 2 * bea * dc) + sp.sum(d ** 2)
k += sp.sum(beea * dc - beacc * beea) * 4 * delta
return j / (nt * d.size), k / (nt * d.size)
def _msge_with_gradient_overdetermined(self, data, delta, xvschema, skipstep):
""" Calculate the mean squared generalization error and it's gradient for overdetermined equation system.
"""
(l, m, t) = data.shape
d = None
l, k = 0, 0
nt = sp.ceil(t / skipstep)
for s in range(0, t, skipstep):
#print(s,drange)
trainset, testset = xvschema(s, t)
(a, b) = self._construct_eqns(sp.atleast_3d(data[:, :, trainset]))
(c, d) = self._construct_eqns(sp.atleast_3d(data[:, :, testset]))
#e = sp.linalg.inv(np.eye(a.shape[1])*delta**2 + a.transpose().dot(a), overwrite_a=True, check_finite=False)
e = sp.linalg.inv(sp.eye(a.shape[1]) * delta ** 2 + a.transpose().dot(a))
ba = b.transpose().dot(a)
dc = d.transpose().dot(c)
bae = ba.dot(e)
baee = bae.dot(e)
baecc = bae.dot(c.transpose().dot(c))
l += sp.sum(baecc * bae - 2 * bae * dc) + sp.sum(d ** 2)
k += sp.sum(baee * dc - baecc * baee) * 4 * delta
return l / (nt * d.size), k / (nt * d.size)
def _update_network(network, net):
# Infer Np and Nt from length of given prop arrays in file
for element in ['pore', 'throat']:
N = [_sp.shape(net[i])[0] for i in net.keys() if i.startswith(element)]
if N:
N = _sp.array(N)
if _sp.all(N == N[0]):
if (network._count(element) == N[0]) \
or (network._count(element) == 0):
network.update({element+'.all': _sp.ones((N[0],),
dtype=bool)})
net.pop(element+'.all', None)
else:
raise Exception('Length of '+element+' data in file ' +
'does not match network')
else:
raise Exception(element+' data in file have inconsistent ' +
'lengths')
# Add data on dummy net to actual network
for item in net.keys():
# Try to infer array types and change if necessary
# Chcek for booleans disguised and 1's and 0's
num0s = _sp.sum(net[item] == 0)
num1s = _sp.sum(net[item] == 1)
if (num1s + num0s) == _sp.shape(net[item])[0]:
net[item] = net[item].astype(bool)
# Write data to network object
if item not in network:
network.update({item: net[item]})
else:
logger.warning('\''+item+'\' already present')
return network
def test_Event_Activity_ground_motion_model_logic_split(self):
num_events = 6
max_weights = 5
ea = Event_Activity(num_events)
indexes = arange(6)
activity = array((indexes*10, indexes*20))
ea.set_event_activity(activity, indexes)
atten_model_weights = [array([.4, .6]),array([.1, .4, .5])]
a = DummyEventSet()
b = DummyEventSet()
source_model = [a, b]
#event_set_indexes = [array([0,1,3]), array([2,4])]
event_set_indexes = [[0,1,3], [2,4]]
for sp, esi, amw in map(None, source_model, event_set_indexes,
atten_model_weights):
sp.atten_model_weights = amw
sp.event_set_indexes = esi
source_model = Source_Model(source_model)
ea.ground_motion_model_logic_split(source_model, apply_weights=True)
self.assert_(allclose(sum(ea.event_activity), sum(activity)))
self.assert_(allclose(ea.event_activity[0, 0, 0, 3], 12.))
self.assert_(allclose(ea.event_activity[0, 0, 0, 4], 4.))
self.assert_(allclose(ea.event_activity[0, 0, 1, 3], 24.))
self.assert_(allclose(ea.event_activity[0, 0, 1, 4], 8.))
def sum2(input, dtype=None):
"""
Returns sum of all non-masked :obj:`Dds` elements-squared.
:type input: :obj:`Dds`
:param input: Input elements for which sum of squared-elements is calculated.
:type dtype: :obj:`numpy.dtype` or dtype :obj:`str`
:param dtype: Type used for summation of elements.
:rtype: scalar
:return: Sum of the squared-elements (i.e. :samp:`scipy.sum((input.asarray())**2, dtype)`).
"""
mskVal = None
if (hasattr(input, "mtype") and (input.mtype != None)):
mskVal = input.mtype.maskValue()
mpiComm = None
if (hasattr(input, "mpi") and hasattr(input.mpi, "comm") and (input.mpi.comm != None)):
mpiComm = input.mpi.comm
inArr = input.subd.asarray()
if (mskVal != None):
s = sp.sum(sp.where(inArr != mskVal, inArr**2, 0), dtype=dtype)
else:
s = sp.sum(inArr**2, dtype=dtype)
if (mpiComm != None):
s = mpiComm.allreduce(s, mango.mpi.SUM)
return s
def score_samples(self, X, y=None):
"""Compute the negative weighted log probabilities for each sample.
Parameters
----------
X : array-like, shape (n_samples, n_features)
List of n_features-dimensional data points. Each row
corresponds to a single data point.
Returns
-------
log_prob : array, shape (n_samples, n_clusters)
Log probabilities of each data point in X.
"""
X = check_array(X, copy=False, order='C', dtype=sp.float64)
nt, d = X.shape
K = sp.empty((nt, self.C))
# Start the prediction for each class
for c in xrange(self.C):
# Compute the constant term
K[:, c] = self.logdet[c] - 2*sp.log(self.prop[c]) + self.cst
# Remove the mean
Xc = X - self.mean[c]
# Do the projection
Px = sp.dot(Xc,
sp.dot(self.Q[c], self.Q[c].T))
temp = sp.dot(Px, self.Q[c]/sp.sqrt(self.a[c]))
K[:, c] += sp.sum(temp**2, axis=1)
K[:, c] += sp.sum((Xc - Px)**2, axis=1)/self.b[c]
return -K
def from_gene(self, gene):
sg = gene.splicegraph.vertices
breakpoints = sp.unique(sg.ravel())
self.segments = sp.zeros((2, 0), dtype='int')
for j in range(1, breakpoints.shape[0]):
s = sp.sum(sg[0, :] < breakpoints[j])
e = sp.sum(sg[1, :] < breakpoints[j])
if s > e:
self.segments = sp.c_[self.segments, [breakpoints[j-1], breakpoints[j]]]
### match nodes to segments
self.seg_match = sp.zeros((0, sg.shape[1]), dtype='bool')
for j in range(sg.shape[1]):
tmp = ((sg[0, j] <= self.segments[0, :]) & (sg[1, j] >= self.segments[1, :]))
if self.seg_match.shape[0] == 0:
self.seg_match = tmp.copy().reshape((1, tmp.shape[0]))
else:
self.seg_match = sp.r_[self.seg_match, tmp.reshape((1, tmp.shape[0]))]
### create edge graph between segments
self.seg_edges = sp.zeros((self.segments.shape[1], self.segments.shape[1]), dtype='bool')
k, l = sp.where(sp.triu(gene.splicegraph.edges))
for m in range(k.shape[0]):
### donor segment
d = sp.where(self.seg_match[k[m], :])[0][-1]
### acceptor segment
a = sp.where(self.seg_match[l[m], :])[0][0]
self.seg_edges[d, a] = True
def collapse_correlation_1d(corr, f_lags, a_lags, weights=None):
r"""Takes a 2D correlation function and collapses to a 1D correlation
function.
Parameters
----------
corr: 2D array
Covariance matrix in terms of frequency lag and angular lag.
The first output from `rebin_corr_freq_lag` right now.
f_lags: 1D array
The frequency lags in terms of Hz.
The third output from `rebin_corr_freq_lag` right now.
a_lags: 1D array
The angular lags in terms of degrees.
weights: 2D array
The weights of `corr`.
The second output from `rebin_corr_freq_lag` right now.
Returns
-------
out_corr: 1D array
The 1D autocorrelation.
out_weights:
The weights for `out_corr`.
x_axis: tuple of 3 1D arrays
`x_axis[1]` is the x - values that correspond to `out_corr`.
`x_axis[0]` and `x_axis[2]` are the left and rightmost points
covered by each lag bin.
Notes
-----
`a_lags` are not the same as the lags from the .ini file.
The lags from the .ini file are the right side of each lag bin,
but you want the centre of the bin when you plot.
To get the right values, you must do: (ask Eric or Liviu)
lags = sp.array(F.params['lags'])
a_lags = copy.deepcopy(lags)
a_lags[0] = 0
a_lags[1:] -= sp.diff(lags)/2.0
"""
if corr.ndim != 2:
msg = "Must start with a 2D correlation function."
raise ValueError(msg)
if len(f_lags) != corr.shape[0] or len(a_lags) != corr.shape[1]:
msg = ("corr.shape must be (len(f_lags), len(a_lags)). Passed: " +
repr(corr.shape) + " vs (" + repr(len(f_lags)) + ", " +
repr(len(a_lags)) + ").")
raise ValueError(msg)
if weights is None:
weights = sp.ones_like(corr)
corr = corr * weights
# Hard code conversion factors to MPc/h for now.
a_fact = 34.0 # Mpc/h per degree at 800MHz.
f_fact = 4.5 # Mpc/h per MHz at 800MHz.
# Hard code lags in MPc/h.
#nbins = 10
nbins = 15
lags = sp.empty(nbins)
lags[0] = 2.0
lags[1] = 4.0
for bin_index in range(2, nbins):
lags[bin_index] = 1.5 * lags[bin_index - 1]
# Calculate the total 1D lags.
separation = a_lags
separation = (a_fact * separation[sp.newaxis, :])**2
separation = separation + (f_fact * f_lags[:, sp.newaxis] / 1.0e6)**2
separation = sp.sqrt(separation)
# Initialize memory for outputs.
out_corr = sp.zeros(nbins)
out_weights = sp.zeros(nbins)
# Rebin.
for lag_index in range(separation.shape[0]):
bin_inds = sp.digitize(separation[lag_index, :], lags)
for bin_index in range(nbins):
out_corr[bin_index] += sp.sum(corr[lag_index,
bin_inds == bin_index])
out_weights[bin_index] += sp.sum(weights[lag_index,
bin_inds == bin_index])
# Normalize.
bad_inds = out_weights < 1.0e-20
out_weights[bad_inds] = 1.0
out_corr /= out_weights
out_weights[bad_inds] = 0.0
# Make real lags to be returned.
x_left = sp.empty(nbins)
x_left[0] = 0
x_left[1:] = lags[:-1]
x_right = lags
x_centre = (x_right + x_left) / 2.0
return out_corr, out_weights, (x_left, x_centre, x_right)
def bovy_dens2d(X, **kwargs):
"""
NAME:
bovy_dens2d
PURPOSE:
plot a 2d density with optional contours
INPUT:
first argument is the density
matplotlib.pyplot.imshow keywords (see http://matplotlib.sourceforge.net/api/axes_api.html#matplotlib.axes.Axes.imshow)
xlabel - (raw string!) x-axis label, LaTeX math mode, no $s needed
ylabel - (raw string!) y-axis label, LaTeX math mode, no $s needed
xrange
yrange
noaxes - don't plot any axes
overplot - if True, overplot
colorbar - if True, add colorbar
shrink= colorbar argument: shrink the colorbar by the factor (optional)
conditional - normalize each column separately (for probability densities, i.e., cntrmass=True)
gcf=True does not start a new figure (does change the ranges and labels)
Contours:
justcontours - if True, only draw contours
contours - if True, draw contours (10 by default)
levels - contour-levels
cntrmass - if True, the density is a probability and the levels are probability masses contained within the contour
cntrcolors - colors for contours (single color or array)
cntrlabel - label the contours
cntrlw, cntrls - linewidths and linestyles for contour
cntrlabelsize, cntrlabelcolors,cntrinline - contour arguments
cntrSmooth - use ndimage.gaussian_filter to smooth before contouring
onedhists - if True, make one-d histograms on the sides
onedhistcolor - histogram color
retAxes= return all Axes instances
retCont= return the contour instance
OUTPUT:
plot to output device, Axes instances depending on input
HISTORY:
2010-03-09 - Written - Bovy (NYU)
"""
overplot = kwargs.pop('overplot', False)
gcf = kwargs.pop('gcf', False)
if not overplot and not gcf:
pyplot.figure()
xlabel = kwargs.pop('xlabel', None)
ylabel = kwargs.pop('ylabel', None)
zlabel = kwargs.pop('zlabel', None)
if 'extent' in kwargs:
extent = kwargs.pop('extent')
else:
xlimits = kwargs.pop('xrange', [0, X.shape[1]])
ylimits = kwargs.pop('yrange', [0, X.shape[0]])
extent = xlimits + ylimits
if not 'aspect' in kwargs:
kwargs['aspect'] = (xlimits[1] - xlimits[0]) / float(ylimits[1] -
ylimits[0])
noaxes = kwargs.pop('noaxes', False)
justcontours = kwargs.pop('justcontours', False)
if ('contours' in kwargs and kwargs['contours']) or \
'levels' in kwargs or justcontours or \
('cntrmass' in kwargs and kwargs['cntrmass']):
contours = True
else:
contours = False
kwargs.pop('contours', None)
if 'levels' in kwargs:
levels = kwargs['levels']
kwargs.pop('levels')
elif contours:
if 'cntrmass' in kwargs and kwargs['cntrmass']:
levels = sc.linspace(0., 1., _DEFAULTNCNTR)
elif True in sc.isnan(sc.array(X)):
levels = sc.linspace(sc.nanmin(X), sc.nanmax(X), _DEFAULTNCNTR)
else:
levels = sc.linspace(sc.amin(X), sc.amax(X), _DEFAULTNCNTR)
cntrmass = kwargs.pop('cntrmass', False)
conditional = kwargs.pop('conditional', False)
cntrcolors = kwargs.pop('cntrcolors', 'k')
cntrlabel = kwargs.pop('cntrlabel', False)
cntrlw = kwargs.pop('cntrlw', None)
cntrls = kwargs.pop('cntrls', None)
cntrSmooth = kwargs.pop('cntrSmooth', None)
cntrlabelsize = kwargs.pop('cntrlabelsize', None)
cntrlabelcolors = kwargs.pop('cntrlabelcolors', None)
cntrinline = kwargs.pop('cntrinline', None)
retCumImage = kwargs.pop('retCumImage', False)
cb = kwargs.pop('colorbar', False)
shrink = kwargs.pop('shrink', None)
onedhists = kwargs.pop('onedhists', False)
onedhistcolor = kwargs.pop('onedhistcolor', 'k')
retAxes = kwargs.pop('retAxes', False)
retCont = kwargs.pop('retCont', False)
if onedhists:
if overplot or gcf: fig = pyplot.gcf()
else: fig = pyplot.figure()
nullfmt = NullFormatter() # no labels
# definitions for the axes
left, width = 0.1, 0.65
bottom, height = 0.1, 0.65
bottom_h = left_h = left + width
rect_scatter = [left, bottom, width, height]
rect_histx = [left, bottom_h, width, 0.2]
rect_histy = [left_h, bottom, 0.2, height]
axScatter = pyplot.axes(rect_scatter)
axHistx = pyplot.axes(rect_histx)
axHisty = pyplot.axes(rect_histy)
# no labels
axHistx.xaxis.set_major_formatter(nullfmt)
axHistx.yaxis.set_major_formatter(nullfmt)
axHisty.xaxis.set_major_formatter(nullfmt)
axHisty.yaxis.set_major_formatter(nullfmt)
fig.sca(axScatter)
ax = pyplot.gca()
ax.set_autoscale_on(False)
if conditional:
plotthis = X / sc.tile(sc.sum(X, axis=0), (X.shape[1], 1))
else:
plotthis = X
if not justcontours:
out = pyplot.imshow(plotthis, extent=extent, **kwargs)
if not overplot:
pyplot.axis(extent)
_add_axislabels(xlabel, ylabel)
_add_ticks()
#Add colorbar
if cb and not justcontours:
if shrink is None:
shrink = sc.amin([float(kwargs.pop('aspect', 1.)) * 0.87, 1.])
CB1 = pyplot.colorbar(out, shrink=shrink)
if not zlabel is None:
if zlabel[0] != '$':
thiszlabel = r'$' + zlabel + '$'
else:
thiszlabel = zlabel
CB1.set_label(thiszlabel)
if contours or retCumImage:
aspect = kwargs.get('aspect', None)
origin = kwargs.get('origin', None)
if cntrmass:
#Sum from the top down!
plotthis[sc.isnan(plotthis)] = 0.
sortindx = sc.argsort(plotthis.flatten())[::-1]
cumul = sc.cumsum(sc.sort(plotthis.flatten())[::-1]) / sc.sum(
plotthis.flatten())
cntrThis = sc.zeros(sc.prod(plotthis.shape))
cntrThis[sortindx] = cumul
cntrThis = sc.reshape(cntrThis, plotthis.shape)
else:
cntrThis = plotthis
if contours:
if not cntrSmooth is None:
cntrThis = ndimage.gaussian_filter(cntrThis,
cntrSmooth,
mode='nearest')
cont = pyplot.contour(cntrThis,
levels,
colors=cntrcolors,
linewidths=cntrlw,
extent=extent,
aspect=aspect,
linestyles=cntrls,
origin=origin)
if cntrlabel:
pyplot.clabel(cont,
fontsize=cntrlabelsize,
colors=cntrlabelcolors,
inline=cntrinline)
if noaxes:
ax.set_axis_off()
#Add onedhists
if not onedhists:
if retCumImage:
return cntrThis
elif retAxes:
return pyplot.gca()
elif retCont:
return cont
elif justcontours:
return cntrThis
else:
return out
histx = sc.nansum(X.T, axis=1) * m.fabs(ylimits[1] - ylimits[0]) / X.shape[
1] #nansum bc nan is *no dens value*
histy = sc.nansum(X.T,
axis=0) * m.fabs(xlimits[1] - xlimits[0]) / X.shape[0]
histx[sc.isnan(histx)] = 0.
histy[sc.isnan(histy)] = 0.
dx = (extent[1] - extent[0]) / float(len(histx))
axHistx.plot(sc.linspace(extent[0] + dx, extent[1] - dx, len(histx)),
histx,
drawstyle='steps-mid',
color=onedhistcolor)
dy = (extent[3] - extent[2]) / float(len(histy))
axHisty.plot(histy,
sc.linspace(extent[2] + dy, extent[3] - dy, len(histy)),
drawstyle='steps-mid',
color=onedhistcolor)
axHistx.set_xlim(axScatter.get_xlim())
axHisty.set_ylim(axScatter.get_ylim())
axHistx.set_ylim(0, 1.2 * sc.amax(histx))
axHisty.set_xlim(0, 1.2 * sc.amax(histy))
if retCumImage:
return cntrThis
elif retAxes:
return (axScatter, axHistx, axHisty)
elif justcontours:
return cntrThis
else:
return out
def __init__(self, CFIE_coeff, TENETHNH, list_of_test_edges_numbers,
list_of_src_edges_numbers, target_mesh, w, eps_r_out,
mu_r_out, eps_r_in, mu_r_in, MOM_FULL_PRECISION, FORMULATION):
self.numberOfMedia = sum(target_mesh.IS_CLOSED_SURFACE) + 1
print "MOM.py: number of possible media =", self.numberOfMedia
print "Target_MoM instanciation..."
t0 = time.clock()
TDS_APPROX, Z_s = 0, 0.0 + 0.0j
N_J, N_M = len(list_of_test_edges_numbers), len(
list_of_test_edges_numbers)
self.Z = zeros((N_J + N_M, N_J + N_M), 'D')
if FORMULATION == "CFIE": # we have CFIE
print "OK, using CFIE formulation"
print "dielectricTarget_MoM: computing the outside interactions..."
coeff = CFIE_coeff
TE, NE, TH, NH = TENETHNH[0], TENETHNH[1], TENETHNH[2], TENETHNH[3]
CFIE = array([
TE * coeff, NE * coeff, -TH *
(1.0 - coeff) * sqrt(mu_0 / (eps_0 * eps_r_out)), -NH *
(1.0 - coeff) * sqrt(mu_0 / (eps_0 * eps_r_out))
], 'D')
signSurfObs, signSurfSrc = 1.0, 1.0
self.Z[:N_J, :N_J], self.Z[:N_J, N_J:N_J + N_M] = Z_CFIE_MoM(
CFIE, list_of_test_edges_numbers, list_of_src_edges_numbers,
target_mesh.RWGNumber_CFIE_OK,
target_mesh.RWGNumber_M_CURRENT_OK,
target_mesh.RWGNumber_signedTriangles,
target_mesh.RWGNumber_edgeVertexes,
target_mesh.RWGNumber_oppVertexes, target_mesh.vertexes_coord,
w, eps_r_out, mu_r_out, signSurfObs, signSurfSrc, TDS_APPROX,
Z_s, MOM_FULL_PRECISION)
print "dielectricTarget_MoM: computing the inside interactions..."
CFIE = array([
TE * coeff, NE * coeff, -TH *
(1.0 - coeff) * sqrt(mu_0 / (eps_0 * eps_r_in)), -NH *
(1.0 - coeff) * sqrt(mu_0 / (eps_0 * eps_r_in))
], 'D')
signSurfObs, signSurfSrc = -1.0, -1.0
self.Z[N_J:N_J + N_M, :N_J], self.Z[
N_J:N_J + N_M, N_J:N_J + N_M] = Z_CFIE_MoM(
CFIE, list_of_test_edges_numbers,
list_of_src_edges_numbers, target_mesh.RWGNumber_CFIE_OK,
target_mesh.RWGNumber_M_CURRENT_OK,
target_mesh.RWGNumber_signedTriangles,
target_mesh.RWGNumber_edgeVertexes,
target_mesh.RWGNumber_oppVertexes,
target_mesh.vertexes_coord, w, eps_r_in, mu_r_in,
signSurfObs, signSurfSrc, TDS_APPROX, Z_s,
MOM_FULL_PRECISION)
elif FORMULATION == "PMCHWT": # we have PMCHWT
print "OK, using PMCHWT formulation"
print "dielectricTarget_MoM: computing the outside interactions..."
signSurfObs, signSurfSrc = 1.0, 1.0
CFIE_for_PMCHWT = array([1.0, 0., 0., 0.], 'D')
Z_EJ, Z_EM = Z_CFIE_MoM(
CFIE_for_PMCHWT, list_of_test_edges_numbers,
list_of_src_edges_numbers, target_mesh.RWGNumber_CFIE_OK,
target_mesh.RWGNumber_M_CURRENT_OK,
target_mesh.RWGNumber_signedTriangles,
target_mesh.RWGNumber_edgeVertexes,
target_mesh.RWGNumber_oppVertexes, target_mesh.vertexes_coord,
w, eps_r_out, mu_r_out, signSurfObs, signSurfSrc, TDS_APPROX,
Z_s, MOM_FULL_PRECISION)
#TENETHNH = array([1.0, 0.0, 1.0, 0.0])
#Z_EJ, Z_nE_J, Z_HJ, Z_nH_J = Z_EH_J_MoM(TENETHNH, list_of_test_edges_numbers, list_of_src_edges_numbers, target_mesh.RWGNumber_CFIE_OK, target_mesh.RWGNumber_signedTriangles, target_mesh.RWGNumber_edgeVertexes, target_mesh.RWGNumber_oppVertexes, target_mesh.vertexes_coord, w, eps_r_out, mu_r_out, signSurfObs, signSurfSrc, TDS_APPROX, Z_s, MOM_FULL_PRECISION)
#Z_EM = -Z_HJ
eta_0 = sqrt(mu_0 / (eps_0 * eps_r_out))
self.Z[:N_J, :
N_J], self.Z[:N_J, N_J:N_J +
N_M] = 1.0 / eta_0 * Z_EJ, 1.0 / eta_0 * Z_EM
self.Z[N_J:N_J + N_M, :N_J] = -eta_0 * Z_EM # Z_HJ = -Z_EM
self.Z[N_J:N_J + N_M, N_J:N_J + N_M] = eta_0 * (
eps_0 * eps_r_out /
(mu_0 * mu_r_out)) * Z_EJ # Z_HM = eps/mu * Z_EJ
print "dielectricTarget_MoM: computing the inside interactions..."
signSurfObs, signSurfSrc = -1.0, -1.0
Z_EJ, Z_EM = Z_CFIE_MoM(
CFIE_for_PMCHWT, list_of_test_edges_numbers,
list_of_src_edges_numbers, target_mesh.RWGNumber_CFIE_OK,
target_mesh.RWGNumber_M_CURRENT_OK,
target_mesh.RWGNumber_signedTriangles,
target_mesh.RWGNumber_edgeVertexes,
target_mesh.RWGNumber_oppVertexes, target_mesh.vertexes_coord,
w, eps_r_in, mu_r_in, signSurfObs, signSurfSrc, TDS_APPROX,
Z_s, MOM_FULL_PRECISION)
#Z_EJ, Z_nE_J, Z_HJ, Z_nH_J = Z_EH_J_MoM(TENETHNH, list_of_test_edges_numbers, list_of_src_edges_numbers, target_mesh.RWGNumber_CFIE_OK, target_mesh.RWGNumber_signedTriangles, target_mesh.RWGNumber_edgeVertexes, target_mesh.RWGNumber_oppVertexes, target_mesh.vertexes_coord, w, eps_r_in, mu_r_in, signSurfObs, signSurfSrc, TDS_APPROX, Z_s, MOM_FULL_PRECISION)
#Z_EM = -Z_HJ
eta_1 = sqrt(mu_0 / (eps_0 * eps_r_in))
self.Z[:N_J, :N_J] += 1.0 / eta_1 * Z_EJ
self.Z[:N_J, N_J:N_J + N_M] += 1.0 / eta_1 * Z_EM
self.Z[N_J:N_J + N_M, :N_J] += -eta_1 * Z_EM # Z_HJ = -Z_EM
self.Z[N_J:N_J + N_M, N_J:N_J + N_M] += eta_1 * (
eps_0 * eps_r_in /
(mu_0 * mu_r_in)) * Z_EJ # Z_HM = eps/mu * Z_EJ
elif FORMULATION == "JMCFIE":
print "OK, using JMCFIE formulation"
print "dielectricTarget_MoM: computing the outside interactions..."
signSurfObs, signSurfSrc = 1.0, 1.0
coeff = CFIE_coeff
TENETHNH = array([1.0, 1.0, 1.0, 1.0])
Z_tE_J, Z_nE_J, Z_tH_J, Z_nH_J = Z_EH_J_MoM(
TENETHNH, list_of_test_edges_numbers,
list_of_src_edges_numbers, target_mesh.RWGNumber_CFIE_OK,
target_mesh.RWGNumber_signedTriangles,
target_mesh.RWGNumber_edgeVertexes,
target_mesh.RWGNumber_oppVertexes, target_mesh.vertexes_coord,
w, eps_r_out, mu_r_out, signSurfObs, signSurfSrc, TDS_APPROX,
Z_s, MOM_FULL_PRECISION)
self.Z[:N_J, :N_J] = coeff * Z_tE_J + (1.0 - coeff) * sqrt(
mu_0 / (eps_0 * eps_r_out)) * Z_nH_J
# Z_EM = -Z_HJ, Z_HM = eps/mu * Z_EJ
#self.Z[:N_J,N_J:N_J + N_M] = coeff * Z_tE_M + (1.0-coeff) * sqrt(mu_0/(eps_0*eps_r_out)) * Z_nH_M
self.Z[:N_J,
N_J:N_J + N_M] = -coeff * Z_tH_J + (1.0 - coeff) * sqrt(
mu_0 /
(eps_0 * eps_r_out)) * (eps_0 * eps_r_out /
(mu_0 * mu_r_out)) * Z_nE_J
self.Z[N_J:N_J + N_M, :N_J] = coeff * sqrt(
mu_0 / (eps_0 * eps_r_out)) * Z_tH_J - (1.0 - coeff) * Z_nE_J
self.Z[N_J:N_J + N_M, N_J:N_J +
N_M] = coeff * sqrt(mu_0 / (eps_0 * eps_r_out)) * (
eps_0 * eps_r_out /
(mu_0 * mu_r_out)) * Z_tE_J + (1.0 - coeff) * Z_nH_J
print "dielectricTarget_MoM: computing the inside interactions..."
signSurfObs, signSurfSrc = -1.0, -1.0
Z_tE_J, Z_nE_J, Z_tH_J, Z_nH_J = Z_EH_J_MoM(
TENETHNH, list_of_test_edges_numbers,
list_of_src_edges_numbers, target_mesh.RWGNumber_CFIE_OK,
target_mesh.RWGNumber_signedTriangles,
target_mesh.RWGNumber_edgeVertexes,
target_mesh.RWGNumber_oppVertexes, target_mesh.vertexes_coord,
w, eps_r_in, mu_r_in, signSurfObs, signSurfSrc, TDS_APPROX,
Z_s, MOM_FULL_PRECISION)
self.Z[:N_J, :N_J] -= coeff * Z_tE_J + (1.0) * (
1.0 - coeff) * sqrt(mu_0 / (eps_0 * eps_r_in)) * Z_nH_J
# Z_EM = -Z_HJ, Z_HM = eps/mu * Z_EJ
#self.Z[:N_J,N_J:N_J + N_M] = coeff * Z_tE_M + (1.0-coeff) * sqrt(mu_0/(eps_0*eps_r_out)) * Z_nH_M
self.Z[:N_J, N_J:N_J +
N_M] -= -coeff * Z_tH_J + (1.0) * (1.0 - coeff) * sqrt(
mu_0 / (eps_0 * eps_r_in)) * (eps_0 * eps_r_in /
(mu_0 * mu_r_in)) * Z_nE_J
self.Z[N_J:N_J + N_M, :N_J] -= coeff * sqrt(
mu_0 /
(eps_0 * eps_r_in)) * Z_tH_J - (1.0) * (1.0 - coeff) * Z_nE_J
self.Z[
N_J:N_J + N_M,
N_J:N_J + N_M] -= coeff * sqrt(mu_0 / (eps_0 * eps_r_in)) * (
eps_0 * eps_r_in /
(mu_0 * mu_r_in)) * Z_tE_J + (1.0) * (1.0 - coeff) * Z_nH_J
else:
print "use another formulation please. Error."
sys.exit(1)
print "Done. time =", time.clock() - t0, "seconds"
self.iter_counter = 0
E_0 = array([-1.0, 0.0, 0.0], 'D')
k_hat = array([0., 0., -1.0], 'd')
r_ref = zeros(3, 'd') #sum(triangles_centroids, axis=0)/T
target_MoM.V_EH_plane_computation(CFIE, TENETHNH, target_mesh, E_0,
k_hat, r_ref, w, eps_r, mu_r,
list_of_test_edges_numbers)
#target_MoM.V_EH_computation(CFIE, target_mesh, J_dip, r_dip, w, eps_r, mu_r, list_of_test_edges_numbers, EXCITATION)
#target_MoM.solveByInversion()
target_MoM.solveByLUdecomposition()
J_dip = array([1.0, 0.0, 0.], 'D')
r_dip = array([0., 0., 2.0], 'd')
target_MoM.V_EH_computation(CFIE, target_mesh, J_dip, r_dip, w,
eps_r, mu_r,
list_of_test_edges_numbers, EXCITATION)
print "inverted MoM RCS =", -sum(
target_MoM.I_CFIE * target_MoM.V_EH[:, 0])
#computeCurrentsVisualization(w, target_mesh, target_MoM.I_CFIE)
# now we try the iterative method
#count = 0
#I_CFIE_bicgstab = bicgstab(target_MoM.Z_CFIE_J, target_MoM.V_CFIE, x0=None, tol=1.0e-05, maxiter=1000, xtype=None, callback=itercount)
#V = V_EH_computation(self, CFIE_coeff, TENETHNH, target_mesh, J_dip, r_dip, w, eps_r, mu_r, list_of_test_edges_numbers, 'dipole', FORMULATION)
#print "bicgstab MoM RCS =", -sum(I_CFIE_bicgstab[0]*V[:,0]), "# of iterations =", count
## computation of the scattered fields
E_0 = array([-1.0, 0.0, 0.0], 'D')
k_hat = array([0., 0., -1.0], 'd')
r_ref = zeros(3, 'd') #sum(triangles_centroids, axis=0)/T
J_obs = array([1.0, 0.0, 0.0], 'D')
E_x, E_inc_x, E_tot_x = [], [], []
for z in arange(-3., -2., 0.05):
r_obs = array([0.0, 0.0, z], 'd')
def gen_anc_afs_plot(
ancestry='AFR',
plot_prefix='/Users/bjv/Dropbox/Cloud_folder/tmp/1kg_AFS_all',
outfile_prefix='/Users/bjv/Dropbox/Cloud_folder/tmp/1kg_AFS_all',
data_filter=1,
chunk_size=10000):
h5f = h5py.File('%s1k_genomes_hg.hdf5' % kg_dir)
ancestries = sp.unique(h5f['indivs']['ancestry'][...])
#ancestries = sp.unique(h5f['indivs']['continent'][...])
for ancestry in ancestries:
anc_filter = h5f['indivs']['ancestry'][...] == ancestry
#anc_filter = h5f['indivs']['continent'][...]==ancestry
num_indivs = sp.sum(anc_filter)
print "%d individuals with %s ancestry are used" % (num_indivs,
ancestry)
sids_list = []
acs = []
for chrom in range(1, 23):
print 'Working on chromosome %d' % chrom
chr_str = 'chr%d' % chrom
num_snps = len(h5f[chr_str]['calldata']['snps'])
assert num_snps == len(h5f[chr_str]['variants']['ID']), 'WTF?'
for start_i in range(0, num_snps, chunk_size):
snps = sp.array(
h5f[chr_str]['calldata']['snps'][start_i:start_i +
chunk_size],
dtype='int8')
sids = h5f[chr_str]['variants']['ID'][start_i:start_i +
chunk_size]
snps = snps[:, anc_filter]
if data_filter < 1:
rand_filt = sp.random.random(len(snps))
rand_filt = rand_filt < data_filter
snps = snps[rand_filt]
sids = sids[rand_filt]
(m, n) = snps.shape
ac = sp.sum(snps, 1)
flip_filter = ac > n
ac[flip_filter] = 2 * n - ac[flip_filter]
#Plotting filter
acs.extend(ac)
sids_list.extend(sids)
if start_i % 1000000 == 0:
print 'Parsed %d SNPs' % start_i
print '%d SNPs loaded and filtered' % num_snps
print '%d ACs and SIDs found' % len(acs)
print 'Storing the AFS'
with open('%s_%s.txt' % (outfile_prefix, ancestry), 'w') as f:
f.write('# %d individuals used\n' % num_indivs)
f.write('SID AC\n')
for sid, ac in izip(sids_list, acs):
f.write('%s %d\n' % (sid, ac))
print 'Plot things'
acs = sp.array(acs, dtype='int')
acs = acs[acs > 0]
acs = acs[acs < 30]
min_ac = acs.min()
max_ac = acs.max()
sp.bincount(acs)
plt.clf()
plt.hist(acs, bins=sp.arange(min_ac - 0.5, max_ac + 1.5, 1))
plt.title('%s AFS' % ancestry)
plt.savefig('%s_%s.png' % (plot_prefix, ancestry))
h5f.close()
def cvglmnet(*,
x,
y,
family='gaussian',
ptype='default',
nfolds=10,
foldid=scipy.empty([0]),
parallel=False,
keep=False,
grouped=True,
**options):
options = glmnetSet(options)
if 0 < len(options['lambdau']) < 2:
raise ValueError('Need more than one value of lambda for cv.glmnet')
nobs = x.shape[0]
# we should not really need this. user must supply the right shape
# if y.shape[0] != nobs:
# y = scipy.transpose(y)
# convert 1d python array of size nobs to 2d python array of size nobs x 1
if len(y.shape) == 1:
y = scipy.reshape(y, [y.size, 1])
# we should not really need this. user must supply the right shape
# if (len(options['offset']) > 0) and (options['offset'].shape[0] != nobs):
# options['offset'] = scipy.transpose(options['offset'])
if len(options['weights']) == 0:
options['weights'] = scipy.ones([nobs, 1], dtype=scipy.float64)
# main call to glmnet
glmfit = glmnet(x=x, y=y, family=family, **options)
is_offset = glmfit['offset']
options['lambdau'] = glmfit['lambdau']
nz = glmnetPredict(glmfit, scipy.empty([0]), scipy.empty([0]), 'nonzero')
if glmfit['class'] == 'multnet':
nnz = scipy.zeros([len(options['lambdau']), len(nz)])
for i in range(len(nz)):
nnz[:, i] = scipy.transpose(scipy.sum(nz[i], axis=0))
nz = scipy.ceil(scipy.median(nnz, axis=1))
elif glmfit['class'] == 'mrelnet':
nz = scipy.transpose(scipy.sum(nz[0], axis=0))
else:
nz = scipy.transpose(scipy.sum(nz, axis=0))
if len(foldid) == 0:
ma = scipy.tile(scipy.arange(nfolds), [1, scipy.floor(nobs / nfolds)])
mb = scipy.arange(scipy.mod(nobs, nfolds))
mb = scipy.reshape(mb, [1, mb.size])
population = scipy.append(ma, mb, axis=1)
mc = scipy.random.permutation(len(population))
mc = mc[0:nobs]
foldid = population[mc]
foldid = scipy.reshape(foldid, [
foldid.size,
])
else:
nfolds = scipy.amax(foldid) + 1
if nfolds < 3:
raise ValueError(
'nfolds must be bigger than 3; nfolds = 10 recommended')
cpredmat = list()
foldid = scipy.reshape(foldid, [
foldid.size,
])
if parallel == True:
num_cores = multiprocessing.cpu_count()
sys.stderr.write("[status]\tParallel glmnet cv with " +
str(num_cores) + " cores\n")
cpredmat = joblib.Parallel(n_jobs=num_cores)(joblib.delayed(doCV)(
i, x, y, family, foldid, nfolds, is_offset, **options)
for i in range(nfolds))
else:
for i in range(nfolds):
newFit = doCV(i, x, y, family, foldid, nfolds, is_offset,
**options)
cpredmat.append(newFit)
if cpredmat[0]['class'] == 'elnet':
cvstuff = cvelnet( cpredmat, options['lambdau'], x, y \
, options['weights'], options['offset'] \
, foldid, ptype, grouped, keep)
elif cpredmat[0]['class'] == 'lognet':
cvstuff = cvlognet(cpredmat, options['lambdau'], x, y \
, options['weights'], options['offset'] \
, foldid, ptype, grouped, keep)
elif cpredmat[0]['class'] == 'multnet':
cvstuff = cvmultnet(cpredmat, options['lambdau'], x, y \
, options['weights'], options['offset'] \
, foldid, ptype, grouped, keep)
elif cpredmat[0]['class'] == 'mrelnet':
cvstuff = cvmrelnet(cpredmat, options['lambdau'], x, y \
, options['weights'], options['offset'] \
, foldid, ptype, grouped, keep)
elif cpredmat[0]['class'] == 'fishnet':
cvstuff = cvfishnet(cpredmat, options['lambdau'], x, y \
, options['weights'], options['offset'] \
, foldid, ptype, grouped, keep)
elif cpredmat[0]['class'] == 'coxnet':
raise NotImplementedError(
'Cross-validation for coxnet not implemented yet.')
#cvstuff = cvcoxnet(cpredmat, options['lambdau'], x, y \
# , options['weights'], options['offset'] \
# , foldid, ptype, grouped, keep)
cvm = cvstuff['cvm']
cvsd = cvstuff['cvsd']
cvname = cvstuff['name']
CVerr = dict()
CVerr['lambdau'] = options['lambdau']
CVerr['cvm'] = scipy.transpose(cvm)
CVerr['cvsd'] = scipy.transpose(cvsd)
CVerr['cvup'] = scipy.transpose(cvm + cvsd)
CVerr['cvlo'] = scipy.transpose(cvm - cvsd)
CVerr['nzero'] = nz
CVerr['name'] = cvname
CVerr['glmnet_fit'] = glmfit
if keep:
CVerr['fit_preval'] = cvstuff['fit_preval']
CVerr['foldid'] = foldid
if ptype == 'auc':
cvm = -cvm
CVerr['lambda_min'] = scipy.amax(
options['lambdau'][cvm <= scipy.amin(cvm)]).reshape([1])
idmin = options['lambdau'] == CVerr['lambda_min']
semin = cvm[idmin] + cvsd[idmin]
CVerr['lambda_1se'] = scipy.amax(options['lambdau'][cvm <= semin]).reshape(
[1])
CVerr['class'] = 'cvglmnet'
return (CVerr)
def corr_est(map1,
map2,
noise1,
noise2,
freq1,
freq2,
lags=(),
speedup=False,
verbose=False):
r"""Calculate the cross correlation function of the maps.
The cross correlation function is a function of f1, f2 and angular lag.
The angular lag bins are passed, all pairs of frequencies are
calculated.
Parameters
----------
lags: array like
Angular lags bins (upper side bin edges).
speedup: boolean
Speeds up the correlation. This works fine, yes? Should be the
normal way if so.
Returns
-------
corr: array
The correlation between 2 maps.
counts: array
The weighting of the correlation based on the maps' weights.
"""
map1_ra = map1.get_axis('ra')
map2_ra = map2.get_axis('ra')
map1_dec = map1.get_axis('dec')
map2_dec = map2.get_axis('dec')
input_map1 = map1[freq1, :, :]
input_map2 = map2[freq2, :, :]
input_noise1 = noise1[freq1, :, :]
input_noise2 = noise2[freq2, :, :]
# Noise weight
input_map1 *= input_noise1
input_map2 *= input_noise2
nlags = len(lags)
nfreq = len(freq1)
corr = sp.zeros((nfreq, nfreq, nlags), dtype=float)
counts = sp.zeros(corr.shape, dtype=float)
# Noting that if DEC != 0, then a degree of RA is less than a degree.
ra_fact = sp.cos(sp.pi * map1.info['dec_centre'] / 180.0)
# Calculate the pairwise lags.
dra = (map1_ra[:, None] - map2_ra[None, :]) * ra_fact
ddec = map1_dec[:, None] - map2_dec[None, :]
lag = dra[:, None, :, None]**2 + ddec[None, :, None, :]**2
lag = sp.sqrt(lag)
# Bin this up.
lag_inds = sp.digitize(lag.flatten(), lags)
if speedup:
print "Starting Correlation (sparse version)"
(nr1, nd1) = (len(map1_ra), len(map1_dec))
(nr2, nd2) = (len(map2_ra), len(map2_dec))
(r1ind, d1ind) = (sp.arange(nr1), sp.arange(nd1))
(r2ind, d2ind) = (sp.arange(nr2), sp.arange(nd2))
ra1_pairind = r1ind.repeat(nr2 * nd1 * nd2)
ra2_pairind = sp.tile(r2ind.repeat(nd2), (1, nr1 * nd1)).flatten()
dec1_pairind = sp.tile(d1ind.repeat(nr2 * nd2), (1, nr1)).flatten()
dec2_pairind = sp.tile(d2ind, (1, nr1 * nr2 * nd1)).flatten()
# precalculate the pair indices for a given lag
# could also imagine calculating the map slices here
posmaskdict = {}
for klag in range(nlags):
mask = (lag_inds == klag)
posmaskdict[repr(klag)] = (ra1_pairind[mask], ra2_pairind[mask],
dec1_pairind[mask], dec2_pairind[mask])
for if1 in range(len(freq1)):
for jf2 in range(len(freq2)):
start = time.time()
data1 = input_map1[if1, :, :]
data2 = input_map2[jf2, :, :]
weights1 = input_noise1[if1, :, :]
weights2 = input_noise2[jf2, :, :]
for klag in range(nlags):
(r1m, r2m, d1m, d2m) = posmaskdict[repr(klag)]
dprod = data1[r1m, d1m] * data2[r2m, d2m]
wprod = weights1[r1m, d1m] * weights2[r2m, d2m]
corr[if1, jf2, klag] += sp.sum(dprod)
counts[if1, jf2, klag] += sp.sum(wprod)
if verbose:
print if1, jf2, (time.time() - start)
print counts[if1, jf2, :]
else:
print "Starting Correlation (full version)"
for if1 in range(len(freq1)):
for jf2 in range(len(freq2)):
start = time.time()
# Calculate the pairwise products.
data1 = input_map1[if1, :, :]
data2 = input_map2[jf2, :, :]
weights1 = input_noise1[if1, :, :]
weights2 = input_noise2[jf2, :, :]
dprod = data1[..., None, None] * data2[None, None, ...]
wprod = weights1[..., None, None] * \
weights2[None, None, ...]
for klag in range(nlags):
mask = (lag_inds == klag)
corr[if1, jf2, klag] += sp.sum(dprod.flatten()[mask])
counts[if1, jf2, klag] += sp.sum(wprod.flatten()[mask])
if verbose:
print if1, jf2, (time.time() - start)
print counts[if1, jf2, :]
mask = (counts < 1e-20)
counts[mask] = 1
corr /= counts
corr[mask] = 0
counts[mask] = 0
return corr, counts
def num_neighbors(self, pores, element='pore', flatten=False,
mode='union'):
r"""
Returns an array containing the number of neigbhoring pores or throats
for each given input pore
Parameters
----------
pores : array_like
Pores whose neighbors are to be counted
flatten : boolean (optional)
If ``False`` (default) the number of pores neighboring each input
pore as an array the same length as ``pores``. If ``True`` the sum
total number of is counted.
element : string
Indicates whether to count number of neighboring pores or throats.
For some complex networks, such as extracted networks, several
throats may exist between two pores, so this query will return
different results depending on whether 'pores' (default) or
'throats' is specified.
mode : string (This is ignored if ``flatten`` is False)
The logic to apply to the returned count of pores.
**'union'** : (Default) The sum of all neighbors connected all
input pores.
**'intersection'** : The number of neighboring pores that are
shared by all input pores.
**'not_intersection'** : The number of neighboring pores that are
NOT shared by any input pores.
Returns
-------
If ``flatten`` is False, a 1D array with number of neighbors in each
element, otherwise a scalar value of the number of neighbors.
Notes
-----
This method literally just counts the number of elements in the array
returned by ``find_neighbor_pores`` or ``find_neighbor_throats`` and
uses the same logic. Explore those methods if uncertain about the
meaning of the ``mode`` argument here.
See Also
--------
find_neighbor_pores
find_neighbor_throats
Examples
--------
>>> import OpenPNM
>>> pn = OpenPNM.Network.TestNet()
>>> pn.num_neighbors(pores=[0, 1], flatten=False)
array([3, 4])
>>> pn.num_neighbors(pores=[0, 1], flatten=True)
5
>>> pn.num_neighbors(pores=[0, 2], flatten=True)
6
>>> pn.num_neighbors(pores=[0, 1], element='throat', mode='union',
... flatten=True)
6
"""
pores = self._parse_locations(pores)
# Count number of neighbors
num = self._find_neighbors(pores, element=element, flatten=flatten,
mode=mode, excl_self=True)
num = sp.array([sp.size(i) for i in num], dtype=int)
if flatten:
num = sp.sum(num)
num = int(num)
return num
# load genotypes
geno_filename = os.path.join(data_dir, 'genotypes.csv')
X = SP.genfromtxt(geno_filename, delimiter=',')
X = X.T
print X.shape
[n_s, n_f] = X.shape
# simulate phenotype
idx = []
for i in range(4):
idxx = int(SP.rand() * 19) * 50
for j in range(10):
idx += [idxx + int(SP.rand() * 50)]
print idx
ypheno = SP.sum(X[:, idx], axis=1)
ypheno = SP.reshape(ypheno, (n_s, 1))
ypheno = (ypheno - ypheno.mean()) / ypheno.std()
pheno_filename = os.path.join(data_dir, 'poppheno.csv')
ypop = SP.genfromtxt(pheno_filename)
print ypop.shape
ypop = SP.reshape(ypop, (n_s, 1))
y = 0.3 * ypop + 0.5 * ypheno + 0.2 * SP.random.randn(n_s, 1)
y = (y - y.mean()) / y.std()
# init
debug = False
n_train = 150
n_test = n_s - n_train
n_reps = 8
f_subset = 0.5
def check_network_health(self):
r"""
This method check the network topological health by checking for:
(1) Isolated pores
(2) Islands or isolated clusters of pores
(3) Duplicate throats
(4) Bidirectional throats (ie. symmetrical adjacency matrix)
(5) Headless throats
Returns
-------
A dictionary containing the offending pores or throat numbers under
each named key.
It also returns a list of which pores and throats should be trimmed
from the network to restore health. This list is a suggestion only,
and is based on keeping the largest cluster and trimming the others.
Notes
-----
- Does not yet check for duplicate pores
- Does not yet suggest which throats to remove
- This is just a 'check' method and does not 'fix' the problems it finds
"""
health = Tools.HealthDict()
health['disconnected_clusters'] = []
health['isolated_pores'] = []
health['trim_pores'] = []
health['duplicate_throats'] = []
health['bidirectional_throats'] = []
health['headless_throats'] = []
health['looped_throats'] = []
# Check for headless throats
hits = sp.where(self['throat.conns'] > self.Np - 1)[0]
if sp.size(hits) > 0:
health['headless_throats'] = sp.unique(hits)
logger.warning('Health check cannot complete due to connectivity '
'errors. Please correct existing errors & recheck.')
return health
# Check for throats that loop back onto the same pore
P12 = self['throat.conns']
hits = sp.where(P12[:, 0] == P12[:, 1])[0]
if sp.size(hits) > 0:
health['looped_throats'] = hits
# Check for individual isolated pores
Ps = self.num_neighbors(self.pores())
if sp.sum(Ps == 0) > 0:
logger.warning(str(sp.sum(Ps == 0)) + ' pores have no neighbors')
health['isolated_pores'] = sp.where(Ps == 0)[0]
# Check for separated clusters of pores
temp = []
Cs = self.find_clusters(self.tomask(throats=self.throats('all')))
if sp.shape(sp.unique(Cs))[0] > 1:
logger.warning('Isolated clusters exist in the network')
for i in sp.unique(Cs):
temp.append(sp.where(Cs == i)[0])
b = sp.array([len(item) for item in temp])
c = sp.argsort(b)[::-1]
for i in range(0, len(c)):
health['disconnected_clusters'].append(temp[c[i]])
if i > 0:
health['trim_pores'].extend(temp[c[i]])
# Check for duplicate throats
am = self.create_adjacency_matrix(sprsfmt='lil')
mergeTs = []
for i in range(0, self.Np):
for j in sp.where(sp.array(am.data[i]) > 1)[0]:
k = am.rows[i][j]
mergeTs.extend(self.find_connecting_throat(i, k))
# Remove duplicates
mergeTs = [list(i) for i in set(tuple(i) for i in mergeTs)]
health['duplicate_throats'] = mergeTs
# Check for bidirectional throats
adjmat = self.create_adjacency_matrix(sprsfmt='coo')
num_full = adjmat.sum()
temp = sprs.triu(adjmat, k=1)
num_upper = temp.sum()
if num_full > num_upper:
biTs = sp.where(self['throat.conns'][:, 0] >
self['throat.conns'][:, 1])[0]
health['bidirectional_throats'] = biTs.tolist()
return health
def ldpred_genomewide(data_file=None,
ld_radius=None,
ld_dict=None,
out_file_prefix=None,
summary_dict=None,
ps=None,
n=None,
h2=None,
num_iter=None,
verbose=False,
zero_jump_prob=0.05,
burn_in=5):
"""
Calculate LDpred for a genome
"""
df = h5py.File(data_file, 'r')
has_phenotypes = False
if 'y' in df:
'Validation phenotypes found.'
y = df['y'][...] # Phenotype
num_individs = len(y)
risk_scores_pval_derived = sp.zeros(num_individs)
has_phenotypes = True
ld_scores_dict = ld_dict['ld_scores_dict']
chrom_ld_dict = ld_dict['chrom_ld_dict']
chrom_ref_ld_mats = ld_dict['chrom_ref_ld_mats']
print('Applying LDpred with LD radius: %d' % ld_radius)
results_dict = {}
cord_data_g = df['cord_data']
#Calculating genome-wide heritability using LD score regression, and partition heritability by chromsomes
herit_dict = ld.get_chromosome_herits(cord_data_g,
ld_scores_dict,
n,
h2=h2,
debug=verbose,
summary_dict=summary_dict)
LDpred_inf_chrom_dict = {}
print('Calculating LDpred-inf weights')
for chrom_str in util.chromosomes_list:
if chrom_str in cord_data_g:
print('Calculating SNP weights for Chromosome %s' %
((chrom_str.split('_'))[1]))
g = cord_data_g[chrom_str]
# Filter monomorphic SNPs
snp_stds = g['snp_stds_ref'][...]
snp_stds = snp_stds.flatten()
ok_snps_filter = snp_stds > 0
pval_derived_betas = g['betas'][...]
pval_derived_betas = pval_derived_betas[ok_snps_filter]
h2_chrom = herit_dict[chrom_str]
start_betas = LDpred_inf.ldpred_inf(
pval_derived_betas,
genotypes=None,
reference_ld_mats=chrom_ref_ld_mats[chrom_str],
h2=h2_chrom,
n=n,
ld_window_size=2 * ld_radius,
verbose=False)
LDpred_inf_chrom_dict[chrom_str] = start_betas
convergence_report = {}
for p in ps:
convergence_report[p] = False
print('Starting LDpred gibbs with f=%0.4f' % p)
p_str = '%0.4f' % p
results_dict[p_str] = {}
if out_file_prefix:
# Preparing output files
raw_effect_sizes = []
ldpred_effect_sizes = []
ldpred_inf_effect_sizes = []
out_sids = []
chromosomes = []
out_positions = []
out_nts = []
chrom_i = 0
num_chrom = len(util.chromosomes_list)
for chrom_str in util.chromosomes_list:
chrom_i += 1
if chrom_str in cord_data_g:
g = cord_data_g[chrom_str]
if verbose and has_phenotypes:
if 'raw_snps_val' in g:
raw_snps = g['raw_snps_val'][...]
else:
raw_snps = g['raw_snps_ref'][...]
# Filter monomorphic SNPs
snp_stds = g['snp_stds_ref'][...]
snp_stds = snp_stds.flatten()
pval_derived_betas = g['betas'][...]
positions = g['positions'][...]
sids = (g['sids'][...]).astype(util.sids_u_dtype)
log_odds = g['log_odds'][...]
nts = (g['nts'][...]).astype(util.nts_u_dtype)
ok_snps_filter = snp_stds > 0
if not sp.all(ok_snps_filter):
snp_stds = snp_stds[ok_snps_filter]
pval_derived_betas = pval_derived_betas[ok_snps_filter]
positions = positions[ok_snps_filter]
sids = sids[ok_snps_filter]
log_odds = log_odds[ok_snps_filter]
nts = nts[ok_snps_filter]
if verbose and has_phenotypes:
raw_snps = raw_snps[ok_snps_filter]
if out_file_prefix:
chromosomes.extend([chrom_str] * len(pval_derived_betas))
out_positions.extend(positions)
out_sids.extend(sids)
raw_effect_sizes.extend(log_odds)
out_nts.extend(nts)
h2_chrom = herit_dict[chrom_str]
if 'chrom_ld_boundaries' in ld_dict:
ld_boundaries = ld_dict['chrom_ld_boundaries'][chrom_str]
res_dict = ldpred_gibbs(
pval_derived_betas,
h2=h2_chrom,
n=n,
p=p,
ld_radius=ld_radius,
verbose=verbose,
num_iter=num_iter,
burn_in=burn_in,
ld_dict=chrom_ld_dict[chrom_str],
start_betas=LDpred_inf_chrom_dict[chrom_str],
ld_boundaries=ld_boundaries,
zero_jump_prob=zero_jump_prob,
print_progress=False)
else:
res_dict = ldpred_gibbs(
pval_derived_betas,
h2=h2_chrom,
n=n,
p=p,
ld_radius=ld_radius,
verbose=verbose,
num_iter=num_iter,
burn_in=burn_in,
ld_dict=chrom_ld_dict[chrom_str],
start_betas=LDpred_inf_chrom_dict[chrom_str],
zero_jump_prob=zero_jump_prob,
print_progress=False)
updated_betas = res_dict['betas']
updated_inf_betas = res_dict['inf_betas']
sum_sqr_effects = sp.sum(updated_betas**2)
if sum_sqr_effects > herit_dict['gw_h2_ld_score_est']:
print(
'Sum of squared updated effects estimates seems too large: %0.4f'
% sum_sqr_effects)
print(
'This suggests that the Gibbs sampler did not convergence.'
)
convergence_report[p] = True
if verbose:
print('Calculating SNP weights for Chromosome %s' %
((chrom_str.split('_'))[1]))
else:
sys.stdout.write('\b\b\b\b\b\b\b%0.2f%%' %
(100.0 *
(min(1,
float(chrom_i + 1) / num_chrom))))
sys.stdout.flush()
updated_betas = updated_betas / (snp_stds.flatten())
updated_inf_betas = updated_inf_betas / (snp_stds.flatten())
ldpred_effect_sizes.extend(updated_betas)
ldpred_inf_effect_sizes.extend(updated_inf_betas)
if verbose and has_phenotypes:
prs = sp.dot(updated_betas, raw_snps)
risk_scores_pval_derived += prs
corr = sp.corrcoef(y, prs)[0, 1]
r2 = corr**2
print(
'The R2 prediction accuracy of PRS using %s was: %0.4f'
% (chrom_str, r2))
if not verbose:
sys.stdout.write('\b\b\b\b\b\b\b%0.2f%%\n' % (100.0))
sys.stdout.flush()
if verbose and has_phenotypes:
num_indivs = len(y)
results_dict[p_str]['y'] = y
results_dict[p_str]['risk_scores_pd'] = risk_scores_pval_derived
print('Prediction accuracy was assessed using %d individuals.' %
(num_indivs))
corr = sp.corrcoef(y, risk_scores_pval_derived)[0, 1]
r2 = corr**2
results_dict[p_str]['r2_pd'] = r2
print(
'The R2 prediction accuracy (observed scale) for the whole genome was: %0.4f (%0.6f)'
% (r2, ((1 - r2)**2) / num_indivs))
if corr < 0:
risk_scores_pval_derived = -1 * risk_scores_pval_derived
auc = util.calc_auc(y, risk_scores_pval_derived)
print('AUC for the whole genome was: %0.4f' % auc)
# Now calibration
denominator = sp.dot(risk_scores_pval_derived.T,
risk_scores_pval_derived)
y_norm = (y - sp.mean(y)) / sp.std(y)
numerator = sp.dot(risk_scores_pval_derived.T, y_norm)
regression_slope = (numerator / denominator) # [0][0]
print(
'The slope for predictions with P-value derived effects is: %0.4f'
% regression_slope)
results_dict[p_str]['slope_pd'] = regression_slope
weights_out_file = '%s_LDpred_p%0.4e.txt' % (out_file_prefix, p)
with open(weights_out_file, 'w') as f:
f.write(
'chrom pos sid nt1 nt2 raw_beta ldpred_beta\n'
)
for chrom, pos, sid, nt, raw_beta, ldpred_beta in zip(
chromosomes, out_positions, out_sids, out_nts,
raw_effect_sizes, ldpred_effect_sizes):
nt1, nt2 = nt[0], nt[1]
f.write('%s %d %s %s %s %0.4e %0.4e\n' %
(chrom, pos, sid, nt1, nt2, raw_beta, ldpred_beta))
weights_out_file = '%s_LDpred-inf.txt' % (out_file_prefix)
with open(weights_out_file, 'w') as f:
f.write(
'chrom pos sid nt1 nt2 raw_beta ldpred_inf_beta \n'
)
for chrom, pos, sid, nt, raw_beta, ldpred_inf_beta in zip(
chromosomes, out_positions, out_sids, out_nts,
raw_effect_sizes, ldpred_inf_effect_sizes):
nt1, nt2 = nt[0], nt[1]
f.write('%s %d %s %s %s %0.4e %0.4e\n' %
(chrom, pos, sid, nt1, nt2, raw_beta, ldpred_inf_beta))
summary_dict[2.0] = {
'name': 'Gibbs sampler fractions used',
'value': str(ps)
}
['Yes' if convergence_report[p] else 'No' for p in ps]
summary_dict[2.1] = {
'name': 'Convergence issues (for each fraction)',
'value': str(['Yes' if convergence_report[p] else 'No' for p in ps])
}
def VCA(Y, d, q, verbose=False, snr_input=0):
R = q
# Initializations
if len(Y.shape) != 2:
sys.exit(
'Input data must be of size L (number of bands i.e. channels) by N (number of pixels)'
)
[L, N] = Y.shape # L number of bands (channels), N number of pixels
R = int(R)
if (R < 0 or R > L):
sys.exit('ENDMEMBER parameter must be integer between 1 and L')
start_time = time.time()
# SNR Estimates
if snr_input == 0:
y_m = np.mean(Y, axis=1, keepdims=True)
Y_o = Y - y_m # data with zero-mean
Ud = linalg.svd(sp.dot(Y_o, Y_o.T) /
float(N))[0][:, :R] # computes the R-projection matrix
x_p = sp.dot(Ud.T, Y_o) # project the zero-mean data onto p-subspace
SNR = auxiliar.estimate_snr(Y, y_m, x_p)
if verbose:
print("SNR estimated = {}[dB]".format(SNR))
else:
SNR = snr_input
if verbose:
print("input SNR = {}[dB]\n".format(SNR))
SNR_th = 15 + 10 * sp.log10(R)
# Choosing Projective Projection or
# projection to p-1 subspace
if SNR < SNR_th:
if verbose:
print("... Select proj. to R-1")
d = R - 1
if snr_input == 0: # it means that the projection is already computed
Ud = Ud[:, :d]
else:
y_m = sp.mean(Y, axis=1, keepdims=True)
Y_o = Y - y_m # data with zero-mean
Ud = linalg.svd(
sp.dot(Y_o, Y_o.T) /
float(N))[0][:, :d] # computes the p-projection matrix
x_p = sp.dot(Ud.T,
Y_o) # project thezeros mean data onto p-subspace
Yp = sp.dot(Ud, x_p[:d, :]) + y_m # again in dimension L
x = x_p[:d, :] # x_p = Ud.T * Y_o is on a R-dim subspace
c = sp.amax(sp.sum(x**2, axis=0))**0.5
y = sp.vstack((x, c * sp.ones((1, N))))
else:
if verbose:
print("... Select the projective proj.")
d = R
Ud = linalg.svd(
sp.dot(Y, Y.T) /
float(N))[0][:, :d] # computes the p-projection matrix
x_p = sp.dot(Ud.T, Y)
Yp = sp.dot(
Ud, x_p[:d, :]
) # again in dimension L (note that x_p has no null mean)
x = sp.dot(Ud.T, Y)
u = sp.mean(x, axis=1,
keepdims=True) #equivalent to u = Ud.T * r_m
y = x / sp.dot(u.T, x)
# VCA algorithm
indice = sp.zeros((R), dtype=int)
A = sp.zeros((R, R))
A[-1, 0] = 1
for i in range(R):
w = sp.random.rand(R, 1)
f = w - sp.dot(A, sp.dot(linalg.pinv(A), w))
f = f / linalg.norm(f)
v = sp.dot(f.T, y)
indice[i] = sp.argmax(sp.absolute(v))
A[:, i] = y[:, indice[i]] # same as x(:,indice(i))
Ae = Yp[:, indice]
duration = time.time() - start_time
M = Y[:, indice]
return M, duration, [indice]
def error(f, x, y):
return sp.sum((f(x) - y)**2)
def normold(vec):
""" Normalize a vector """
return sp.sqrt(sp.sum(vec**2))
def test_Darcy_alg():
# Generate Network and clean up some of boundaries
divs = [1, 50, 10]
Lc = 0.00004
pn = OpenPNM.Network.Cubic(shape=divs, spacing=Lc)
pn.add_boundaries()
Ps = pn.pores(['front_boundary', 'back_boundary'])
pn.trim(pores=Ps)
# Generate Geometry objects for internal and boundary pores
Ps = pn.pores('boundary', mode='not')
Ts = pn.find_neighbor_throats(pores=Ps, mode='intersection', flatten=True)
geom = OpenPNM.Geometry.Toray090(network=pn, pores=Ps, throats=Ts)
Ps = pn.pores('boundary')
Ts = pn.find_neighbor_throats(pores=Ps, mode='not_intersection')
boun = OpenPNM.Geometry.Boundary(network=pn, pores=Ps, throats=Ts)
# Create Phase object and associate with a Physics object
air = OpenPNM.Phases.Air(network=pn)
Ps = pn.pores()
Ts = pn.throats()
phys = OpenPNM.Physics.GenericPhysics(network=pn,
phase=air,
pores=Ps,
throats=Ts)
from OpenPNM.Physics import models as pm
phys.add_model(propname='throat.hydraulic_conductance',
model=pm.hydraulic_conductance.hagen_poiseuille,
calc_pore_len=False)
phys.regenerate() # Update the conductance values
# Setup Algorithm objects
Darcy1 = OpenPNM.Algorithms.StokesFlow(network=pn, phase=air)
inlets = pn.pores('bottom_boundary')
Ps = pn.pores('top_boundary')
outlets = Ps[pn['pore.coords'][Ps, 1] < (divs[1] * Lc / 2)]
P_out = 0 # Pa
Q_in = 0.6667 * (Lc**2) * divs[1] * divs[0] # m^3/s
Darcy1.set_boundary_conditions(bctype='Neumann_group',
bcvalue=-Q_in,
pores=inlets)
Darcy1.set_boundary_conditions(bctype='Dirichlet',
bcvalue=P_out,
pores=outlets)
Darcy1.run()
Darcy1.return_results()
print('pore pressure for Darcy1 algorithm:')
print((air['pore.pressure']))
Darcy2 = OpenPNM.Algorithms.StokesFlow(network=pn, phase=air)
inlets = pn.pores('bottom_boundary')
outlets = pn.pores('top_boundary')
P_out = 10 # Pa
P_in = 1000 # Pa
Darcy2.set_boundary_conditions(bctype='Dirichlet',
bcvalue=P_in,
pores=inlets)
Darcy2.set_boundary_conditions(bctype='Dirichlet',
bcvalue=P_out,
pores=outlets)
Darcy2.run()
print('pore pressure for Darcy2 algorithm:')
print((Darcy2['pore.pressure']))
Q = -Darcy2.rate(inlets)
K = Q * air['pore.viscosity'][0] * divs[2] * Lc / (divs[0] * divs[1] *
Lc**2 * (P_in - P_out))
Vp = sp.sum(pn['pore.volume']) + sp.sum(pn['throat.volume'])
Vb = sp.prod(divs) * Lc**3
e = Vp / Vb
print(('Effective permeability: ', K, '- Porosity: ', e))
a = round(sp.absolute(Darcy1.rate(outlets))[0], 16)
pore_prop = 'pore.bcval_Neumann_group'
b = round(sp.absolute(sp.unique(Darcy1[pore_prop]))[0], 16)
assert a == b
a = round(sp.absolute(Darcy2.rate(inlets))[0], 16)
b = round(sp.absolute(Darcy2.rate(outlets))[0], 16)
assert a == b
def ldpred_gibbs(beta_hats,
genotypes=None,
start_betas=None,
h2=None,
n=1000,
ld_radius=100,
num_iter=60,
burn_in=10,
p=None,
zero_jump_prob=0.05,
tight_sampling=False,
ld_dict=None,
reference_ld_mats=None,
ld_boundaries=None,
verbose=False,
print_progress=True):
"""
LDpred (Gibbs Sampler)
"""
t0 = time.time()
m = len(beta_hats)
n = float(n)
# If no starting values for effects were given, then use the infinitesimal model starting values.
if start_betas is None and verbose:
print(
'Initializing LDpred effects with posterior mean LDpred-inf effects.'
)
print('Calculating LDpred-inf effects.')
start_betas = LDpred_inf.ldpred_inf(
beta_hats,
genotypes=genotypes,
reference_ld_mats=reference_ld_mats,
h2=h2,
n=n,
ld_window_size=2 * ld_radius,
verbose=False)
curr_betas = sp.copy(start_betas)
assert len(
curr_betas
) == m, 'Betas returned by LDpred_inf do not have the same length as expected.'
curr_post_means = sp.zeros(m)
avg_betas = sp.zeros(m)
# Iterating over effect estimates in sequential order
iter_order = sp.arange(m)
# Setting up the marginal Bayes shrink
Mp = m * p
hdmp = (h2 / Mp)
hdmpn = hdmp + 1.0 / n
hdmp_hdmpn = (hdmp / hdmpn)
c_const = (p / sp.sqrt(hdmpn))
d_const = (1.0 - p) / (sp.sqrt(1.0 / n))
for k in range(num_iter): # Big iteration
# Force an alpha shrink if estimates are way off compared to heritability estimates. (Improves MCMC convergence.)
h2_est = max(0.00001, sp.sum(curr_betas**2))
if tight_sampling:
alpha = min(1.0 - zero_jump_prob, 1.0 / h2_est,
(h2 + 1.0 / sp.sqrt(n)) / h2_est)
else:
alpha = 1.0 - zero_jump_prob
rand_ps = sp.random.random(m)
rand_norms = stats.norm.rvs(0.0, (hdmp_hdmpn) * (1.0 / n), size=m)
if ld_boundaries is None:
for i, snp_i in enumerate(iter_order):
start_i = max(0, snp_i - ld_radius)
focal_i = min(ld_radius, snp_i)
stop_i = min(m, snp_i + ld_radius + 1)
# Local LD matrix
D_i = ld_dict[snp_i]
# Local (most recently updated) effect estimates
local_betas = curr_betas[start_i:stop_i]
# Calculate the local posterior mean, used when sampling.
local_betas[focal_i] = 0.0
res_beta_hat_i = beta_hats[snp_i] - sp.dot(D_i, local_betas)
b2 = res_beta_hat_i**2
d_const_b2_exp = d_const * sp.exp(-b2 * n / 2.0)
if sp.isreal(d_const_b2_exp):
numerator = c_const * sp.exp(-b2 / (2.0 * hdmpn))
if sp.isreal(numerator):
if numerator == 0.0:
postp = 0.0
else:
postp = numerator / (numerator + d_const_b2_exp)
assert sp.isreal(
postp
), 'The posterior mean is not a real number? Possibly due to problems with summary stats, LD estimates, or parameter settings.'
else:
postp = 0.0
else:
postp = 1.0
curr_post_means[snp_i] = hdmp_hdmpn * postp * res_beta_hat_i
if rand_ps[i] < postp * alpha:
# Sample from the posterior Gaussian dist.
proposed_beta = rand_norms[i] + hdmp_hdmpn * res_beta_hat_i
else:
# Sample 0
proposed_beta = 0.0
curr_betas[snp_i] = proposed_beta # UPDATE BETA
else:
for i, snp_i in enumerate(iter_order):
start_i = ld_boundaries[snp_i][0]
stop_i = ld_boundaries[snp_i][1]
focal_i = snp_i - start_i
# Local LD matrix
D_i = ld_dict[snp_i]
# Local (most recently updated) effect imates
local_betas = curr_betas[start_i:stop_i]
# Calculate the local posterior mean, used when sampling.
local_betas[focal_i] = 0.0
res_beta_hat_i = beta_hats[snp_i] - sp.dot(D_i, local_betas)
b2 = res_beta_hat_i**2
d_const_b2_exp = d_const * sp.exp(-b2 * n / 2.0)
if sp.isreal(d_const_b2_exp):
numerator = c_const * sp.exp(-b2 / (2.0 * hdmpn))
if sp.isreal(numerator):
if numerator == 0.0:
postp = 0.0
else:
postp = numerator / (numerator + d_const_b2_exp)
assert sp.isreal(
postp
), 'Posterior mean is not a real number? Possibly due to problems with summary stats, LD estimates, or parameter settings.'
else:
postp = 0.0
else:
postp = 1.0
curr_post_means[snp_i] = hdmp_hdmpn * postp * res_beta_hat_i
if rand_ps[i] < postp * alpha:
# Sample from the posterior Gaussian dist.
proposed_beta = rand_norms[i] + hdmp_hdmpn * res_beta_hat_i
else:
# Sample 0
proposed_beta = 0.0
curr_betas[snp_i] = proposed_beta # UPDATE BETA
if verbose and print_progress:
sys.stdout.write('\b\b\b\b\b\b\b%0.2f%%' %
(100.0 * (min(1,
float(k + 1) / num_iter))))
sys.stdout.flush()
if k >= burn_in:
avg_betas += curr_post_means # Averaging over the posterior means instead of samples.
if verbose and print_progress:
sys.stdout.write('\b\b\b\b\b\b\b%0.2f%%\n' % (100.0))
sys.stdout.flush()
avg_betas = avg_betas / float(num_iter - burn_in)
t1 = time.time()
t = (t1 - t0)
if verbose:
print('Took %d minutes and %0.2f seconds' % (t / 60, t % 60))
return {'betas': avg_betas, 'inf_betas': start_betas}
def sunsal(M,
y,
AL_iters=1000,
lambda_0=0.,
positivity=False,
addone=False,
tol=1e-4,
x0=None,
verbose=False):
[LM, p] = M.shape # mixing matrixsize
[L, N] = y.shape # data set size
if LM != L:
sys.exit('mixing matrix M and data set y are inconsistent')
AL_iters = int(AL_iters)
if (AL_iters < 0):
sys.exit('AL_iters must a positive integer')
# If lambda is scalar convert it into vector
lambda_0 = (lambda_0 * sp.ones((N, p))).T
if (lambda_0 < 0).any():
sys.exit('lambda_0 must be positive')
# compute mean norm
norm_m = splin.norm(M) * (25 + p) / float(p)
# rescale M and Y and lambda
M = M / norm_m
y = y / norm_m
lambda_0 = lambda_0 / norm_m**2
if x0 is not None:
if (x0.shape[0] == p) or (x0.shape[0] == N):
sys.exit('initial X is not inconsistent with M or Y')
#---------------------------------------------
# just least squares
#---------------------------------------------
if (lambda_0.sum() == 0) and (not positivity) and (not addone):
z = sp.dot(splin.pinv(M), y)
# primal and dual residues
res_p = 0.
res_d = 0.
return z, res_p, res_d, None
#---------------------------------------------
# least squares constrained (sum(x) = 1)
#---------------------------------------------
SMALL = 1e-12
if (lambda_0.sum() == 0) and (addone) and (not positivity):
F = sp.dot(M.T, M)
# test if F is invertible
if LA.cond(F) > SMALL:
# compute the solution explicitly
IF = splin.inv(F)
z = sp.dot(sp.dot(IF, M.T), y) - (1. / IF.sum()) * sp.dot(
sp.sum(IF, axis=1, keepdims=True),
(sp.dot(sp.dot(sp.sum(IF, axis=0, keepdims=True), M.T), y) -
1.))
# primal and dual residues
res_p = 0
res_d = 0
return z, res_p, res_d, None
else:
sys.exit('Bad conditioning of M.T*M')
#---------------------------------------------
# Constants and initializations
#---------------------------------------------
mu_AL = 0.01
mu = 10 * lambda_0.mean() + mu_AL
[UF, SF] = splin.svd(sp.dot(M.T, M))[:2]
IF = sp.dot(sp.dot(UF, sp.diag(1. / (SF + mu))), UF.T)
Aux = (1. / IF.sum()) * sp.sum(IF, axis=1, keepdims=True)
x_aux = sp.sum(Aux, axis=1, keepdims=True)
IF1 = IF - sp.dot(Aux, sp.sum(IF, axis=0, keepdims=True))
yy = sp.dot(M.T, y)
#---------------------------------------------
# Initializations
#---------------------------------------------
# no intial solution supplied
if x0 is None:
x = sp.dot(sp.dot(IF, M.T), y)
else:
x = x0
z = x
# scaled Lagrange Multipliers
d = 0 * z
#---------------------------------------------
# AL iterations - main body
#---------------------------------------------
tol1 = sp.sqrt(N * p) * tol
tol2 = sp.sqrt(N * p) * tol
i = 1
res_p = sp.inf
res_d = sp.inf
maskz = sp.ones(z.shape)
mu_changed = 0
#--------------------------------------------------------------------------
# constrained leat squares (CLS) X >= 0
#--------------------------------------------------------------------------
if (lambda_0.sum() == 0) and (not addone):
while (i <= AL_iters) and ((abs(res_p) > tol1) or (abs(res_d) > tol2)):
# save z to be used later
if (i % 10) == 1:
z0 = z
# minimize with respect to z
z = sp.maximum(x - d, 0)
# minimize with respect to x
x = sp.dot(IF, yy + mu * (z + d))
# Lagrange multipliers update
d -= (x - z)
# update mu so to keep primal and dual residuals whithin a factor of 10
if (i % 10) == 1:
# primal residue
res_p = splin.norm(x - z)
# dual residue
res_d = mu * splin.norm(z - z0)
if verbose:
print("i = {:d}, res_p = {:f}, res_d = {:f}\n").format(
i, res_p, res_d)
# update mu
if res_p > 10 * res_d:
mu = mu * 2
d = d / 2
mu_changed = True
elif res_d > 10 * res_p:
mu = mu / 2
d = d * 2
mu_changed = True
if mu_changed:
# update IF and IF1
IF = sp.dot(sp.dot(UF, sp.diag(1. / (SF + mu))), UF.T)
# Aux = (1./IF.sum()) * sp.sum(IF,axis=1,keepdims=True)
# x_aux = sp.sum(Aux,axis=1,keepdims=True)
# IF1 = IF - sp.dot(Aux,sp.sum(IF,axis=0,keepdims=True))
mu_changed = False
i += 1
#--------------------------------------------------------------------------
# Fully constrained leat squares (FCLS) X >= 0
#--------------------------------------------------------------------------
elif (lambda_0.sum() == 0) and addone:
while (i <= AL_iters) and ((abs(res_p) > tol1) or (abs(res_d) > tol2)):
# save z to be used later
if (i % 10) == 1:
z0 = z
# minimize with respect to z
z = sp.maximum(x - d, 0)
# minimize with respect to x
x = sp.dot(IF1, yy + mu * (z + d)) + x_aux
# Lagrange multipliers update
d -= (x - z)
# update mu so to keep primal and dual residuals whithin a factor of 10
if (i % 10) == 1:
# primal residue
res_p = splin.norm(x - z)
# dual residue
res_d = mu * splin.norm(z - z0)
if verbose:
print("i = {:d}, res_p = {:f}, res_d = {:f}\n").format(
i, res_p, res_d)
# update mu
if res_p > 10 * res_d:
mu = mu * 2
d = d / 2
mu_changed = True
elif res_d > 10 * res_p:
mu = mu / 2
d = d * 2
mu_changed = True
if mu_changed:
# update IF and IF1
IF = sp.dot(sp.dot(UF, sp.diag(1. / (SF + mu))), UF.T)
Aux = (1. / IF.sum()) * sp.sum(IF, axis=1, keepdims=True)
x_aux = sp.sum(Aux, axis=1, keepdims=True)
IF1 = IF - sp.dot(Aux, sp.sum(IF, axis=0, keepdims=True))
mu_changed = False
i += 1
#--------------------------------------------------------------------------
# generic SUNSAL: lambda > 0
#--------------------------------------------------------------------------
else:
# implement soft_th
while (i <= AL_iters) and ((abs(res_p) > tol1) or (abs(res_d) > tol2)):
# save z to be used later
if (i % 10) == 1:
z0 = z
# minimize with respect to z
nu = x - d
z = sp.sign(nu) * sp.maximum(sp.absolute(nu) - lambda_0 / mu, 0)
# teste for positivity
if positivity:
z = sp.maximum(z, 0)
# teste for sum-to-one
if addone:
x = sp.dot(IF1, yy + mu * (z + d)) + x_aux
else:
x = sp.dot(IF, yy + mu * (z + d))
# Lagrange multipliers update
d -= (x - z)
# update mu so to keep primal and dual residuals whithin a factor of 10
if (i % 10) == 1:
# primal residue
res_p = splin.norm(x - z)
# dual residue
res_d = mu * splin.norm(z - z0)
if verbose:
print("i = {:d}, res_p = {:f}, res_d = {:f}\n").format(
i, res_p, res_d)
# update mu
if res_p > 10 * res_d:
mu = mu * 2
d = d / 2
mu_changed = True
elif res_d > 10 * res_p:
mu = mu / 2
d = d * 2
mu_changed = True
if mu_changed:
# update IF and IF1
IF = sp.dot(sp.dot(UF, sp.diag(1. / (SF + mu))), UF.T)
Aux = (1. / IF.sum()) * sp.sum(IF, axis=1, keepdims=True)
x_aux = sp.sum(Aux, axis=1, keepdims=True)
IF1 = IF - sp.dot(Aux, sp.sum(IF, axis=0, keepdims=True))
mu_changed = False
i += 1
return x, res_p, res_d, i
def test_linear_solvers():
pn = OpenPNM.Network.Cubic([1, 40, 30], spacing=0.0001)
geom = OpenPNM.Geometry.Toray090(network=pn,
pores=pn.pores(),
throats=pn.throats())
air = OpenPNM.Phases.Air(network=pn)
phys_air = OpenPNM.Physics.Standard(network=pn,
phase=air,
pores=pn.pores(),
throats=pn.throats())
BC1_pores = pn.pores(labels=['left'])
BC2_pores = pn.pores(labels=['right'])
alg_1 = OpenPNM.Algorithms.FickianDiffusion(network=pn, phase=air)
alg_1.set_boundary_conditions(bctype='Dirichlet',
bcvalue=1,
pores=BC1_pores)
alg_1.set_boundary_conditions(bctype='Dirichlet',
bcvalue=0,
pores=BC2_pores)
alg_1.run(iterative_solver='gmres')
alg_2 = OpenPNM.Algorithms.FickianDiffusion(network=pn, phase=air)
alg_2.set_boundary_conditions(bctype='Neumann',
bcvalue=-1e-11,
pores=BC1_pores)
alg_2.set_boundary_conditions(bctype='Dirichlet',
bcvalue=0,
pores=BC2_pores)
alg_2.run(iterative_solver='cg')
alg_3 = OpenPNM.Algorithms.FickianDiffusion(network=pn, phase=air)
alg_3.set_boundary_conditions(bctype='Neumann_group',
bcvalue=-3e-10,
pores=BC1_pores)
alg_3.set_boundary_conditions(bctype='Dirichlet',
bcvalue=0,
pores=BC2_pores)
alg_3.run()
alg_4 = OpenPNM.Algorithms.FickianDiffusion(network=pn, phase=air)
alg_4.set_boundary_conditions(bctype='Neumann_group',
bcvalue=-3e-10,
pores=BC1_pores)
alg_4.set_boundary_conditions(bctype='Dirichlet',
bcvalue=0,
pores=BC2_pores)
alg_4.setup()
alg_4.solve()
assert round(sp.absolute(alg_1.rate(BC1_pores))[0], 14) ==\
round(sp.absolute(alg_1.rate(BC2_pores))[0], 14)
assert round(sp.absolute(alg_2.rate(BC2_pores))[0], 14) ==\
round(sp.absolute(sp.unique(alg_2['pore.bcval_Neumann']))[0] *
len(BC1_pores), 14)
assert round(sp.absolute(alg_3.rate(BC2_pores))[0], 14) ==\
round(sp.absolute(sp.unique(alg_3['pore.bcval_Neumann_group']))[0], 14)
assert round(sp.absolute(alg_4.rate(BC2_pores))[0], 14) ==\
round(sp.absolute(sp.unique(alg_4['pore.bcval_Neumann_group']))[0], 14)
assert round(sp.absolute(sp.sum(alg_1.rate(BC1_pores, mode='single'))), 14) ==\
round(sp.absolute(alg_1.rate(BC1_pores))[0], 14)
assert round(sp.absolute(sp.sum(alg_2.rate(BC2_pores, mode='single'))), 14) ==\
round(sp.absolute(alg_2.rate(BC2_pores))[0], 14)
assert round(sp.absolute(sp.sum(alg_3.rate(BC2_pores, mode='single'))), 14) ==\
round(sp.absolute(alg_3.rate(BC2_pores))[0], 14)
assert round(sp.absolute(sp.sum(alg_4.rate(BC2_pores, mode='single'))), 14) ==\
round(sp.absolute(alg_4.rate(BC2_pores))[0], 14)
BC5_pores = [37, 57]
assert round(sp.absolute(sp.sum(alg_3.rate(BC5_pores, mode='single'))), 14) ==\
round(sp.absolute(alg_3.rate(BC5_pores))[0], 14)
assert round(sp.absolute(sp.sum(alg_4.rate(BC5_pores, mode='single'))), 14) ==\
round(sp.absolute(alg_4.rate(BC5_pores))[0], 14)
BC2_pores = pn.pores('left_boundary')
alg.set_boundary_conditions(bctype='Neumann_group',
bcvalue=0.2 * 1e-11,
pores=BC2_pores)
#-----------------------------------------------------------------------------------------------
alg.set_source_term(source_name=['pore.blah1', 'pore.blah2'],
pores=sp.r_[500:700],
tol=1e-10)
alg.set_source_term(source_name='pore.blah3', pores=sp.r_[800:900])
alg.setup()
alg.solve()
alg.return_results()
print('--------------------------------------------------------------')
print(('steps: ', alg._steps))
print(('tol_reached: ', alg._tol_reached))
print('--------------------------------------------------------------')
print(('reaction from the physics for pores [500:700]:',\
sp.sum(1.5e-13*air['pore.mole_fraction'][sp.r_[500:600]]**1.7+1.5e-14)\
+sp.sum(0.5e-13*air['pore.mole_fraction'][sp.r_[600:700]]**1.5+2.5e-14)\
+sp.sum(0.9e-13*air['pore.mole_fraction'][sp.r_[600:700]]**1.9+4.15e-14)))
print(('rate from the algorithm for pores [500:700]:',
alg.rate(sp.r_[500:700])[0]))
print('--------------------------------------------------------------')
print(
('reaction from the physics for pores [800:900]:',
sp.sum(0.3e-11 *
sp.exp(0.5 * air['pore.mole_fraction'][sp.r_[800:900]]**2 - 0.34) +
2e-14)))
print(('rate from the algorithm for pores [800:900]:',
alg.rate(sp.r_[800:900])[0]))
def CDF(datos, datosmodelo):
modelosort = sp.sort(datosmodelo)
datosort = sp.sort(datos)
cdf = (sp.array([sp.sum(modelosort <= yy)
for yy in datosort]) / (len(datosmodelo)))
return cdf
def normalize(matrix):
norms = sp.sqrt(sp.sum(matrix**2, axis=1))
norms[norms == 0] = 1
matrix /= norms[:, sp.newaxis]
return matrix
def get_scattering_matrix(self):
"""
Calculates the normalized scattering matrix
Scattering matrix is calculated in linear polarization [VV, HV, VH, HH]
If needed, convert to circular [RR, LR, RL, LL] with linear_to_circular.
"""
# Size the scattering matrix
scattering_matrix = zeros((4, len(self.frequency)), dtype=complex)
# Wavelength
wavelength = c / self.frequency
# Wavenumber
k = 2.0 * pi / wavelength
# Number of vertices and faces
nv = self.vertices.shape[0]
nf = self.faces.shape[0]
# Incident angles and direction cosines
cpi = cos(self.phi_inc)
spi = sin(self.phi_inc)
cti = cos(self.theta_inc)
sti = sin(self.theta_inc)
ui = sti * cpi
vi = sti * spi
wi = cti
incident_direction = [ui, vi, wi]
# Observation angles and direction cosines
cpo = cos(self.phi_obs)
spo = sin(self.phi_obs)
cto = cos(self.theta_obs)
sto = sin(self.theta_obs)
uo = sto * cpo
vo = sto * spo
wo = cto
uuo = cto * cpo
vvo = cto * spo
wwo = -sto
# Incident field in global Cartesian
Ei = 1.0
Ei_V = [cti * cpi * Ei, cti * spi * Ei, -sti * Ei]
Ei_H = [-spi * Ei, cpi * Ei, 0.0]
# Position vectors to vertices
r = zeros((nv, 3))
for i_vert in range(nv):
r[i_vert] = [self.vertices[i_vert][0], self.vertices[i_vert][1], self.vertices[i_vert][2]]
# Edge vectors and normals
# Loop over faces
for i_face in range(nf):
A = r[self.faces[i_face][1]] - r[self.faces[i_face][0]]
B = r[self.faces[i_face][2]] - r[self.faces[i_face][1]]
C = r[self.faces[i_face][0]] - r[self.faces[i_face][2]]
# Outward directed normals
normal = cross(A, B) + 0.
# Edge lengths
dist = [norm(A), norm(B), norm(C)]
ss = 0.5 * sum(dist)
area = sqrt(ss * (ss - dist[0]) * (ss - dist[1]) * (ss - dist[2]))
# Unit normals
normal = normal / norm(normal)
# Just a normal check for illumination
if dot(normal, incident_direction) >= 0.0:
# Local angles
beta = arccos(normal[2])
alpha = arctan2(normal[1] + 0., normal[0] + 0.)
# Local direction cosines
ca = cos(alpha)
sa = sin(alpha)
cb = cos(beta)
sb = sin(beta)
# Rotation matrices
rotation1 = array([[ca, sa, 0.0], [-sa, ca, 0.0], [0.0, 0.0, 1.0]])
rotation2 = array([[cb, 0.0, -sb], [0.0, 1.0, 0.0], [sb, 0.0, cb]])
# Transform incident direction
[ui_t, vi_t, wi_t] = rotation2.dot(rotation1.dot(incident_direction))
sti_t = sqrt(ui_t * ui_t + vi_t * vi_t) * sign(wi_t)
cti_t = sqrt(1.0 - sti_t * sti_t)
phi_t = arctan2(vi_t + 0., ui_t + 0.)
cpi_t = cos(phi_t)
spi_t = sin(phi_t)
# Phase at the three vertices
v1, v2, v3 = self.faces[i_face][0], self.faces[i_face][1], self.faces[i_face][2]
alpha1 = k * ((self.vertices[v1][0]) * (uo + ui) +
(self.vertices[v1][1]) * (vo + vi) +
(self.vertices[v1][2]) * (wo + wi))
alpha2 = k * ((self.vertices[v2][0] - self.vertices[v1][0]) * (uo + ui) +
(self.vertices[v2][1] - self.vertices[v1][1]) * (vo + vi) +
(self.vertices[v2][2] - self.vertices[v1][2]) * (wo + wi)) + alpha1
alpha3 = k * ((self.vertices[v3][0] - self.vertices[v1][0]) * (uo + ui) +
(self.vertices[v3][1] - self.vertices[v1][1]) * (vo + vi) +
(self.vertices[v3][2] - self.vertices[v1][2]) * (wo + wi)) + alpha1
exp1 = exp(1j * alpha1)
exp2 = exp(1j * alpha2)
exp3 = exp(1j * alpha3)
# Incident field in local Cartesian
Ei_V2 = rotation2.dot(rotation1.dot(Ei_V))
Ei_H2 = rotation2.dot(rotation1.dot(Ei_H))
# Incident field in local Spherical
Et_v = Ei_V2[0] * cti_t * cpi_t + Ei_V2[1] * cti_t * spi_t - Ei_V2[2] * sti_t
Ep_v = -Ei_V2[0] * spi_t + Ei_V2[1] * cpi_t
Et_h = Ei_H2[0] * cti_t * cpi_t + Ei_H2[1] * cti_t * spi_t - Ei_H2[2] * sti_t
Ep_h = -Ei_H2[0] * spi_t + Ei_H2[1] * cpi_t
# Reflection coefficients
#Rs = 0.001
Rs = 0.0
gamma_perpendicular = -1.0 / (2.0 * Rs * cti_t + 1.0)
gamma_parallel = 0.0
if (2.0 * Rs + cti_t) != 0.0:
gamma_parallel = -cti_t / (2.0 * Rs + cti_t)
# Surface currents in local Cartesian
Jx_v = -Et_v * cpi_t * gamma_parallel + Ep_v * spi_t * cti_t * gamma_perpendicular
Jy_v = -Et_v * spi_t * gamma_parallel - Ep_v * cpi_t * cti_t * gamma_perpendicular
Jx_h = -Et_h * cpi_t * gamma_parallel + Ep_h * spi_t * cti_t * gamma_perpendicular
Jy_h = -Et_h * spi_t * gamma_parallel - Ep_h * cpi_t * cti_t * gamma_perpendicular
# Now loop over all the frequencies
for i_freq in range(len(self.frequency)):
# Area integral
Ic = surface_integral(alpha1[i_freq], alpha2[i_freq], alpha3[i_freq], exp1[i_freq],
exp2[i_freq], exp3[i_freq], area)
# Scattered field components in local coordinates
Es2_v = [Jx_v * Ic, Jy_v * Ic, 0.0]
Es2_h = [Jx_h * Ic, Jy_h * Ic, 0.0]
# Transform back to global coordinates
Es_v = rotation1.T.dot(rotation2.T.dot(Es2_v))
Es_h = rotation1.T.dot(rotation2.T.dot(Es2_h))
Ev_v = uuo * Es_v[0] + vvo * Es_v[1] + wwo * Es_v[2]
Eh_v = -spo * Es_v[0] + cpo * Es_v[1]
Ev_h = uuo * Es_h[0] + vvo * Es_h[1] + wwo * Es_h[2]
Eh_h = -spo * Es_h[0] + cpo * Es_h[1]
# Set the scattering matrix
scattering_matrix[0][i_freq] += Ev_v
scattering_matrix[1][i_freq] += Ev_h
scattering_matrix[2][i_freq] += Eh_v
scattering_matrix[3][i_freq] += Eh_h
return scattering_matrix * sqrt(4.0 * pi) / wavelength
def solve_from_eig(noise_evalsinv,
noise_evects,
dirty_map,
return_noise_diag=False,
feedback=0):
"""Converts a dirty map to a clean map using the eigen decomposition of the
noise inverse.
"""
# Check the shapes.
if noise_evects.ndim != 4:
raise ValueError("Expected 4D array for 'noise_evects`.")
if noise_evalsinv.shape != (noise_evects.shape[-1], ):
raise ValueError("Wrong number of eigenvalues.")
if dirty_map.shape != noise_evects.shape[:-1]:
raise ValueError("Dirty map and noise don't have matching dimensions.")
if dirty_map.size != noise_evects.shape[-1]:
raise ValueError("Eigen space not the same total size as map space.")
n = noise_evalsinv.shape[0]
nf = dirty_map.shape[0]
nr = dirty_map.shape[1]
nd = dirty_map.shape[2]
# Copy the eigenvalues.
noise_evalsinv = noise_evalsinv.copy()
# Find poorly constrained modes and zero them out.
bad_inds = noise_evalsinv < 1. / constants.T_huge**2
n_bad = sp.sum(bad_inds)
if feedback > 1:
print("Discarding %d modes of %d. %f percent." %
(n_bad, n, 100. * n_bad / n))
noise_evalsinv[bad_inds] = 1.
# Rotate the dirty map into the diagonal noise space.
if feedback > 1:
print "Rotating map to eigenspace."
map_rot = sp.zeros(n, dtype=sp.float64)
for ii in xrange(nf):
for jj in xrange(nr):
for kk in xrange(nd):
tmp = noise_evects[ii, jj, kk, :].copy()
map_rot += dirty_map[ii, jj, kk] * tmp
# Multiply by the (diagonal) noise (inverse, inverse). Zero out any poorly
# constrained modes.
map_rot[bad_inds] = 0
# Take inverse and multiply.
map_rot = map_rot / noise_evalsinv
# Now rotate back to the origional space.
if feedback > 1:
print "Rotating back to map space."
clean_map = algebra.zeros_like(dirty_map)
for ii in xrange(nf):
for jj in xrange(nr):
for kk in xrange(nd):
tmp = noise_evects[ii, jj, kk, :].copy()
clean_map[ii, jj, kk] = sp.sum(map_rot * tmp)
if return_noise_diag:
if feedback > 1:
print "Getting noise diagonal."
noise_diag = algebra.zeros_like(dirty_map)
noise_evals = 1. / noise_evalsinv
noise_evals[bad_inds] = constants.T_huge**2
for ii in xrange(nf):
for jj in xrange(nr):
for kk in xrange(nd):
tmp = noise_evects[ii, jj, kk, :].copy()
noise_diag[ii, jj, kk] = sp.sum(tmp**2 * noise_evals)
return clean_map, noise_diag
else:
return clean_map
ref + '.aparc.a2009s.32k_fs.reduce3.very_smooth.left.dfs'))
count1 = 0
rho_rho = []
rho_all = []
s1 = sp.load('labs_all_split2_data_60clusters.npz')
l = s1['lab_sub1']
l1 = sp.reshape(l, (l.shape[0] * l.shape[1]), order='F')
#s2=sp.load('labs_all_split2_data_60clusters.npz');
l = s1['lab_sub2']
l2 = sp.reshape(l, (l.shape[0] * l.shape[1]), order='F')
l12 = sp.concatenate((l1[:, None], l2[:, None]), axis=1)
print sp.sum(sp.absolute(l12[:, 1] - l12[:, 0]))
l12 = reorder_labels(l12)
print sp.sum(sp.absolute(l12[:, 1] - l12[:, 0]))
l1 = sp.reshape(l12[:, 0], (l.shape[0], l.shape[1]), order='F')
l2 = sp.reshape(l12[:, 1], (l.shape[0], l.shape[1]), order='F')
perm1 = sp.mod(17 * sp.arange(max(l1.flatten()) + 1), max(l1.flatten()) + 1)
# Plot labels
for ind in range(4): #range(l.shape[1]):
lab1 = l1[:, ind]
lab2 = l2[:, ind]
view_patch(dfs_left_sm,
perm1[lab1],
hidx[0]] == project.split(':')[1]))[0]
nan_frac = sp.mean(sp.isnan(psi[curr_idx, :]).astype('float'), axis=0)
kidx = sp.where(nan_frac < 0.3)[0]
keep_mat[p, kidx] = True
kidx = keep_mat.max(axis=0)
else:
#kidx = sp.where(sp.sum(sp.isnan(psi), axis=0) < (0.3 * psi.shape[0]))[0]
kidx = sp.where(sp.mean(sp.isnan(psi).astype('float'), axis=0) < 0.3)[0]
psi = psi[:, kidx]
gidx = gidx[kidx]
cidx = cidx[kidx]
event_idx = event_idx[kidx]
if not keep_mat is None:
keep_mat = keep_mat[:, kidx]
print('%i events remain after filtering' % sp.sum(kidx), file=sys.stderr)
print('affecting %i genes' % sp.unique(gidx).shape[0], file=sys.stderr)
print('...done', file=sys.stderr)
### get dev from mean per subtype
if per_subtype:
### generate PSI distribution plots
p_num = projects.shape[0]
panels = sp.ceil(sp.sqrt(p_num)).astype('int')
gs = gridspec.GridSpec(panels, panels)
fig = plt.figure(figsize=(4 * panels, 4 * panels), dpi=200)
for p, project in enumerate(projects):
print('computing mean for subtype %s (%i/%i)' %
(project, p + 1, projects.shape[0]))
### operate on a part of the matrix
def execute(self, nprocesses=1):
"""Worker funciton."""
params = self.params
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params,
params['output_root'] + 'params.ini',
prefix=prefix)
save_noise_diag = params['save_noise_diag']
in_root = params['input_root']
all_out_fname_list = []
all_in_fname_list = []
# Figure out what the band names are.
bands = params['bands']
if not bands:
map_files = glob.glob(in_root + 'dirty_map_' + pol_str + "_*.npy")
bands = []
root_len = len(in_root + 'dirty_map_')
for file_name in map_files:
bands.append(file_name[root_len:-4])
# Loop over files to process.
for pol_str in params['polarizations']:
for band in bands:
if band == -1:
band_str = ''
else:
band_str = "_" + repr(band)
dmap_fname = (in_root + 'dirty_map_' + pol_str + band_str +
'.npy')
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(dmap_fname))
# Load the dirty map and the noise matrix.
dirty_map = algebra.load(dmap_fname)
dirty_map = algebra.make_vect(dirty_map)
if dirty_map.axes != ('freq', 'ra', 'dec'):
msg = ("Expeced dirty map to have axes ('freq',"
"'ra', 'dec'), but it has axes: " +
str(dirty_map.axes))
raise ce.DataError(msg)
shape = dirty_map.shape
# Initialize the clean map.
clean_map = algebra.info_array(sp.zeros(dirty_map.shape))
clean_map.info = dict(dirty_map.info)
clean_map = algebra.make_vect(clean_map)
# If needed, initialize a map for the noise diagonal.
if save_noise_diag:
noise_diag = algebra.zeros_like(clean_map)
if params["from_eig"]:
# Solving from eigen decomposition of the noise instead of
# the noise itself.
# Load in the decomposition.
evects_fname = (in_root + 'noise_evects_' + pol_str +
+band_str + '.npy')
if self.feedback > 1:
print "Using dirty map: " + dmap_fname
print "Using eigenvectors: " + evects_fname
evects = algebra.open_memmap(evects_fname, 'r')
evects = algebra.make_mat(evects)
evals_inv_fname = (in_root + 'noise_evalsinv_' + pol_str +
"_" + repr(band) + '.npy')
evals_inv = algebra.load(evals_inv_fname)
evals_inv = algebra.make_mat(evals_inv)
# Solve for the map.
if params["save_noise_diag"]:
clean_map, noise_diag = solve_from_eig(
evals_inv, evects, dirty_map, True, self.feedback)
else:
clean_map = solve_from_eig(evals_inv, evects,
dirty_map, False,
self.feedback)
# Delete the eigen vectors to recover memory.
del evects
else:
# Solving from the noise.
noise_fname = (in_root + 'noise_inv_' + pol_str +
band_str + '.npy')
if self.feedback > 1:
print "Using dirty map: " + dmap_fname
print "Using noise inverse: " + noise_fname
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_fname))
noise_inv = algebra.open_memmap(noise_fname, 'r')
noise_inv = algebra.make_mat(noise_inv)
# Two cases for the noise. If its the same shape as the map
# then the noise is diagonal. Otherwise, it should be
# block diagonal in frequency.
if noise_inv.ndim == 3:
if noise_inv.axes != ('freq', 'ra', 'dec'):
msg = ("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec'), but it has: " +
str(noise_inv.axes))
raise ce.DataError(msg)
# Noise inverse can fit in memory, so copy it.
noise_inv_memory = sp.array(noise_inv, copy=True)
# Find the non-singular (covered) pixels.
max_information = noise_inv_memory.max()
good_data = noise_inv_memory < 1.0e-10 * max_information
# Make the clean map.
clean_map[good_data] = (dirty_map[good_data] /
noise_inv_memory[good_data])
if save_noise_diag:
noise_diag[good_data] = \
1/noise_inv_memory[good_data]
elif noise_inv.ndim == 5:
if noise_inv.axes != ('freq', 'ra', 'dec', 'ra',
'dec'):
msg = ("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec', 'ra', 'dec'), "
"but it has: " + str(noise_inv.axes))
raise ce.DataError(msg)
# Arrange the dirty map as a vector.
dirty_map_vect = sp.array(dirty_map) # A view.
dirty_map_vect.shape = (shape[0], shape[1] * shape[2])
frequencies = dirty_map.get_axis('freq') / 1.0e6
# Allowcate memory only once.
noise_inv_freq = sp.empty(
(shape[1], shape[2], shape[1], shape[2]),
dtype=float)
if self.feedback > 1:
print "Inverting noise matrix."
# Block diagonal in frequency so loop over frequencies.
for ii in xrange(dirty_map.shape[0]):
if self.feedback > 1:
print "Frequency: ", "%5.1f" % (
frequencies[ii]),
if self.feedback > 2:
print ", start mmap read:",
sys.stdout.flush()
noise_inv_freq[...] = noise_inv[ii, ...]
if self.feedback > 2:
print "done, start eig:",
sys.stdout.flush()
noise_inv_freq.shape = (shape[1] * shape[2],
shape[1] * shape[2])
# Solve the map making equation by diagonalization.
noise_inv_diag, Rot = sp.linalg.eigh(
noise_inv_freq, overwrite_a=True)
if self.feedback > 2:
print "done",
map_rotated = sp.dot(Rot.T, dirty_map_vect[ii])
# Zero out infinite noise modes.
bad_modes = (noise_inv_diag <
1.0e-5 * noise_inv_diag.max())
if self.feedback > 1:
print ", discarded: ",
print "%4.1f" % (100.0 * sp.sum(bad_modes) /
bad_modes.size),
print "% of modes",
if self.feedback > 2:
print ", start rotations:",
sys.stdout.flush()
map_rotated[bad_modes] = 0.
noise_inv_diag[bad_modes] = 1.0
# Solve for the clean map and rotate back.
map_rotated /= noise_inv_diag
map = sp.dot(Rot, map_rotated)
if self.feedback > 2:
print "done",
sys.stdout.flush()
# Fill the clean array.
map.shape = (shape[1], shape[2])
clean_map[ii, ...] = map
if save_noise_diag:
# Using C = R Lambda R^T
# where Lambda = diag(1/noise_inv_diag).
temp_noise_diag = 1 / noise_inv_diag
temp_noise_diag[bad_modes] = 0
# Multiply R by the diagonal eigenvalue matrix.
# Broadcasting does equivalent of mult by diag
# matrix.
temp_mat = Rot * temp_noise_diag
# Multiply by R^T, but only calculate the
# diagonal elements.
for jj in range(shape[1] * shape[2]):
temp_noise_diag[jj] = sp.dot(
temp_mat[jj, :], Rot[jj, :])
temp_noise_diag.shape = (shape[1], shape[2])
noise_diag[ii, ...] = temp_noise_diag
# Return workspace memory to origional shape.
noise_inv_freq.shape = (shape[1], shape[2],
shape[1], shape[2])
if self.feedback > 1:
print ""
sys.stdout.flush()
elif noise_inv.ndim == 6:
if save_noise_diag:
# OLD WAY.
#clean_map, noise_diag, chol = solve(noise_inv,
# dirty_map, True, feedback=self.feedback)
# NEW WAY.
clean_map, noise_diag, noise_inv_diag, chol = \
solve(noise_fname, noise_inv, dirty_map,
True, feedback=self.feedback)
else:
# OLD WAY.
#clean_map, chol = solve(noise_inv, dirty_map,
# False, feedback=self.feedback)
# NEW WAY.
clean_map, noise_inv_diag, chol = \
solve(noise_fname, noise_inv, dirty_map,
False, feedback=self.feedback)
if params['save_cholesky']:
chol_fname = (params['output_root'] + 'chol_' +
pol_str + band_str + '.npy')
sp.save(chol_fname, chol)
if params['save_noise_inv_diag']:
noise_inv_diag_fname = (params['output_root'] +
'noise_inv_diag_' +
pol_str + band_str +
'.npy')
algebra.save(noise_inv_diag_fname, noise_inv_diag)
# Delete the cholesky to recover memory.
del chol
else:
raise ce.DataError("Noise matrix has bad shape.")
# In all cases delete the noise object to recover memeory.
del noise_inv
# Write the clean map to file.
out_fname = (params['output_root'] + 'clean_map_' + pol_str +
band_str + '.npy')
if self.feedback > 1:
print "Writing clean map to: " + out_fname
algebra.save(out_fname, clean_map)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(out_fname))
if save_noise_diag:
noise_diag_fname = (params['output_root'] + 'noise_diag_' +
pol_str + band_str + '.npy')
algebra.save(noise_diag_fname, noise_diag)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_diag_fname))
# Check the clean map for faileur.
if not sp.alltrue(sp.isfinite(clean_map)):
n_bad = sp.sum(sp.logical_not(sp.isfinite(clean_map)))
msg = ("Non finite entries found in clean map. Solve"
" failed. %d out of %d entries bad" %
(n_bad, clean_map.size))
raise RuntimeError(msg)
alg.set_boundary_conditions(bctype='Dirichlet', bcvalue=0.56, pores=BC2_pores)
alg.set_source_term(source_name='pore.blah1',pores=sp.r_[500:700],tol=1e-9)
alg.set_source_term(source_name='pore.blah2',pores=sp.r_[800:1300],tol=1e-11)
alg.set_source_term(source_name='pore.blah3',pores=sp.r_[800:1300],tol=1e-10)
alg.set_source_term(source_name='pore.blah1',pores=sp.r_[300:400])
alg.setup()
alg.solve(iterative_solver='cg',tol=1e-20)
alg.return_results()
print('--------------------------------------------------------------')
print(('steps: ',alg._steps))
print(('tol_reached: ',alg._tol_reached))
print('--------------------------------------------------------------')
print(('reaction from the physics for pores [500:700]:',\
sp.sum(0.5e-13*air['pore.mole_fraction'][sp.r_[500:700]]**1.5+2.5e-14)))
print(('rate from the physics for pores [500:700]:',\
alg.rate(sp.r_[500:700])[0]))
print('--------------------------------------------------------------')
print(('reaction from the physics for pores [800:900]:',\
sp.sum(0.16e-14*sp.log(4*air['pore.mole_fraction'][sp.r_[800:1300]]**(-1.4)+0.133)/sp.log(10)-5.1e-14)\
+sp.sum(0.8e-11*sp.exp(0.5*air['pore.mole_fraction'][sp.r_[800:1300]]**2-0.34)+2e-14)))
print(('rate from the physics for pores [800:900]:',\
alg.rate(sp.r_[800:1300])[0]))
print('--------------------------------------------------------------')
print(('reaction from the physics for pores [300:400]:',\
sp.sum(0.5e-13*air['pore.mole_fraction'][sp.r_[300:400]]**1.5+2.5e-14)))
print(('rate from the physics for pores [300:400]:',\
alg.rate(sp.r_[300:400])[0]))
print('--------------------------------------------------------------')
hidx = sp.where((ctypes == ct) & istumor)[0]
### no confident events in current chunk
if hidx.shape[0] < 2:
continue
tmp_psi = tmp_psi_[hidx, :]
tmp_iso1 = tmp_iso1_[hidx, :]
tmp_iso2 = tmp_iso2_[hidx, :]
for mr in mr_t:
n_idx = sp.c_[tmp_iso1.max(axis=0),
tmp_iso2.max(axis=0)].min(axis=1) < mr
tmp_psi[:, n_idx] = sp.nan
idx_nn = ~sp.isnan(tmp_psi)
d_psi = sp.nanmax(tmp_psi, axis=0) - sp.nanmin(tmp_psi, axis=0)
d_psi[sp.isnan(d_psi)] = 0
for dp in d_psi_t:
if not (mr, ct, dp) in k_idx:
k_idx[(mr, ct,
dp)] = tmp_idx[(sp.sum(idx_nn, axis=0) >= nan_t)
& (d_psi >= dp)]
else:
k_idx[(mr, ct, dp)] = sp.r_[k_idx[(
mr, ct, dp)], tmp_idx[(sp.sum(idx_nn, axis=0) >= nan_t)
& (d_psi >= dp)]]
cPickle.dump(
(d_psi_t, k_idx),
open(re.sub(r'.hdf5$', '', infname) + '.psi_filt_per_ct.pickle', 'w'), -1)
IN.close()
# get time_in profile
tpin = gp.get_timein_profile().copy()
tpin.update(gp.covar.get_timein_profile())
# get counter profile
cp = gp.get_counter_profile().copy()
cp.update(gp.covar.get_counter_profile())
print '\nrotation of G'
print gp0.time['cache_Xrchanged']
print tp['Wr'], '(', tpin['Wr'], ')'
print tp['Wr'] / tpin['Wr']
print '\nThe whole thing'
print sp.sum(tp.values()), '(', sp.sum(tpin.values()), ')'
import ipdb
ipdb.set_trace()
if 0:
gp.LML()
import pylab as pl
pl.ion()
pl.figure(1, figsize=(20, 10))
#gp.covar._profile(show=True)
gp._profile(show=True, rot=90)
pl.figure(2, figsize=(20, 10))
gp.covar._profile(show=True, rot=90)
# ipdb.set_trace()
lasso.set_params(alpha=alpha_cv)
ifold = 0 #what is ifold
for train_index, test_index in kf.split(X):
print(('running fold %d' % ifold))
X_train, X_test, y_train, y_test = X[train_index], X[test_index], y[
train_index], y[test_index]
K_train = K[train_index][:, train_index]
K_test = K[test_index][:, train_index]
model_fit = lasso.fit(X_train, y_train,
K=K_train) #Fit the model to the training set
ytrain_star = model_fit.predict(X_train, K_train)
ytest_star = model_fit.predict(X_test, K_test)
MSE_train_final[ifold] = mean_squared_error(ytrain_star, y_train)
MSE_test_final[ifold] = mean_squared_error(ytest_star, y_test)
W_nonzero_final[ifold] = sp.sum(model_fit.coef_ != 0)
slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(
y_test, ytest_star)
rsquared_final[ifold] = r_value**2
kendall_final[ifold] = sp.stats.kendalltau(y_test, ytest_star)[0]
ifold += 1
for train_index, test_index in kf12.split(X):
X_train, X_test, y_train, y_test, plantings_train, plantings_test = X[
train_index], X[test_index], y[train_index], y[test_index], plantings[
train_index], plantings[
test_index] #Split into training and testing data sets
K_train = K[train_index][:, train_index]
K_test = K[test_index][:, train_index]
lasso.set_params(alpha=alpha_cv)
