def _peak_angles_kmax(kmax, horn_spacing, nu, position):
"""
Return the spherical coordinates (theta, phi) of the beam peaks,
in radians up to a maximum diffraction order.
Parameters
----------
kmax : int, optional
The diffraction order above which the peaks are ignored.
For instance, a value of kmax=2 will model the synthetic beam by
(2 * kmax + 1)**2 = 25 peaks and a value of kmax=0 will only
sample the central peak.
horn_spacing : float
The spacing between horns, in meters.
nu : float
The frequency at which the interference peaks are computed.
position : array of shape (..., 3)
The focal plane positions for which the angles of the
interference peaks are computed.
"""
lmbda = c / nu
position = -position / np.sqrt(
np.sum(position**2, axis=-1))[..., None]
kx, ky = np.mgrid[-kmax:kmax+1, -kmax:kmax+1]
nx = position[..., 0, None] + lmbda * (
ky.ravel() - kx.ravel()) / horn_spacing / np.sqrt(2)
ny = position[..., 1, None] - lmbda * (
kx.ravel() + ky.ravel()) / horn_spacing / np.sqrt(2)
local_dict = {'nx': nx, 'ny': ny}
theta = ne.evaluate('arcsin(sqrt(nx**2 + ny**2))',
local_dict=local_dict)
phi = ne.evaluate('arctan2(ny, nx)', local_dict=local_dict)
return theta, phi
def assert_missing_op(self, op, expr, local_dict):
msg = "expected NotImplementedError regarding '%s'" % op
try:
evaluate(expr, local_dict)
except NotImplementedError, nie:
if "'%s'" % op not in nie.args[0]:
self.fail(msg)
def _find_particles_near_halo(self, ii, ipos, ismooth, mHI):
"""Find the particles near a halo, paying attention to periodic box conditions"""
#Find particles near each halo
sub_pos=self.sub_cofm[ii]
grid_radius = self.sub_radii[ii]
#Need a local for numexpr
box = self.box
#Gather all nearby cells, paying attention to periodic box conditions
for dim in np.arange(3):
jpos = sub_pos[dim]
jjpos = ipos[:,dim]
indj = np.where(ne.evaluate("(abs(jjpos-jpos) < grid_radius+ismooth) | (abs(jjpos-jpos+box) < grid_radius+ismooth) | (abs(jjpos-jpos-box) < grid_radius+ismooth)"))
if np.size(indj) == 0:
return (np.array([]), np.array([]), np.array([]))
ipos = ipos[indj]
# Update smooth and rho arrays as well:
ismooth = ismooth[indj]
mHI = mHI[indj]
jjpos = ipos[:,dim]
# BC 1:
ind_bc1 = np.where(ne.evaluate("(abs(jjpos-jpos+box) < grid_radius+ismooth)"))
ipos[ind_bc1,dim] = ipos[ind_bc1,dim] + box
# BC 2:
ind_bc2 = np.where(ne.evaluate("(abs(jjpos-jpos-box) < grid_radius+ismooth)"))
ipos[ind_bc2,dim] = ipos[ind_bc2,dim] - box
#if np.size(ind_bc1)>0 or np.size(ind_bc2)>0:
# print "Fixed some periodic cells!"
return (ipos, ismooth, mHI)
def boo_product(qlm1, qlm2):
if qlm1.ndim==2 and qlm2.ndim==2:
prod = np.empty([len(qlm1), len(qlm2)])
code ="""
#pragma omp parallel for
for(int p=0; p<Nqlm1[0]; ++p)
for(int q=0; q<Nqlm2[0]; ++q)
{
prod(p,q) = real(qlm1(p,0)*conj(qlm2(q,0)));
for(int m=1; m<Nqlm1[1]; ++m)
prod(p,q) += 2.0*real(qlm1(p,m)*conj(qlm2(q,m)));
prod(p,q) *= 4.0*M_PI/(2.0*(Nqlm1[1]-1)+1);
}
"""
weave.inline(
code,['qlm1', 'qlm2', 'prod'],
type_converters =converters.blitz,
extra_compile_args =['-O3 -fopenmp'],
extra_link_args=['-lgomp'],
verbose=2, compiler='gcc')
return prod
else:
n = np.atleast_2d(numexpr.evaluate(
"""real(complex(real(a), -imag(a)) * b)""",
{'a':qlm1, 'b':qlm2}
))
p = numexpr.evaluate(
"""4*pi/(2*l+1)*(2*na + nb)""",
{
'na': n[:,1:].sum(-1),
'nb': n[:,0],
'l': n.shape[1]-1,
'pi': np.pi
})
return p
def test_in_place(self):
x = arange(10000.).reshape(1000,10)
evaluate("x + 3", out=x)
assert_equal(x, arange(10000.).reshape(1000,10) + 3)
y = arange(10)
evaluate("(x - 3) * y + (x - 3)", out=x)
assert_equal(x, arange(10000.).reshape(1000,10) * (arange(10) + 1))
def remove_nan(arr, val=0., ncore=None):
"""
Replace NaN values in array with a given value.
Parameters
----------
arr : ndarray
Input array.
val : float, optional
Values to be replaced with NaN values in array.
ncore : int, optional
Number of cores that will be assigned to jobs.
Returns
-------
ndarray
Corrected array.
"""
arr = dtype.as_float32(arr)
val = np.float32(val)
with mproc.set_numexpr_threads(ncore):
ne.evaluate('where(arr!=arr, val, arr)', out=arr)
return arr
def circ_mask(arr, axis, ratio=1, val=0., ncore=None):
"""
Apply circular mask to a 3D array.
Parameters
----------
arr : ndarray
Arbitrary 3D array.
axis : int
Axis along which mask will be performed.
ratio : int, optional
Ratio of the mask's diameter in pixels to
the smallest edge size along given axis.
val : int, optional
Value for the masked region.
Returns
-------
ndarray
Masked array.
"""
arr = dtype.as_float32(arr)
val = np.float32(val)
_arr = arr.swapaxes(0, axis)
dx, dy, dz = _arr.shape
mask = _get_mask(dy, dz, ratio)
with mproc.set_numexpr_threads(ncore):
ne.evaluate('where(mask, _arr, val)', out=_arr)
return _arr.swapaxes(0, axis)
def xfun(k, a, b, c, d):
"""Xi-Psi function.
Parameters
----------
k : (n, 1) array
a : float or (m, ) array
b : float or (m, ) array
c : float or (m, ) array
d : float or (m, ) array
Returns
-------
(n, m) array
"""
# out0 = (np.cos(k * (d-a)) * np.exp(d) - np.cos(k * (c-a)) * np.exp(c)
# + k * (np.sin(k * (d-a)) * np.exp(d) - np.sin(k * (c-a)) * np.exp(c)))\
# / (1 + k**2)
# out1 = (np.sin(k[1:] * (d-a)) - np.sin(k[1:] * (c-a))) / k[1:]
out0 = ne.evaluate(("(cos(k * (d-a)) * exp(d) - cos(k * (c-a)) * exp(c)"
"+ k * (sin(k * (d-a)) * exp(d) - sin(k * (c-a)) * exp(c)))"
"/ (1 + k**2)"))
k1 = k[1:]
out1 = ne.evaluate("(sin(k1 * (d-a)) - sin(k1 * (c-a))) / k1")
out1 = np.vstack([(d - c) * np.ones_like(a), out1])
return out0 - out1
def test_empty_string2(self):
a = np.array([b"p", b"pepe"])
b = np.array([b"pepe2", b""])
res = evaluate("(a == b'') & (b == b'pepe2')")
assert_array_equal(res, np.array([False, False]))
res2 = evaluate("(a == b'pepe') & (b == b'')")
assert_array_equal(res, np.array([False, False]))
def flo_refractive_index(wlngth, degC, psu):
"""
Helper function for flo_zhang_scatter_coeffs
@param wlngth backscatter measurement wavlength (nm)
@param degC in situ water temperature (deg_C)
@param psu in site practical salinity (psu)
@retval nsw absolute refractive index of seawater
@retval dnds partial derivative of seawater refractive index with regards to
seawater.
"""
# refractive index of air is from Ciddor (1996, Applied Optics).
n_air = ne.evaluate('1.0 + (5792105.0 / (238.0185 - 1 / (wlngth/1e3)**2)'
'+ 167917.0 / (57.362 - 1 / (wlngth/1e3)**2)) / 1e8')
# refractive index of seawater is from Quan and Fry (1994, Applied Optics)
n0 = 1.31405
n1 = 1.779e-4
n2 = -1.05e-6
n3 = 1.6e-8
n4 = -2.02e-6
n5 = 15.868
n6 = 0.01155
n7 = -0.00423
n8 = -4382.0
n9 = 1.1455e6
nsw = ne.evaluate('n0 + (n1 + n2 * degC + n3 * degC**2) * psu + n4 * degC**2'
'+ (n5 + n6 * psu + n7 * degC) / wlngth + n8 / wlngth**2'
'+ n9 / wlngth**3')
# pure seawater
nsw = ne.evaluate('nsw * n_air')
dnds = ne.evaluate('(n1 + n2 * degC + n3 * degC**2 + n6 / wlngth) * n_air')
return nsw, dnds
def flo_density_seawater(degC, psu):
"""
Helper function for flo_zhang_scatter_coeffs
@param degC in situ water temperature
@param psu in site practical salinity
@retval rho_sw density of seawater
"""
# density of water and seawater,unit is Kg/m^3, from UNESCO,38,1981
a0 = 8.24493e-1
a1 = -4.0899e-3
a2 = 7.6438e-5
a3 = -8.2467e-7
a4 = 5.3875e-9
a5 = -5.72466e-3
a6 = 1.0227e-4
a7 = -1.6546e-6
a8 = 4.8314e-4
b0 = 999.842594
b1 = 6.793952e-2
b2 = -9.09529e-3
b3 = 1.001685e-4
b4 = -1.120083e-6
b5 = 6.536332e-9
# density for pure water
rho_w = ne.evaluate('b0 + b1 * degC + b2 * degC**2 + b3 * degC**3'
'+ b4 * degC**4 + b5 * degC**5')
# density for pure seawater
rho_sw = ne.evaluate('rho_w + ((a0 + a1 * degC + a2 * degC**2'
'+ a3 * degC**3 + a4 * degC**4) * psu'
'+ (a5 + a6 * degC + a7 * degC**2) * psu**1.5 + a8 * psu**2)')
return rho_sw
def _nbr(swir2, nir, output_scaling=1.0, **kwargs):
""" Return the Normalized Burn Ratio (NBR)
NBR is calculated as:
.. math::
NBR = \\frac{(NIR - SWIR2)}{(NIR + SWIR2)}
where:
- :math:`SWIR2` is the shortwave infrared band
- :math:`NIR` is the near infrared band
Args:
swir2 (np.ndarray): SWIR2 band
nir (np.ndarray): NIR band
output_scaling (float): scaling factor for output NDVI (default: 1.0)
Returns:
np.ndarray: NBR
"""
expr = '(nir - swir2) / (nir + swir2)'
if output_scaling == 1.0:
return ne.evaluate(expr)
else:
return ne.evaluate(expr) * output_scaling
def evaluate_expr(expr, xs, ys, zs, arr=None, alpha=None):
"""Evaluate symbolic expression on a numerical grid.
Args:
expr (symbolic): sympy or symengine expression
xs (np.ndarray): 1D-array of x values
ys (np.ndarray): 1D-array of y values
zs (np.ndarray): 1D-array of z values
arr (np.ndarray): additional 1D-array to multiply expression by
alpha (float): multiply expression by gaussian with exponent alpha
Note:
See :meth:`exatomic.algorithms.orbital_util.numerical_grid_from_field_params`
for grid construction details.
"""
subs = {_x: 'xs', _y: 'ys', _z: 'zs'}
# Multiply with an additional array (angular term)
if arr is not None:
return evaluate('arr * ({})'.format(str(expr.subs(subs))))
# Multiply by an exponential decay factor
if alpha is not None:
expr = str((expr * exp(-alpha * _r ** 2)).subs(subs))
return evaluate(expr)
# Just evaluate the expression numerically
return evaluate(str(expr.subs(subs)))
def _ndvi(red, nir, output_scaling=1.0, **kwargs):
""" Return the Normalized Difference Vegetation Index (NDVI)
NDVI is calculated as:
.. math::
NDVI = \\frac{(NIR - RED)}{(NIR + RED)}
where:
- :math:`RED` is the red band
- :math:`NIR` is the near infrared band
Args:
red (np.ndarray): red band
nir (np.ndarray): NIR band
output_scaling (float): scaling factor for output NDVI (default: 1.0)
Returns:
np.ndarray: NDVI
"""
expr = '(nir - red) / (nir + red)'
if output_scaling == 1.0:
return ne.evaluate(expr)
else:
return ne.evaluate(expr) * output_scaling
def _ndmi(swir1, nir, output_scaling=1.0, **kwargs):
""" Return the Normalized Difference Moisture Index (NDMI)
NDMI is calculated as:
.. math::
NDMI = \\frac{(NIR - SWIR1)}{(NIR + SWIR1)}
where:
- :math:`SWIR1` is the shortwave infrared band
- :math:`NIR` is the near infrared band
Args:
swir1 (np.ndarray): SWIR1 band
nir (np.ndarray): NIR band
output_scaling (float): scaling factor for output NDVI (default: 1.0)
Returns:
np.ndarray: NDMI
"""
expr = '(nir - swir1) / (nir + swir1)'
if output_scaling == 1.0:
return ne.evaluate(expr)
else:
return ne.evaluate(expr) * output_scaling
def sphere_rotate(ra, dec, rapol, decpol, ra0):
""" rotate ra,dec to a new spherical coordinate system where the pole is
at rapol,decpol and the zeropoint is at ra=ra0
revert flag allows to reverse the transformation
"""
x,y,z=torect(ra,dec)
tmppol=cv_coord(rapol,decpol,1,degr=True,fr='sph',to='rect') #pole axis
tmpvec1=cv_coord(ra0,0,1,degr=True,fr='sph',to='rect') #x axis
tmpvec1=numpy.array(tmpvec1)
tmpvec1[2]=(-tmppol[0]*tmpvec1[0]-tmppol[1]*tmpvec1[1])/tmppol[2]
tmpvec1/=numpy.sqrt((tmpvec1**2).sum())
tmpvec2=numpy.cross(tmppol,tmpvec1) # y axis
Axx,Axy,Axz=tmpvec1
Ayx,Ayy,Ayz=tmpvec2
Azx,Azy,Azz=tmppol
xnew = numexpr.evaluate('x*Axx+y*Axy+z*Axz')
ynew = numexpr.evaluate('x*Ayx+y*Ayy+z*Ayz')
znew = numexpr.evaluate('x*Azx+y*Azy+z*Azz')
del x,y,z
tmp = fromrect(xnew,ynew,znew)
return (tmp[0],tmp[1])
def get_smooth_length(bar):
"""Figures out if the particles are from AREPO or GADGET
and computes the smoothing length.
Note the Volume array in HDF5 is comoving and this returns a comoving smoothing length
If we are Arepo, this smoothing length is cell radius, where
cell volume = 4/3 π (cell radius) **3 and cell volume = mass / density
Arguments:
Baryon particles from a simulation
Returns:
Array of smoothing lengths in code units.
"""
# Are we arepo? If we are a modern version we should have this array.
if "Volume" in bar.keys():
volume = np.array(bar["Volume"], dtype=np.float32)
radius = ne.evaluate("(scal*volume)**poww")
elif "Number of faces of cell" in bar.keys():
rho = np.array(bar["Density"])
mass = np.array(bar["Masses"])
radius = ne.evaluate("(scal*mass/rho)**poww")
else:
# If we are gadget, the SmoothingLength array is actually the smoothing length.
# Note the definition used in Gadget differs from that used here: in gadget volume = h^3
# due to the normalisation of the SPH kernel.
radius = np.array(bar["SmoothingLength"]) * scal ** poww
return radius
def calculate_neuron_outputs(self, data):
# assemble hidden layer output with all kinds of neurons
assert len(self.neurons) > 0, "Model must have hidden neurons"
hidden_layer_output = [] # H
for neuron_function, _, weight_matrix, bias_vector in self.neurons:
# The input to the neuron function is x * weights + bias, data.dot(weight_matrix) does the matrix mult
activation_function_input = data.dot(weight_matrix) + bias_vector
activation_function_output = np.zeros(activation_function_input.shape)
# calculate the activation function output
if neuron_function == "lin":
activation_function_output = activation_function_input # We already computed this with the dot + bias
# If it is a supported Numexpr function, use that library, which takes advantage of multiple cores and
# vectorization to greatly speed this part of the process
elif neuron_function == "sigm":
ne.evaluate("1/(1+exp(-activation_function_input))", out=activation_function_output)
elif neuron_function == "tanh":
ne.evaluate('tanh(activation_function_input)', out=activation_function_output)
else:
print("INFO: You are using", neuron_function, "instead of a supported function.")
print(" if speed is a concern, consider implementing it here as a Numexpr function")
activation_function_output = neuron_function(activation_function_input) # custom <numpy.ufunc>
hidden_layer_output.append(activation_function_output)
# Combine all the column results from the neurons
hidden_layer_output = np.hstack(hidden_layer_output)
return hidden_layer_output
def distance_matrix_numexpr(size):
r = np.random.rand(size, 2)
r_i = r[:, np.newaxis]
r_j = r[np.newaxis, :]
d_ij = ne.evaluate("sum((r_i - r_j)**2, 2)")
d_ij = ne.evaluate("sqrt(d_ij)")
def solve(self, u, v, dx, dy):
import numexpr as ne
nx, ny = u.shape
assert u.shape == tuple(self.shape)
fu = fft2(u)
fv = fft2(v)
mpx = self.mpx
mmx = self.mmx
dpx = self.dpx
dmx = self.dmx
mpy = self.mpy
mmy = self.mmy
dpy = self.dpy
dmy = self.dmy
d = ne.evaluate("fu*mmy * dmx + fv * mmx * dmy")
lapl = ne.evaluate("mpy * mmy * dpx * dmx + mpx*mmx *dpy *dmy")
lapl[0, 0] = 1.0
p = d / lapl
px = np.real(ifft2(mpy * dpx * p))
py = np.real(ifft2(mpx * dpy * p))
# self.p = np.real(ifft2(p))
u -= px
v -= py
return px, py
def advance(u, u_1, u_2, f_a, Cx2, Cy2, dt2, V=None, step1=False):
if step1:
#ne.set_vml_num_threads(2)
dt = sqrt(dt2)
Cx2 = 0.5*Cx2; Cy2 = 0.5*Cy2; dt2 = 0.5*dt2
D1 = 1; D2 = 0
else:
D1 = 2; D2 = 1
u_xx = ne.evaluate("u_10 - 2*u_11 + u_12", local_dict={"u_10":u_1[:-2,1:-1],
"u_11":u_1[1:-1,1:-1],
"u_12":u_1[2:,1:-1]})
u_yy = ne.evaluate("u_10 - 2*u_11 + u_12", local_dict={"u_10":u_1[1:-1,:-2],
"u_11":u_1[1:-1,1:-1],
"u_12":u_1[1:-1,2:]})
u[1:-1,1:-1] = ne.evaluate("D1*u_11 - D2*u_21 + Cx2*u_xx + Cy2*u_yy + dt2*f_a",
local_dict={"u_11":u_1[1:-1,1:-1], "u_21":u_2[1:-1,1:-1],
"D1":D1, "D2":D2,"u_xx": u_xx, "u_yy":u_yy,
"Cx2":Cx2, "Cy2":Cy2, "dt2":dt2, "f_a":f_a[1:-1,1:-1]})
if step1:
u[1:-1,1:-1] += ne.evaluate("dt*V", local_dict={"V":V[1:-1,1:-1], "dt":dt})
# Boundary condition u=0
j = 0
u[:,j] = 0
j = u.shape[1]-1
u[:,j] = 0
i = 0
u[i,:] = 0
i = u.shape[0]-1
u[i,:] = 0
return u
def covariance_wind(f, c0, rho, distance, L, Cv, steps=10, initial=0.001):
"""Covariance. Wind fluctuations only.
:param f: Frequency
:param c0: Speed of sound
:param rho: Spatia separation
:param distance: Distance
:param L: Correlation length
:param Cv: Variance of wind speed
:param initial: Initial value
"""
f = np.asarray(f)
k = 2.*np.pi*f / c0
K0 = 2.*np.pi / L
A = 5.0/(18.0*np.pi*gamma(1./3.)) # Equation 11, see text below. Approximate result is 0.033
gamma_v = 3./10.*np.pi**2.*A*k**2.*K0**(-5./3.)*4.*(Cv/c0)**2. # Equation 28, only wind fluctuations
krho = k * rho
t = krho[:, None] * np.linspace(0.00000000001, 1., steps) # Fine discretization for integration
#t[t==0.0] = 1.e-20
gamma56 = gamma(5./6.)
bessel56 = besselk(5./6., t)
bessel16 = besselk(1./6., t)
integration = cumtrapz(ne.evaluate("2.0**(1./6.)*t**(5./6.)/gamma56 * (bessel56 - t/2.0 * bessel16 )"), initial=initial)[:,-1]
B = ne.evaluate("2.0*gamma_v * distance / krho * integration")
#B = 2.0*gamma_v * distance / krho * cumtrapz((2.0**(1./6.)*t**(5./6.)/gamma(5./6.) * (besselk(5./6., t) - t/2.0*besselk(1./6., t)) ), initial=initial)[:,-1]
return B
def grad(fld, bnd=True):
"""2nd order centeral diff, 1st order @ boundaries if bnd"""
# vx, vy, vz = fld.component_views()
if bnd:
fld = viscid.extend_boundaries(fld, order=0, crd_order=0)
if fld.iscentered("Cell"):
crdx, crdy, crdz = fld.get_crds_cc(shaped=True)
# divcenter = "Cell"
# divcrds = coordinate.NonuniformCartesianCrds(fld.crds.get_clist(np.s_[1:-1]))
# divcrds = fld.crds.slice_keep(np.s_[1:-1, 1:-1, 1:-1])
elif fld.iscentered("Node"):
crdx, crdy, crdz = fld.get_crds_nc(shaped=True)
# divcenter = "Node"
# divcrds = coordinate.NonuniformCartesianCrds(fld.crds.get_clist(np.s_[1:-1]))
# divcrds = fld.crds.slice_keep(np.s_[1:-1, 1:-1, 1:-1])
else:
raise NotImplementedError("Can only do cell and node centered gradients")
v = fld.data
g = viscid.zeros(fld['x=1:-1, y=1:-1, z=1:-1'].crds, nr_comps=3)
xp, xm = crdx[2:, :, :], crdx[:-2, : , : ] # pylint: disable=bad-whitespace
yp, ym = crdy[ :, 2:, :], crdy[: , :-2, : ] # pylint: disable=bad-whitespace
zp, zm = crdz[ :, :, 2:], crdz[: , : , :-2] # pylint: disable=bad-whitespace
vxp, vxm = v[2: , 1:-1, 1:-1], v[ :-2, 1:-1, 1:-1] # pylint: disable=bad-whitespace
vyp, vym = v[1:-1, 2: , 1:-1], v[1:-1, :-2, 1:-1] # pylint: disable=bad-whitespace
vzp, vzm = v[1:-1, 1:-1, 2: ], v[1:-1, 1:-1, :-2] # pylint: disable=bad-whitespace
g['x'].data[...] = ne.evaluate("(vxp-vxm)/(xp-xm)")
g['y'].data[...] = ne.evaluate("(vyp-vym)/(yp-ym)")
g['z'].data[...] = ne.evaluate("(vzp-vzm)/(zp-zm)")
return g
def rotate_clip(data_np, theta_deg, rotctr_x=None, rotctr_y=None,
out=None):
"""
Rotate numpy array `data_np` by `theta_deg` around rotation center
(rotctr_x, rotctr_y). If the rotation center is omitted it defaults
to the center of the array.
No adjustment is done to the data array beforehand, so the result will
be clipped according to the size of the array (the output array will be
the same size as the input array).
"""
# If there is no rotation, then we are done
if math.fmod(theta_deg, 360.0) == 0.0:
return data_np
ht, wd = data_np.shape[:2]
if rotctr_x == None:
rotctr_x = wd // 2
if rotctr_y == None:
rotctr_y = ht // 2
yi, xi = numpy.mgrid[0:ht, 0:wd]
xi -= rotctr_x
yi -= rotctr_y
cos_t = numpy.cos(numpy.radians(theta_deg))
sin_t = numpy.sin(numpy.radians(theta_deg))
#t1 = time.time()
if have_numexpr:
ap = ne.evaluate("(xi * cos_t) - (yi * sin_t) + rotctr_x")
bp = ne.evaluate("(xi * sin_t) + (yi * cos_t) + rotctr_y")
else:
ap = (xi * cos_t) - (yi * sin_t) + rotctr_x
bp = (xi * sin_t) + (yi * cos_t) + rotctr_y
#print "rotation in %.5f sec" % (time.time() - t1)
#ap = numpy.rint(ap).astype('int').clip(0, wd-1)
#bp = numpy.rint(bp).astype('int').clip(0, ht-1)
# Optomizations to reuse existing intermediate arrays
numpy.rint(ap, out=ap)
ap = ap.astype('int')
ap.clip(0, wd-1, out=ap)
numpy.rint(bp, out=bp)
bp = bp.astype('int')
bp.clip(0, ht-1, out=bp)
if out != None:
out[:, :, ...] = data_np[bp, ap]
newdata = out
else:
newdata = data_np[bp, ap]
new_ht, new_wd = newdata.shape[:2]
assert (wd == new_wd) and (ht == new_ht), \
Exception("rotated cutout is %dx%d original=%dx%d" % (
new_wd, new_ht, wd, ht))
return newdata
def get_all_peak_over_threshold(sig, thresh, front, peak_span = 3, use_numexpr = False):
"""
Simple solution for detectng peak aboe (or below) a threshold
"""
peak_span = max(peak_span, 3)
k = (peak_span-1)//2
sig_center = sig[k:-k]
if use_numexpr:
if front == '+':
pos_spike, = np.where(numexpr.evaluate( '(sig1<sig2) & (sig2>thresh) & (sig2>=sig3)'))
elif front == '-':
pos_spike, = np.where(numexpr.evaluate( '(sig1>sig2) & (sig2<thresh) & (sig2<=sig3)'))
else :
if front == '+':
peaks = sig_center>thresh
for i in range(k):
peaks &= sig_center>sig[i:i+sig_center.size]
peaks &= sig_center>=sig[peak_span-i-1:peak_span-i-1+sig_center.size]
elif front == '-':
peaks = sig_center<thresh
for i in range(k):
peaks &= sig_center<sig[i:i+sig_center.size]
peaks &= sig_center<=sig[peak_span-i-1:peak_span-i-1+sig_center.size]
ind_peak, = np.where(peaks)
ind_peak = ind_peak + k
return ind_peak
def sideband(t, plus, minus, freq=0, phase=0, offset=False, offset_fit_lin=0,offset_fit_quad=0, origin=0):
if freq==0 and phase==0:
return (plus - minus), np.zeros(t.shape)
elif offset:
if (not max(plus) == 0):
time_step = t[1]-t[0]
freq_calibrated = getFreq(plus,freq,offset_fit_lin,offset_fit_quad);
freq_integ_array = np.cumsum(freq_calibrated)*time_step
# np.savetxt('time.out', t, delimiter=',')
return ( np.cos(2 * np.pi * (freq_integ_array/1.0e9) + phase*np.pi/180.0) * plus - np.cos(2 * np.pi * (freq_integ_array/1.0e9) + phase*np.pi/180.0) * minus,
-np.sin(2 * np.pi * (freq_integ_array/1.0e9)+ phase*np.pi/180.0) * plus - np.sin(2 * np.pi * (freq_integ_array/1.0e9) + phase*np.pi/180.0) * minus)
else:
# For ML0218 Mixer
# return ( np.cos(2 * np.pi * (freq/1.0e9 * t)+ phase*np.pi/180.0) * plus - np.cos(2 * np.pi * (freq/1.0e9 * t) + phase*np.pi/180.0) * minus,
# +np.sin(2 * np.pi * (freq/1.0e9 * t)+ phase*np.pi/180.0) * plus +np.sin(2 * np.pi * (freq/1.0e9 * t) + phase*np.pi/180.0) * minus)
#
# For IQ0317 Mixer
# return ( np.cos(2 * np.pi * (freq/1.0e9 * t)+ phase*np.pi/180.0) * plus - np.cos(2 * np.pi * (freq/1.0e9 * t) + phase*np.pi/180.0) * minus,
# -np.sin(2 * np.pi * (freq/1.0e9 * t)+ phase*np.pi/180.0) * plus - np.sin(2 * np.pi * (freq/1.0e9 * t) + phase*np.pi/180.0) * minus)
wts = ne.evaluate('2 * (freq/1.0e9 * (t-origin))+ phase/180.0') * np.pi
cosdata = ne.evaluate('cos(wts)')
sindata = ne.evaluate('sin(wts)')
# return ne.evaluate('cosdata * (plus - minus)'), ne.evaluate('- sindata * (plus + minus)')
return ne.evaluate('cosdata * (plus - minus)'), ne.evaluate('- sindata * (plus + minus)')
def apply_linear_filterbank(b, a, x, zi):
X = x
output = empty_like(X)
for sample in xrange(X.shape[0]):
x = X[sample]
for curf in xrange(zi.shape[2]):
#y = b[:, 0, curf]*x+zi[:, 0, curf]
y = numexpr.evaluate('b*x+zi', local_dict={
'b':b[:, 0, curf],
'x':x,
'zi':zi[:, 0, curf]})
for i in xrange(b.shape[1]-2):
#zi[:, i, curf] = b[:, i+1, curf]*x+zi[:, i+1, curf]-a[:, i+1, curf]*y
zi[:, i, curf] = numexpr.evaluate('b*x+zi-a*y', local_dict={
'b':b[:, i+1, curf],
'x':x,
'zi':zi[:, i+1, curf],
'a':a[:, i+1, curf],
'y':y})
i = b.shape[1]-2
#zi[:, i, curf] = b[:, i+1, curf]*x-a[:, i+1, curf]*y
zi[:, i, curf] = numexpr.evaluate('b*x-a*y', local_dict={
'b':b[:, i+1, curf],
'x':x,
'a':a[:, i+1, curf],
'y':y})
x = y
output[sample] = y
return output
def set_rho(self, rho):
"""
Set the initial density matrix
:param rho: 2D numpy array or sting containing the density matrix
:return: self
"""
if isinstance(rho, str):
# density matrix is supplied as a string
ne.evaluate("%s + 0j" % rho, local_dict=vars(self), out=self.rho)
elif isinstance(rho, np.ndarray):
# density matrix is supplied as an array
# perform the consistency checks
assert rho.shape == self.rho.shape,\
"The grid size does not match with the density matrix"
# make sure the density matrix is stored as a complex array
np.copyto(self.rho, rho.astype(np.complex))
else:
raise ValueError("density matrix must be either string or numpy.array")
# normalize
self.rho /= self.rho.trace() * self.dX
return self
def correlate(self):
'''Estimates the time-delay in whole samples, then aligns and finds the complex visibility for two signals in the data key of self.parent_row and self.child_row.
Generates self.convolution_spectrum, self.zscore, self.expected_overlap, and self.real_overlap, which can be used for plotting purposes.
Creates two signals, one corresponding to the cos and the other to the sin signal, as self.cos_signal and self.sin_signal
In order to do a correlation without opening a file and populating self.child_row and self.parent_row, assign these variables manually.'''
child_fft = scipy.fftpack.fft(self.child_row['data'])
parent_fft = scipy.fftpack.fft(self.parent_row['data'])
child_inverse_conjugate = -child_fft.conjugate()
fft_auto_child = child_fft*child_inverse_conjugate
fft_convolution = parent_fft*child_inverse_conjugate
self.convolution_spectrum = numpy.abs(scipy.fftpack.ifft(fft_convolution/fft_auto_child)) #Roth window saved in self.convolution_spectrum
assert self.time_inacc < sec/2, "This system clock cannot *possibly* be that inaccurate!"
expected_tdiff = int((self.child_row['utcendtime']-self.parent_row['utcendtime'])*hz) #our initial guess for the location of the tdiff peak
sample_inacc = int(self.time_inacc*hz) #around the guessed place of tdiff.
if expected_tdiff-sample_inacc < 0: expected_tdiff = sample_inacc
elif expected_tdiff+sample_inacc > sample_size: expected_tdiff = sample_size-sample_inacc
cropped_convolution_spectrum = self.convolution_spectrum[expected_tdiff-sample_inacc:expected_tdiff+sample_inacc]
#later measurements of tdiff will have a 0 point at expected_tdiff-sample_inacc and a total length of around 2*sample_inacc (may wrap around)
tdiff = numpy.argmax(cropped_convolution_spectrum)
tdiff += expected_tdiff-sample_inacc #offset for the real convolution_spectrum
self.zscore = (self.convolution_spectrum[tdiff]-numpy.average(self.convolution_spectrum))/numpy.std(self.convolution_spectrum)
#timespan = math.fabs(child_row['utcendtime'] - parent_row['utcendtime']) + sec
self.expected_overlap = (float(sec) - 1.0*math.fabs(self.child_row['utcendtime']-self.parent_row['utcendtime']))*hz
self.real_overlap = (float(sec) - 1.0*math.fabs(float(tdiff)/hz))*hz
int_delay = int(numpy.copysign(numpy.floor(numpy.fabs(tdiff)), tdiff))
self.abs_delay = int(abs(int_delay)) #always positive :)
parent_signal, child_signal = self.parent_row['data'][self.abs_delay:], self.child_row['data'][0:len(self.child_row['data'])-self.abs_delay]
h_child_length = len(child_signal)
h_child_new_length = int(2**numpy.ceil(numpy.log2(h_child_length)))
h_child_diff_length = h_child_new_length - h_child_length
h_child_signal = scipy.fftpack.hilbert(numpy.append(child_signal, numpy.zeros(h_child_diff_length)))[0:h_child_length]
self.cos_signal, self.sin_signal = evaluate("parent_signal*child_signal"), evaluate("h_child_signal*parent_signal")
def get_following_peak(ind_spike, sig, sign):
"""
Give the first following peak after a point on one signal.
Params:
* ind_spike: index where are detected spike (corssing threshold)
* sig: a numpy array.
* sign: signa of peak '-' or '+'
"""
sig1 = sig[:-2]
sig2 = sig[1:-1]
sig3 = sig[2:]
if sign == '+':
all_peaks, = np.where(numexpr.evaluate( '(sig1<=sig2) & ( sig2>sig3)'))
elif sign == '-':
all_peaks, = np.where(numexpr.evaluate( '(sig1>=sig2) & ( sig2<sig3)'))
ind_peaks = -np.ones(ind_spike.size, dtype = 'i')
for i, ind in enumerate(ind_spike):
possible = all_peaks[all_peaks>ind]
if possible.size>0:
ind_peaks[i] = possible[0]
return ind_peaks
def computePotEvapPM(dataset, lterms=True, lmeans=False):
''' function to compute potential evapotranspiration (according to Penman-Monteith method:
https://en.wikipedia.org/wiki/Penman%E2%80%93Monteith_equation,
http://www.fao.org/docrep/x0490e/x0490e06.htm#formulation%20of%20the%20penman%20monteith%20equation)
'''
# get net radiation at surface
if 'netrad' in dataset: Rn = dataset['netrad'][:] # net radiation
if 'Rn' in dataset: Rn = dataset['Rn'][:] # alias
else: Rn = computeNetRadiation(dataset, asVar=False) # try to compute
# heat flux in and out of the ground
if 'grdflx' in dataset:
G = dataset['grdflx'][:] # heat release by the soil
else:
raise VariableError, "Cannot determine soil heat flux for PET calculation."
# get wind speed
if 'U2' in dataset: u2 = dataset['U2'][:]
elif lmeans and 'U10' in dataset: u2 = wind(dataset['U10'][:], z=10)
elif 'u10' in dataset and 'v10' in dataset:
u2 = wind(u=dataset['u10'][:], v=dataset['v10'][:], z=10)
else:
raise VariableError, "Cannot determine 2m wind speed for PET calculation."
# get psychrometric variables
if 'ps' in dataset: p = dataset['ps'][:]
else:
raise VariableError, "Cannot determine surface air pressure for PET calculation."
g = gamma(p) # psychrometric constant (pressure-dependent)
if 'Q2' in dataset: ea = dataset['Q2'][:]
elif 'q2' in dataset:
ea = dataset['q2'][:] * dataset['ps'][:] * 28.96 / 18.02
else:
raise VariableError, "Cannot determine 2m water vapor pressure for PET calculation."
# get temperature
if lmeans and 'Tmean' in dataset: T = dataset['Tmean'][:]
elif 'T2' in dataset: T = dataset['T2'][:]
else:
raise VariableError, "Cannot determine 2m mean temperature for PET calculation."
# get saturation water vapor
if 'Tmin' in dataset and 'Tmax' in dataset:
es = e_sat(dataset['Tmin'][:], dataset['Tmax'][:])
# else: Es = e_sat(T) # backup, but not very accurate
else:
raise VariableError, "'Tmin' and 'Tmax' are required to compute saturation water vapor pressure for PET calculation."
D = Delta(T) # slope of saturation vapor pressure w.r.t. temperature
# compute potential evapotranspiration according to Penman-Monteith method
# (http://www.fao.org/docrep/x0490e/x0490e06.htm#fao%20penman%20monteith%20equation)
if lterms:
Dgu = evaluate(
'( D + g * (1 + 0.34 * u2) ) * 86400') # common denominator
rad = evaluate('0.0352512 * D * (Rn + G) / Dgu') # radiation term
wnd = evaluate(
'g * u2 * (es - ea) * 0.9 / T / Dgu') # wind term (vapor deficit)
pet = evaluate(
'( 0.0352512 * D * (Rn + G) + ( g * u2 * (es - ea) * 0.9 / T ) ) / ( D + g * (1 + 0.34 * u2) ) / 86400'
)
import numpy as np
assert np.allclose(pet, rad + wnd, equal_nan=True)
rad = Variable(data=rad,
name='petrad',
units='kg/m^2/s',
axes=dataset['ps'].axes)
wnd = Variable(data=wnd,
name='petwnd',
units='kg/m^2/s',
axes=dataset['ps'].axes)
else:
pet = evaluate(
'( 0.0352512 * D * (Rn + G) + ( g * u2 * (es - ea) * 0.9 / T ) ) / ( D + g * (1 + 0.34 * u2) ) / 86400'
)
# N.B.: units have been converted to SI (mm/day -> 1/86400 kg/m^2/s, kPa -> 1000 Pa, and Celsius to K)
pet = Variable(data=pet,
name='pet',
units='kg/m^2/s',
axes=dataset['ps'].axes)
assert 'waterflx' not in dataset or pet.units == dataset[
'waterflx'].units, pet
# return new variable(s)
return (pet, rad, wnd) if lterms else pet
def subsolvIP(self, alfa, beta, low, upp, p0, q0, P, Q, b):
"""
This function subsolv solves the MMA subproblem with interior
point method:
minimize SUM[ p0j/(uppj-xj) + q0j/(xj-lowj) ] + a0*z +
+ SUM[ ci*yi + 0.5*di*(yi)^2 ],
subject to SUM[ pij/(uppj-xj) + qij/(xj-lowj) ] - ai*z - yi <= bi,
alfaj <= xj <= betaj, yi >= 0, z >= 0.
Input: m, n, low, upp, alfa, beta, p0, q0, P, Q, a0, a, b, c, d.
Output: xmma,ymma,zmma, slack variables and Lagrange multiplers.
"""
# Initialize the variable values
epsi = 1
x = 0.5 * (alfa + beta)
y = np.ones([self.m])
z = 1
lam = np.ones([self.m])
xsi = 1.0 / (x - alfa)
xsi = np.maximum(xsi, 1.0)
eta = np.maximum(1.0 / (beta - x), 1.0)
mu = np.maximum(np.ones([self.m]), 0.5 * self.c)
zet = 1
s = np.ones([self.m])
epsiIt = 1
design_state = DesignState(x=x,
y=y,
z=z,
lam=lam,
xsi=xsi,
eta=eta,
mu=mu,
zet=zet,
s=s)
if self.IP > 0:
print(str("*" * 80))
while epsi > self.epsimin: # Loop over epsilon
self.iPrint(["Interior Point it.", "epsilon"], [epsiIt, epsi], 0)
# compute residual
residuNorm, residuMax = self.resKKT(
alfa,
beta,
low,
upp,
p0,
q0,
P,
Q,
b,
design_state,
epsi,
)
# Solve the NL KKT problem for a given epsilon
it_NL = 1
relaxloopEpsi = []
while residuNorm > 0.9 * epsi and it_NL < 200:
self.iPrint(
["NL it.", "Norm(res)", "Max(|res|)"],
[it_NL, residuNorm, residuMax],
1,
)
# precompute useful data -> time consuming!!!
(
delx,
dely,
delz,
dellam,
diagx,
diagy,
diagxinv,
diaglamyi,
GG,
) = self.preCompute(
alfa,
beta,
low,
upp,
p0,
q0,
P,
Q,
b,
design_state,
epsi,
)
# assemble and solve the system: dlam or dx
diagxinvGG = diagxinv * GG
AA = self.JacDual(diagxinvGG, diaglamyi, GG, z, zet)
bb = self.RHSdual(dellam, delx, dely, delz, diagxinvGG, diagy,
GG)
solut = np.linalg.solve(AA, bb)
dlam = solut[0:self.m]
dz = solut[self.m]
dx = -delx * diagxinv - np.dot((diagxinv * GG).T, dlam)
dy = -dely / diagy + dlam / diagy
dxsi = ne.evaluate(
"-xsi + epsi / (x - alfa) - (xsi * dx) / (x - alfa)")
deta = ne.evaluate(
"-eta + epsi / (beta - x) + (eta * dx) / (beta - x)")
dmu = -mu + epsi / y - (mu * dy) / y
dzet = -zet + epsi / z - zet * dz / z
ds = -s + epsi / lam - (s * dlam) / lam
# store variables
design_state_old = copy.copy(design_state)
# relaxation of the newton step for staying in feasible region
len_xx = self.local_n * 2 + self.m * 4 + 2
xx = np.zeros(len_xx)
np.concatenate((y, [z], lam, xsi, eta, mu, [zet], s), out=xx)
dxx = np.zeros(len_xx)
np.concatenate((dy, [dz], dlam, dxsi, deta, dmu, [dzet], ds),
out=dxx)
stepxx = ne.evaluate("-1.01 * dxx / xx")
local_stmxx = np.max(stepxx)
stmxx = self.comm.allreduce(local_stmxx, op=MPI.MAX)
stepalfa = ne.evaluate("-1.01 * dx / (x - alfa)")
local_stmalfa = np.max(stepalfa)
stmalfa = self.comm.allreduce(local_stmalfa, op=MPI.MAX)
stepbeta = ne.evaluate("1.01 * dx / (beta - x)")
local_stmbeta = np.max(stepbeta)
stmbeta = self.comm.allreduce(local_stmbeta, op=MPI.MAX)
stmalbe = np.maximum(stmalfa, stmbeta)
stmalbexx = np.maximum(stmalbe, stmxx)
stminv = np.maximum(stmalbexx, 1.0)
step = 1.0 / np.maximum(stmalbexx, 1.0)
itto = 1
resinewNorm = 2 * residuNorm
resinewMax = 1e10
while resinewNorm > residuNorm and itto < 200:
self.iPrint(
["relax. it.", "Norm(res)", "step"],
[itto, resinewNorm, step],
2,
)
# compute new point
design_state = self.getNewPoint(
design_state_old,
dx,
dy,
dz,
dlam,
dxsi,
deta,
dmu,
dzet,
ds,
step,
)
x = design_state.x
y = design_state.y
z = design_state.z
lam = design_state.lam
xsi = design_state.xsi
eta = design_state.eta
mu = design_state.mu
s = design_state.s
zet = design_state.zet
z = design_state.z
# compute the residual
resinewNorm, resinewMax = self.resKKT(
alfa,
beta,
low,
upp,
p0,
q0,
P,
Q,
b,
design_state,
epsi,
)
# update step
step /= 2.0
itto += 1
if itto > 198:
warning(f"Line search iteration limit {itto} reached")
self.iPrint(["relax. it.", "Norm(res)", "step"],
[itto, resinewNorm, step], 2)
residuNorm = resinewNorm
residuMax = resinewMax
step *= 2.0
it_NL += 1
if it_NL > 500:
warning(
f"Iteration limit of the Newton solver ({it_NL}) reached")
epsi *= 0.1
epsiIt += 1
if self.IP > 0:
print(str("*" * 80))
return x, y, z, lam
def global_res_norm_square(local_residual):
local_residuNorm = ne.evaluate("sum(local_residual ** 2)")
residuNorm = self.comm.allreduce(local_residuNorm, op=MPI.SUM)
return residuNorm
def resKKT(
self,
alfa,
beta,
low,
upp,
p0,
q0,
P,
Q,
b,
design_state,
epsi,
):
x = design_state.x
y = design_state.y
z = design_state.z
lam = design_state.lam
xsi = design_state.xsi
eta = design_state.eta
mu = design_state.mu
s = design_state.s
zet = design_state.zet
z = design_state.z
Mdiag = self.Mdiag
ux1 = upp - x
xl1 = x - low
ux2 = ux1 * ux1
xl2 = xl1 * xl1
uxinv1 = 1.0 / ux1
xlinv1 = 1.0 / xl1
plam = p0 + np.dot(P.T, lam)
qlam = q0 + np.dot(Q.T, lam)
local_gvec = (P * self.Mdiag).dot(uxinv1) + (Q *
self.Mdiag).dot(xlinv1)
gvec = self.comm.allreduce(local_gvec, op=MPI.SUM)
dpsidx = ne.evaluate("plam / ux2 - qlam / xl2")
def global_res_norm_square(local_residual):
local_residuNorm = ne.evaluate("sum(local_residual ** 2)")
residuNorm = self.comm.allreduce(local_residuNorm, op=MPI.SUM)
return residuNorm
def global_residual_max(local_residual):
local_residuMax = np.linalg.norm(local_residual, np.inf)
residuMax = self.comm.allreduce(local_residuMax, op=MPI.MAX)
return residuMax
# rex
local_residu_x = ne.evaluate(
"(dpsidx * Mdiag - Mdiag*xsi + Mdiag*eta)")
residu_x_norm = global_res_norm_square(
ne.evaluate("local_residu_x / sqrt(Mdiag)")
) # This components is in the dual space, the norm has
# to be weighted b the inverse of the mass matrix a.T * M^{-1} * a
# do a sqrt if you're going to do **2 later
residu_x_max = global_residual_max(local_residu_x)
# rey
residu_y = self.c + self.d * y - mu - lam
residu_y_norm = np.sum(residu_y**2)
residu_y_max = np.linalg.norm(residu_y, np.inf)
# rez
residu_z = self.a0 - zet - np.dot(self.a, lam)
residu_z_norm = residu_z**2
residu_z_max = np.abs(residu_z)
# relam
residu_lam = gvec - self.a * z - y + s + b
residu_lam_norm = np.sum(residu_lam**2)
residu_lam_max = np.linalg.norm(residu_lam, np.inf)
# rexsi
local_residu_xsi = ne.evaluate(
"(xsi * (x - alfa) - epsi) * sqrt(Mdiag)")
residu_xsi_norm = global_res_norm_square(local_residu_xsi)
residu_xsi_max = global_residual_max(local_residu_xsi)
# reeta
local_residu_eta = ne.evaluate(
"(eta * (beta - x) - epsi)* sqrt(Mdiag)")
residu_eta_norm = global_res_norm_square(local_residu_eta)
residu_eta_max = global_residual_max(local_residu_eta)
# remu
residu_mu = mu * y - epsi
residu_mu_norm = np.sum(residu_mu**2)
residu_mu_max = np.linalg.norm(residu_mu, np.inf)
# rezet
residu_zet = zet * z - epsi
residu_zet_norm = residu_zet**2
residu_zet_max = np.abs(residu_zet)
# res
residu_s = lam * s - epsi
residu_s_norm = np.sum(residu_s**2)
residu_s_max = np.linalg.norm(residu_s, np.inf)
residu_norm = np.sqrt(residu_x_norm + residu_y_norm + residu_lam_norm +
residu_xsi_norm + residu_eta_norm +
residu_mu_norm + residu_s_norm + residu_z_norm +
residu_zet_norm)
residu_max = np.max((
residu_x_max,
residu_y_max,
residu_lam_max,
residu_xsi_max,
residu_eta_max,
residu_mu_max,
residu_s_max,
residu_z_max,
residu_zet_max,
))
return residu_norm, residu_max
import math
math.log(10)
# math.log? can be used to check documentation
import numpy as np
np.log(10)
#==============================================================================
# paradigm
#==============================================================================
loops = 25000
# 1: standard lib
from math import *
a = range(1, loops)
def f(x):
return 3 * log(x) + cos(x) ** 2
%timeit r = [f(x) for x in a]
# 2: numpy is faster
import numpy as np
a = np.arange(1, loops)
%timeit r = 3 * np.log(a) + np.cos(a) ** 2
# 3: numexpr is fastest for numerical expressions
import numexpr as ne
ne.set_num_threads(4)
f = '3 * log(a) + cos(a) ** 2'
%timeit r = ne.evaluate(f)
def subt(A,B):
A_3D = A[:, np.newaxis]
return ne.evaluate('A_3D - B')
def apply_cut(arr, cut):
return ne.evaluate(cut_to_selection(cut), local_dict=arr)
def get_slope_aspect(input_dem):
"""
Calculate the slope and aspect from the input DEM
:param input_dem: `file` the input DEM
:return: :class:`numpy.ndarray` the output slope values
:return: :class:`numpy.ndarray` the output aspect values
"""
np.seterr(divide='ignore')
if type(input_dem) == str:
ds = gdal.Open(input_dem)
else:
ds = input_dem
cols = ds.RasterXSize
rows = ds.RasterYSize
geotransform = ds.GetGeoTransform()
pixel_x_size = abs(geotransform[1])
pixel_y_size = abs(geotransform[5])
band = ds.GetRasterBand(1)
elevation_array = band.ReadAsArray(0, 0, cols, rows)
elevation_array = elevation_array.astype(float)
nodata_value = band.GetNoDataValue()
if nodata_value is not None:
elevation_array[np.where(elevation_array == nodata_value)] = np.nan
else:
elevation_array[np.where(elevation_array is None)] = np.nan
mask = np.isnan(elevation_array)
x_m_array, y_m_array = get_pixel_size_grids(ds)
dzdx_array = ndimage.sobel(elevation_array, axis=1) / (8. * pixel_x_size)
dzdx_array = numexpr.evaluate("dzdx_array * pixel_x_size / x_m_array")
del x_m_array
dzdy_array = ndimage.sobel(elevation_array, axis=0) / (8. * pixel_y_size)
dzdy_array = numexpr.evaluate("dzdy_array * pixel_y_size / y_m_array")
del y_m_array
# Slope
hypotenuse_array = np.hypot(dzdx_array, dzdy_array)
slope_array = numexpr.evaluate(
"arctan(hypotenuse_array) / RADIANS_PER_DEGREE")
slope_array[mask] = np.nan
del hypotenuse_array
# Aspect
# Convert angles from conventional radians to compass heading 0-360
aspect_array = numexpr.evaluate(
"(450 - arctan2(dzdy_array, -dzdx_array) / RADIANS_PER_DEGREE) % 360")
# Derive reclassifed aspect...
aspect_array_reclassify = reclassify_aspect(aspect_array)
del aspect_array
ds = None
return slope_array, aspect_array_reclassify
def work(n):
a = arange(n)
evaluate('a+a')
def filter(db, query, user_dict):
# these should be translated to a bunch or or/and statements within gemini
# so they are supported, but must be translated before getting here.
if query == "False":
return []
if "any(" in query or "all(" in query or \
("sum(" in query and not query.startswith("sum(") and query.count("sum(") == 1):
return None
user_dict['where'] = np.where
if query.startswith("not "):
# "~" is not to numexpr.
query = "~" + query[4:]
sum_cmp = False
if query.startswith("sum("):
assert query[-1].isdigit()
query, sum_cmp = query[4:].rsplit(")", 1)
query = "(%s) %s" % (query, sum_cmp)
query = query.replace(".", "__")
query = " & ".join("(%s)" % token for token in query.split(" and "))
query = " | ".join("(%s)" % token for token in query.split(" or "))
conn = sqlite3.connect(db)
cur = conn.cursor()
samples = get_samples(cur)
# convert gt_col[index] to gt_col__sample_name
patt = "(%s)\[(\d+)\]" % "|".join((g[0] for g in gt_cols_types))
def subfn(x):
"""Turn gt_types[1] into gt_types__sample"""
field, idx = x.groups()
return "%s__%s" % (field, fix_sample_name(samples[int(idx)]))
query = re.sub(patt, subfn, query)
if os.environ.get('GEMINI_DEBUG') == 'TRUE':
print >> sys.stderr, query[:250] + "..."
carrays = load(db, query=query)
if len(carrays) == 0 or max(len(carrays[c]) for c in carrays) == 0 or \
any(not any(carrays[c]) for c in carrays):
# need this 2nd check above because of the place-holders in load()
raise NoGTIndexException
# loop through and create a cache of "$gt__$sample"
for gt_col in carrays:
if not gt_col in query: continue
for i, sample_array in enumerate(carrays[gt_col]):
sample = fix_sample_name(samples[i])
if not sample in query: continue
user_dict["%s__%s" % (gt_col, sample)] = sample_array
# had to special-case count. it won't quite be as efficient
if "|count|" in query:
tokens = query[2:-2].split("|count|")
icmp = tokens[-1]
# a list of carrays, so not in memory.
res = [bcolz.eval(tok, user_dict=user_dict) for tok in tokens[:-1]]
# in memory after this, but just a single axis array.
res = np.sum(res, axis=0)
res = ne.evaluate('res%s' % icmp)
else:
res = bcolz.eval(query, user_dict=user_dict)
try:
if res.shape[0] == 1 and len(res.shape) > 1:
res = res[0]
except AttributeError:
return []
variant_ids, = np.where(res)
#variant_ids = np.array(list(bcolz.eval(query, user_dict=user_dict,
# vm="numexpr").wheretrue()))
# variant ids are 1-based.
if len(variant_ids) > 0:
return 1 + variant_ids
else:
return []
def test_reductions(self):
# Check that they compile OK.
assert_equal(disassemble(
NumExpr("sum(x**2+2, axis=None)", [('x', double)])),
[(b'mul_ddd', b't3', b'r1[x]', b'r1[x]'),
(b'add_ddd', b't3', b't3', b'c2[2.0]'),
(b'sum_ddn', b'r0', b't3', None)])
assert_equal(disassemble(
NumExpr("sum(x**2+2, axis=1)", [('x', double)])),
[(b'mul_ddd', b't3', b'r1[x]', b'r1[x]'),
(b'add_ddd', b't3', b't3', b'c2[2.0]'),
(b'sum_ddn', b'r0', b't3', 1)])
assert_equal(disassemble(
NumExpr("prod(x**2+2, axis=2)", [('x', double)])),
[(b'mul_ddd', b't3', b'r1[x]', b'r1[x]'),
(b'add_ddd', b't3', b't3', b'c2[2.0]'),
(b'prod_ddn', b'r0', b't3', 2)])
# Check that full reductions work.
x = zeros(100000) + .01 # checks issue #41
assert_allclose(evaluate("sum(x+2,axis=None)"), sum(x + 2, axis=None))
assert_allclose(evaluate("sum(x+2,axis=0)"), sum(x + 2, axis=0))
assert_allclose(evaluate("prod(x,axis=0)"), prod(x, axis=0))
assert_allclose(evaluate("min(x)"), np.min(x))
assert_allclose(evaluate("max(x,axis=0)"), np.max(x, axis=0))
x = arange(10.0)
assert_allclose(evaluate("sum(x**2+2,axis=0)"), sum(x ** 2 + 2, axis=0))
assert_allclose(evaluate("prod(x**2+2,axis=0)"), prod(x ** 2 + 2, axis=0))
assert_allclose(evaluate("min(x**2+2,axis=0)"), np.min(x ** 2 + 2, axis=0))
assert_allclose(evaluate("max(x**2+2,axis=0)"), np.max(x ** 2 + 2, axis=0))
x = arange(100.0)
assert_allclose(evaluate("sum(x**2+2,axis=0)"), sum(x ** 2 + 2, axis=0))
assert_allclose(evaluate("prod(x-1,axis=0)"), prod(x - 1, axis=0))
assert_allclose(evaluate("min(x-1,axis=0)"), np.min(x - 1, axis=0))
assert_allclose(evaluate("max(x-1,axis=0)"), np.max(x - 1, axis=0))
x = linspace(0.1, 1.0, 2000)
assert_allclose(evaluate("sum(x**2+2,axis=0)"), sum(x ** 2 + 2, axis=0))
assert_allclose(evaluate("prod(x-1,axis=0)"), prod(x - 1, axis=0))
assert_allclose(evaluate("min(x-1,axis=0)"), np.min(x - 1, axis=0))
assert_allclose(evaluate("max(x-1,axis=0)"), np.max(x - 1, axis=0))
# Check that reductions along an axis work
y = arange(9.0).reshape(3, 3)
assert_allclose(evaluate("sum(y**2, axis=1)"), sum(y ** 2, axis=1))
assert_allclose(evaluate("sum(y**2, axis=0)"), sum(y ** 2, axis=0))
assert_allclose(evaluate("sum(y**2, axis=None)"), sum(y ** 2, axis=None))
assert_allclose(evaluate("prod(y**2, axis=1)"), prod(y ** 2, axis=1))
assert_allclose(evaluate("prod(y**2, axis=0)"), prod(y ** 2, axis=0))
assert_allclose(evaluate("prod(y**2, axis=None)"), prod(y ** 2, axis=None))
assert_allclose(evaluate("min(y**2, axis=1)"), np.min(y ** 2, axis=1))
assert_allclose(evaluate("min(y**2, axis=0)"), np.min(y ** 2, axis=0))
assert_allclose(evaluate("min(y**2, axis=None)"), np.min(y ** 2, axis=None))
assert_allclose(evaluate("max(y**2, axis=1)"), np.max(y ** 2, axis=1))
assert_allclose(evaluate("max(y**2, axis=0)"), np.max(y ** 2, axis=0))
assert_allclose(evaluate("max(y**2, axis=None)"), np.max(y ** 2, axis=None))
# Check integers
x = arange(10.)
x = x.astype(int)
assert_allclose(evaluate("sum(x**2+2,axis=0)"), sum(x ** 2 + 2, axis=0))
assert_allclose(evaluate("prod(x**2+2,axis=0)"), prod(x ** 2 + 2, axis=0))
assert_allclose(evaluate("min(x**2+2,axis=0)"), np.min(x ** 2 + 2, axis=0))
assert_allclose(evaluate("max(x**2+2,axis=0)"), np.max(x ** 2 + 2, axis=0))
# Check longs
x = x.astype(int)
assert_allclose(evaluate("sum(x**2+2,axis=0)"), sum(x ** 2 + 2, axis=0))
assert_allclose(evaluate("prod(x**2+2,axis=0)"), prod(x ** 2 + 2, axis=0))
assert_allclose(evaluate("min(x**2+2,axis=0)"), np.min(x ** 2 + 2, axis=0))
assert_allclose(evaluate("max(x**2+2,axis=0)"), np.max(x ** 2 + 2, axis=0))
# Check complex
x = x + .1j
assert_allclose(evaluate("sum(x**2+2,axis=0)"), sum(x ** 2 + 2, axis=0))
assert_allclose(evaluate("prod(x-1,axis=0)"), prod(x - 1, axis=0))
def _worker(qout=None):
ra = np.arange(1e3)
rows = evaluate('ra > 0')
#print "Succeeded in evaluation!\n"
if qout is not None:
qout.put("Done")
def test_compare_copy(self):
sarr = self.str_array1
expr = 'sarr'
res1 = eval(expr)
res2 = evaluate(expr)
assert_array_equal(res1, res2)
def run(self):
a = arange(3)
assert_array_equal(evaluate('a**3'), array([0, 1, 8]))
def test_big_int(self):
# Big ints should be promoted to longs.
res = evaluate('2**40')
assert_array_equal(res, 2 ** 40)
self.assertEqual(res.dtype.name, 'int64')
def test_compare_constant(self):
sarr = self.str_array1
expr = 'sarr >= %r' % self.str_constant
res1 = eval(expr)
res2 = evaluate(expr)
assert_array_equal(res1, res2)
def test_small_long(self):
# Small longs should not be downgraded to ints.
res = evaluate('42L')
assert_array_equal(res, 42)
self.assertEqual(res.dtype.name, 'int64')
def test_small_uint32(self):
# Small uint32 should not be downgraded to ints.
a = np.uint32(42)
res = evaluate('a')
assert_array_equal(res, 42)
self.assertEqual(res.dtype.name, 'int64')
def test_all_scalar(self):
a = 3.
b = 4.
assert_allclose(evaluate("a+b"), a + b)
expr = NumExpr("2*a+3*b", [('a', double), ('b', double)])
assert_equal(expr(a, b), 2 * a + 3 * b)
def test_small_int(self):
# Small ints (32-bit ones) should not be promoted to longs.
res = evaluate('2')
assert_array_equal(res, 2)
self.assertEqual(res.dtype.name, 'int32')
def test_right_shift(self):
x = arange(10, dtype='i4')
assert_array_equal(evaluate("x>>2"), x >> 2)
def test_neg(self):
a = array([2 ** 31 - 1, 2 ** 31, 2 ** 32, 2 ** 63 - 1], dtype=int64)
res = evaluate('-a')
assert_array_equal(res, [1 - 2 ** 31, -(2 ** 31), -(2 ** 32), 1 - 2 ** 63])
self.assertEqual(res.dtype.name, 'int64')
def test_true_div(self):
x = arange(10, dtype='i4')
assert_array_equal(evaluate("x/2"), x / 2)
assert_array_equal(evaluate("x/2", truediv=False), x / 2)
assert_array_equal(evaluate("x/2", truediv='auto'), x / 2)
assert_array_equal(evaluate("x/2", truediv=True), x / 2.0)
def test_rational_expr(self):
a = arange(1e6)
b = arange(1e6) * 0.1
x = (a + 2 * b) / (1 + a + 4 * b * b)
y = evaluate("(a + 2*b) / (1 + a + 4*b*b)")
assert_array_almost_equal(x, y)
def test_simple_expr(self):
x = arange(1e6)
y = evaluate("x")
assert_array_equal(x, y)
def test_left_shift(self):
x = arange(10, dtype='i4')
assert_array_equal(evaluate("x<<2"), x << 2)
def test_simple(self):
a = array([1., 2., 3.])
b = array([4., 5., 6.])
c = array([7., 8., 9.])
x = evaluate("2*a + 3*b*c")
assert_array_equal(x, array([86., 124., 168.]))
def test_zero_div(self):
x = arange(100, dtype='i4')
y = evaluate("1/x")
x2 = zeros(100, dtype='i4')
x2[1] = 1
assert_array_equal(x2, y)
def test_str_contains_withemptystr2(self):
withemptystr = array([b'abc', b'def', b''])
res = evaluate('contains(withemptystr, b"")')
assert_equal(res, [True, True, True])
def test_simple_expr_small_array(self):
x = arange(100.0)
y = evaluate("x")
assert_array_equal(x, y)