def test(what='fortran'):
import lut2func as l2f
if what == 'purpy':
l2f.interpolators = l2f.interpolators_pure_python
elif what == 'numba':
l2f.interpolators = l2f.interpolators_numba_python
else:
l2f.interpolators = l2f.interpolators_fortran
# example for R^2-->R^3
luta = np.arange(24, dtype=np.float).reshape(6, 4, order='C')
lutb = np.arange(24, dtype=np.float).reshape(6, 4, order='F') ** 2
lutc = np.sqrt(np.arange(24, dtype=np.float).reshape(6, 4, order='F'))
luts = (luta, lutb, lutc)
xx = np.array([3., 4., 6., 7., 9., 15.])
yy = np.array([1., 5., 10, 15])[::-1]
axes = (xx, yy)
# Variant A: luts is a tuple of luts
funca = l2f.lut2func(luts, axes)
funcav = l2f.lut2func(luts, axes, vectorial=True)
# Variant B: luts is a Nd Array, last dimension is dimension of result
luts = np.array((luta, lutb, lutc), dtype=np.float32).transpose([1, 2, 0])
print(luts.shape)
funcb = l2f.lut2func(luts, axes)
funcbv = l2f.lut2func(luts, axes, vectorial=True)
print(luts.shape, len(axes))
# sys.exit()
nn = 10000
import time
pos = np.array([3.5, 11.])
ppos = np.array([pos for i in range(nn)])
a = time.time()
print('VEC (%i): map-coordinates' % nn)
for ff in (funcav, funcbv):
a = time.time()
erg = ff(ppos)
print(erg[0], ': %.2f us' % (((time.time() - a)) / float(nn) * 1.e6))
print('SEQ (%i): %s, map-coordinates, %s, map-coordinates ' % (nn, what, what))
for ff in (funca, funcav, funcb, funcbv):
a = time.time()
for i in range(nn):
_ = ff(pos)
print(_, ': %.2f us' % ((time.time() - a) / float(nn) * 1.e6))
def jlut2func(dum):
func = l2f.lut2func(dum["lut"], dum["axes"])
def jfunc(woo):
return func(woo).reshape(dum["ny"], dum["nx"])
return jfunc
def jlut2func(dum):
func = l2f.lut2func(dum['lut'], dum['axes'])
def jfunc(woo):
return func(woo).reshape(dum['ny'], dum['nx'])
return jfunc
def jlut2func(dum):
func = l2f.lut2func(dum['lut'], dum['axes'])
def jfunc(woo):
#return func(woo).reshape(dum['ny'],dum['nx'],order='F')
return np.array(func(woo).reshape(dum['ny'], dum['nx']), order='F')
return jfunc
def test3to1():
# example for R^3-->R^1
import lut2func as l2f
# from lut2func import lut2func
lut3d = np.arange(3 * 4 * 5, dtype=np.float).reshape(3, 4, 5)
x0 = np.array([3., 4., 6.])
x1 = np.array([1., 4., 8., 10.])
x2 = np.array([1., 2., 4., 8., 16.])
func3d = l2f.lut2func((lut3d,), (x0, x1, x2))
print(func3d([3.5, 5., 10.]))
print(l2f.interpolators)
def generate_jacobian_lut(luts, axes):
"""
generates a jacobian-lut at exactly the same
positions (axes points) as the original luts
"""
func = l2f.lut2func(luts, axes)
if isinstance(luts, tuple):
ny = len(luts)
elif isinstance(luts, np.ndarray):
ny = luts.shape[-1]
else:
print "Luts is not of right type"
return None
nx = len(axes)
dimi = np.array([len(ax) for ax in (axes)])
dimj = [len(ax) for ax in (axes)]
dimj.append(ny * nx)
out = np.zeros(dimj)
a = np.zeros(len(axes))
b = np.zeros(len(axes))
a = np.array([ax.min() for ax in axes])
b = np.array([ax.max() for ax in axes])
def njac(x):
return numerical_jacoby(a, b, x, func, nx, ny, delta=0.01)
print ("Filling the jacobian ...")
progress = wln.wln(dimi.prod(), "Progress", modulo=10000)
for i in np.arange(dimi.prod()):
idx = np.unravel_index(i, dimi)
wo = np.array([ax[i] for i, ax in zip(idx, axes)])
out[idx] = njac(wo).ravel()
progress.event()
print ("")
print ("Done")
return {"lut": out, "ny": ny, "nx": nx, "axes": axes}
def generate_jacobian_lut(luts, axes):
'''
generates a jacobian-lut at exactly the same
positions (axes points) as the original luts
'''
func = l2f.lut2func(luts, axes)
if isinstance(luts, tuple):
ny = len(luts)
elif isinstance(luts, np.ndarray):
ny = luts.shape[-1]
else:
print 'Luts is not of right type'
return None
nx = len(axes)
dimi = np.array([len(ax) for ax in (axes)])
dimj = [len(ax) for ax in (axes)]
dimj.append(ny * nx)
out = np.zeros(dimj)
a = np.zeros(len(axes))
b = np.zeros(len(axes))
a = np.array([ax.min() for ax in axes])
b = np.array([ax.max() for ax in axes])
def njac(x):
return numerical_jacoby(a, b, x, func, nx, ny, delta=0.01)
print('Filling the jacobian ...')
progress = wln.wln(dimi.prod(), 'Progress', modulo=10000)
for i in np.arange(dimi.prod()):
idx = np.unravel_index(i, dimi)
wo = np.array([ax[i] for i, ax in zip(idx, axes)])
out[idx] = njac(wo).ravel()
progress.event()
print('')
print('Done')
return {'lut': out, 'ny': ny, 'nx': nx, 'axes': axes}
luta = np.arange(240000, dtype=np.float).reshape(6, 40000, order="C")
lutb = np.arange(240000, dtype=np.float).reshape(6, 40000, order="F") ** 2
lutc = np.sqrt(np.arange(240000, dtype=np.float).reshape(6, 40000, order="F"))
# either as Tuple
luts1 = (luta, lutb, lutc)
# or as single np.array
luts2 = np.array((luta, lutb, lutc)).transpose([1, 2, 0])
# second create the axes
xx1 = np.array([3.0, 4.0, 6.0, 7.0, 9.0, 15.0])
xx2 = np.linspace(-10.0, 30.0, 40000)
# as a tuple
axes = (xx1, xx2)
# finaly create the func
func1 = l2f.lut2func(luts1, axes)
func2 = l2f.lut2func(luts2, axes)
# for the jacobians
# use the function luts
jlut = generate_jacobian_lut(luts2, axes)
# save the jacobians
lut2ncdf(jlut, "jlut_test.nc")
# and make a function from it
jfun = ncdf2func("jlut_test.nc")
# now test the jacobian function:
print jfun([3.0, 10.0])
# and compare it with a numerical_jacoby
a = np.array([3, 1])
b = np.array([15, 15])
def __init__(self,corefile=os.path.join(about_me()['cawadir'],'luts','cloud_core_meris.nc4'),used_ab=None):
self.about_me=about_me()
with Dataset(corefile,'r') as ncds:
#get the full lut
self.lut=np.array(ncds.variables['lut'][:],order='F')
self.jlut=np.array(ncds.variables['jlut'][:],order='F')
self.axes=tuple([np.array(ncds.variables[a][:]) for a in ncds.variables['lut'].dimensions[:-1]])
self.jaxes=tuple([np.array(ncds.variables[a][:]) for a in ncds.variables['jlut'].dimensions[:-1]])
self.ny_nx=np.array(ncds.variables['jaco'][:])
self.wb=ncds.getncattr('win_bnd').split(',')
self.ab=ncds.getncattr('abs_bnd').split(',')
self.cha={bb:
{kk:ncds.groups['cha'].groups[bb].getncattr(kk)
for kk in ('bwvl','cwvl')}
for bb in self.wb+self.ab}
if used_ab is None:
self.ab_idx=None
else:
self.ab=[used_ab,]
self.ab_idx=pos_in_list(ncds.getncattr('abs_bnd').split(','),used_ab)[0]
#generic forward
self._forward=lut2func.lut2func(self.lut,self.axes)
#generic jacobian
self._jacobian=lut2jacobian_lut.jlut2func({'lut':self.jlut
,'axes':self.jaxes
,'ny':self.ny_nx[0]
,'nx':self.ny_nx[1]})
#global predefinition of input for speed reasons
self.xaa=np.zeros(2)
self.par=np.zeros(5)
self.mes=np.zeros(len(self.wb)+len(self.ab))
#local predefine inp for speed
inp=np.zeros(7)
def forward(woo,par):
'''
Input:
woo: state (ctp,cot)
par: parameter (alb, sza, vza, azd, dwvl)
aot is aot at winband [0]
wvc is sqrt of wvc
Output:
normalized radiances at winbands and absbands
'''
inp[:2],inp[2:]=woo,par
if self.ab_idx is None:
return self._forward(inp)
else:
return self._forward(inp)[((0,1+self.ab_idx),)]
self.forward =forward
def jforward(woo,geo):
'''
as forward, but returns jacobian
output must be limited to the first three elements (the state and not the geometry)
'''
inp[:2],inp[2:]=woo,geo
if self.ab_idx is None:
return self._jacobian(inp)[:,:2]
else:
return self._jacobian(inp)[(0,1+self.ab_idx),:2]
self.jforward=jforward
#min_state
a=np.array([self.axes[i].min() for i in range(2)])
#max_state
b=np.array([self.axes[i].max() for i in range(2)])
self.inverter=oe.my_inverter(self.forward,a,b,jaco=self.jforward)
#finaly preset SE
sew=[0.0001 for i in self.wb]
sea=[0.0001 for i in self.ab]
self.SE=np.diag(sew+sea)
def __init__(self,
corefile=os.path.join(about_me()['cawadir'], 'luts',
'cloud_core_meris.nc4'),
used_ab=None):
self.about_me = about_me()
with Dataset(corefile, 'r') as ncds:
#get the full lut
self.lut = np.array(ncds.variables['lut'][:], order='F')
self.jlut = np.array(ncds.variables['jlut'][:], order='F')
self.axes = tuple([
np.array(ncds.variables[a][:])
for a in ncds.variables['lut'].dimensions[:-1]
])
self.jaxes = tuple([
np.array(ncds.variables[a][:])
for a in ncds.variables['jlut'].dimensions[:-1]
])
self.ny_nx = np.array(ncds.variables['jaco'][:])
self.wb = ncds.getncattr('win_bnd').split(',')
self.ab = ncds.getncattr('abs_bnd').split(',')
self.cha = {
bb: {
kk: ncds.groups['cha'].groups[bb].getncattr(kk)
for kk in ('bwvl', 'cwvl')
}
for bb in self.wb + self.ab
}
if used_ab is None:
self.ab_idx = None
else:
self.ab = [
used_ab,
]
self.ab_idx = pos_in_list(
ncds.getncattr('abs_bnd').split(','), used_ab)[0]
#generic forward
self._forward = lut2func.lut2func(self.lut, self.axes)
#generic jacobian
self._jacobian = lut2jacobian_lut.jlut2func({
'lut': self.jlut,
'axes': self.jaxes,
'ny': self.ny_nx[0],
'nx': self.ny_nx[1]
})
#global predefinition of input for speed reasons
self.xaa = np.zeros(2)
self.par = np.zeros(5)
self.mes = np.zeros(len(self.wb) + len(self.ab))
#local predefine inp for speed
inp = np.zeros(7)
def forward(woo, par):
'''
Input:
woo: state (ctp,cot)
par: parameter (alb, sza, vza, azd, dwvl)
aot is aot at winband [0]
wvc is sqrt of wvc
Output:
normalized radiances at winbands and absbands
'''
inp[:2], inp[2:] = woo, par
if self.ab_idx is None:
return self._forward(inp)
else:
return self._forward(inp)[((0, 1 + self.ab_idx), )]
self.forward = forward
def jforward(woo, geo):
'''
as forward, but returns jacobian
output must be limited to the first three elements (the state and not the geometry)
'''
inp[:2], inp[2:] = woo, geo
if self.ab_idx is None:
return self._jacobian(inp)[:, :2]
else:
return self._jacobian(inp)[(0, 1 + self.ab_idx), :2]
self.jforward = jforward
#min_state
a = np.array([self.axes[i].min() for i in range(2)])
#max_state
b = np.array([self.axes[i].max() for i in range(2)])
self.inverter = oe.my_inverter(self.forward, a, b, jaco=self.jforward)
#finaly preset SE
sew = [0.0001 for i in self.wb]
sea = [0.0001 for i in self.ab]
self.SE = np.diag(sew + sea)
def __init__(self, ocean_lut=os.path.join('.', 'luts', 'ocean', 'ocean_core_meris.nc4')):
# sys.stderr.write('entering CawaTcwvOceanCore... \n')
libdir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'libs')
sys.path.append(libdir)
with Dataset(ocean_lut, 'r') as ncds:
# get the full lut
self.lut = np.array(ncds.variables['lut'][:], order='F')
self.jlut = np.array(ncds.variables['jlut'][:], order='F')
self.axes = tuple([np.array(ncds.variables[a][:]) for a in ncds.variables['lut'].dimensions[:-1]])
self.jaxes = tuple([np.array(ncds.variables[a][:]) for a in ncds.variables['jlut'].dimensions[:-1]])
self.ny_nx = np.array(ncds.variables['jaco'][:])
self.wb = ncds.getncattr('win_bnd').split(',')
self.ab = ncds.getncattr('abs_bnd').split(',')
# generic forward
self._forward = lut2func.lut2func(self.lut, self.axes)
# generic jacobian
self._jacobian = lut2jacobian_lut.jlut2func({'lut': self.jlut,
'axes': self.jaxes,
'ny': self.ny_nx[0],
'nx': self.ny_nx[1]})
# global predefinition of input for speed reasons
self.xaa = np.zeros(3)
self.par = np.zeros(3)
self.mes = np.zeros(len(self.wb) + len(self.ab))
# local predefine inp for speed
inp = np.zeros(6)
def forward(woo, geo):
"""
Input:
woo: state (wvc aot wsp)
geo: azi vie suz
aot is aot at winband [0]
wvc is sqrt of wvc
Output:
normalized radiances at winbands
-np.log(effective_transmission)/sqrt(amf) at absbands
effective_transmission= L_toa/L_0
with L_0 is normalized radiance without water vapor
"""
inp[:3], inp[3:] = woo, geo
return self._forward(inp)
self.forward = forward
def jforward(woo, geo):
"""
as forward, but returns jacobian
output must be limited to the first three elements (the state and not the geometry)
"""
inp[:3], inp[3:] = woo, geo
return self._jacobian(inp)[:, :3]
self.jforward = jforward
# min_state
a = np.array([self.axes[i].min() for i in range(3)])
# max_state
b = np.array([self.axes[i].max() for i in range(3)])
self.inverter = oe.my_inverter(self.forward, a, b, jaco=self.jforward)
# finaly preset SE
sew = [0.0001 for i in self.wb]
sea = [0.001 for i in self.ab]
self.SE = np.diag(sew + sea)
return out
return function
if __name__ == '__main__':
from lut2func import lut2func
# example for R^2-->R^3
luta = np.arange(24, dtype=np.float).reshape(6, 4, order='C')
lutb = np.arange(24, dtype=np.float).reshape(6, 4, order='F') ** 2
lutc = np.sqrt(np.arange(24, dtype=np.float).reshape(6, 4, order='F'))
luts = (luta, lutb, lutc)
xx = np.array([3., 4., 6., 7., 9., 15.])
yy = np.array([1., 5., 10, 15])[::-1]
axes = (xx, yy)
funca = lut2func(luts, axes)
luts = np.array((luta, lutb, lutc)).transpose([1, 2, 0])
funcb = lut2func(luts, axes)
import time
for ff in (funca, funcb):
a = time.time()
for i in range(1000):
_ = ff(np.array([3.5, 11.]))
print _, ': %i us' % ((time.time() - a) * 1000)
def __init__(self,
ocean_lut=os.path.join('.', 'luts', 'ocean',
'ocean_core_meris.nc4')):
# sys.stderr.write('entering CawaTcwvOceanCore... \n')
libdir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'libs')
sys.path.append(libdir)
with Dataset(ocean_lut, 'r') as ncds:
# get the full lut
self.lut = np.array(ncds.variables['lut'][:], order='F')
self.jlut = np.array(ncds.variables['jlut'][:], order='F')
self.axes = tuple([
np.array(ncds.variables[a][:])
for a in ncds.variables['lut'].dimensions[:-1]
])
self.jaxes = tuple([
np.array(ncds.variables[a][:])
for a in ncds.variables['jlut'].dimensions[:-1]
])
self.ny_nx = np.array(ncds.variables['jaco'][:])
self.wb = ncds.getncattr('win_bnd').split(',')
self.ab = ncds.getncattr('abs_bnd').split(',')
# generic forward
self._forward = lut2func.lut2func(self.lut, self.axes)
# generic jacobian
self._jacobian = lut2jacobian_lut.jlut2func({
'lut': self.jlut,
'axes': self.jaxes,
'ny': self.ny_nx[0],
'nx': self.ny_nx[1]
})
# global predefinition of input for speed reasons
self.xaa = np.zeros(3)
self.par = np.zeros(3)
self.mes = np.zeros(len(self.wb) + len(self.ab))
# local predefine inp for speed
inp = np.zeros(6)
def forward(woo, geo):
"""
Input:
woo: state (wvc aot wsp)
geo: azi vie suz
aot is aot at winband [0]
wvc is sqrt of wvc
Output:
normalized radiances at winbands
-np.log(effective_transmission)/sqrt(amf) at absbands
effective_transmission= L_toa/L_0
with L_0 is normalized radiance without water vapor
"""
inp[:3], inp[3:] = woo, geo
return self._forward(inp)
self.forward = forward
def jforward(woo, geo):
"""
as forward, but returns jacobian
output must be limited to the first three elements (the state and not the geometry)
"""
inp[:3], inp[3:] = woo, geo
return self._jacobian(inp)[:, :3]
self.jforward = jforward
# min_state
a = np.array([self.axes[i].min() for i in range(3)])
# max_state
b = np.array([self.axes[i].max() for i in range(3)])
self.inverter = oe.my_inverter(self.forward, a, b, jaco=self.jforward)
# finaly preset SE
sew = [0.0001 for i in self.wb]
sea = [0.001 for i in self.ab]
self.SE = np.diag(sew + sea)
def generate_jacobian_lut(luts, axes, xidx=None):
'''
generates a jacobian-lut at exactly the same
positions (axes points) as the original luts
xidx gives the positions, where the jacobian
will be calulated, the other positions are ignored
(e.g. geometry in LUTs )
'''
func = l2f.lut2func(luts, axes)
if isinstance(luts, tuple):
ny = len(luts)
dtype = luts[0].dtype
elif isinstance(luts, np.ndarray):
ny = luts.shape[-1]
dtype = luts.dtype
else:
print('Lut is not of right type')
return None
if xidx is None:
nx = len(axes)
else:
nx = len(xidx)
dimi = np.array([len(ax) for ax in (axes)])
dimj = [len(ax) for ax in (axes)]
dimj.append(ny * nx)
out = np.zeros(dimj, dtype=dtype)
a = np.zeros(len(axes))
b = np.zeros(len(axes))
a = np.array([ax.min() for ax in axes])
b = np.array([ax.max() for ax in axes])
def njac(x, dx=None):
return numerical_jacoby(a,
b,
x,
func,
nx,
ny,
delta=0.01,
xidx=xidx,
dx=dx)
def optimal_dx(i, ax):
if i == 0:
dx = np.abs(ax[1] - ax[0]) / 2.
elif i == ax.size - 1:
dx = np.abs(ax[-1] - ax[-2]) / 2.
else:
dx = min(np.abs(ax[i + 1] - ax[i]), np.abs(ax[i] - ax[i - 1])) / 2.
return dx
print('Filling the jacobian ...')
progress = wln(dimi.prod(), 'Progress', modulo=10000)
for i in np.arange(dimi.prod()):
idx = np.unravel_index(i, dimi)
wo = np.array([ax[i] for i, ax in zip(idx, axes)])
dx = np.array([optimal_dx(i, ax) for i, ax in zip(idx, axes)])
out[idx] = njac(wo, dx).ravel()
progress.event()
print('')
print('Done')
return {'lut': out, 'ny': ny, 'nx': nx, 'axes': axes, 'xidx': xidx}
# either as Tuple
luts1 = (luta, lutb, lutc, lutd)
# or as single np.array
luts2 = np.array((luta, lutb, lutc, lutd)).transpose([1, 2, 3, 0])
print('R^3 --> R^4, LUT shape:', luts2.shape)
#second create the axes
xx1 = np.array([3., 4., 6., 7., 9., 15.], dtype=dtype)
xx2 = np.linspace(-10., 30., 400, dtype=dtype)
xx3 = np.linspace(-1., 3., 100, dtype=dtype)
#as a tuple
axes = (xx1, xx2, xx3)
#finaly create the func
func1 = l2f.lut2func(luts1, axes)
func2 = l2f.lut2func(luts2, axes)
#for the jacobians
#use the function luts
jlut = generate_jacobian_lut(luts2, axes, xidx=[0, 2])
#save the jacobians
lut2ncdf(jlut, 'jlut3_test.nc')
#and make a function from it
jfun = ncdf2func('jlut3_test.nc')
# now test the jacobian function:
print('interploated:')
print(jfun([3., 10., 1.]))
#and compare it with a numerical_jacoby
