def test(what='fortran'): import lut2func as l2f if what == 'purpy': l2f.interpolators = l2f.interpolators_pure_python elif what == 'numba': l2f.interpolators = l2f.interpolators_numba_python else: l2f.interpolators = l2f.interpolators_fortran # example for R^2-->R^3 luta = np.arange(24, dtype=np.float).reshape(6, 4, order='C') lutb = np.arange(24, dtype=np.float).reshape(6, 4, order='F') ** 2 lutc = np.sqrt(np.arange(24, dtype=np.float).reshape(6, 4, order='F')) luts = (luta, lutb, lutc) xx = np.array([3., 4., 6., 7., 9., 15.]) yy = np.array([1., 5., 10, 15])[::-1] axes = (xx, yy) # Variant A: luts is a tuple of luts funca = l2f.lut2func(luts, axes) funcav = l2f.lut2func(luts, axes, vectorial=True) # Variant B: luts is a Nd Array, last dimension is dimension of result luts = np.array((luta, lutb, lutc), dtype=np.float32).transpose([1, 2, 0]) print(luts.shape) funcb = l2f.lut2func(luts, axes) funcbv = l2f.lut2func(luts, axes, vectorial=True) print(luts.shape, len(axes)) # sys.exit() nn = 10000 import time pos = np.array([3.5, 11.]) ppos = np.array([pos for i in range(nn)]) a = time.time() print('VEC (%i): map-coordinates' % nn) for ff in (funcav, funcbv): a = time.time() erg = ff(ppos) print(erg[0], ': %.2f us' % (((time.time() - a)) / float(nn) * 1.e6)) print('SEQ (%i): %s, map-coordinates, %s, map-coordinates ' % (nn, what, what)) for ff in (funca, funcav, funcb, funcbv): a = time.time() for i in range(nn): _ = ff(pos) print(_, ': %.2f us' % ((time.time() - a) / float(nn) * 1.e6))
def jlut2func(dum): func = l2f.lut2func(dum["lut"], dum["axes"]) def jfunc(woo): return func(woo).reshape(dum["ny"], dum["nx"]) return jfunc
def jlut2func(dum): func = l2f.lut2func(dum['lut'], dum['axes']) def jfunc(woo): return func(woo).reshape(dum['ny'], dum['nx']) return jfunc
def jlut2func(dum): func = l2f.lut2func(dum['lut'], dum['axes']) def jfunc(woo): #return func(woo).reshape(dum['ny'],dum['nx'],order='F') return np.array(func(woo).reshape(dum['ny'], dum['nx']), order='F') return jfunc
def test3to1(): # example for R^3-->R^1 import lut2func as l2f # from lut2func import lut2func lut3d = np.arange(3 * 4 * 5, dtype=np.float).reshape(3, 4, 5) x0 = np.array([3., 4., 6.]) x1 = np.array([1., 4., 8., 10.]) x2 = np.array([1., 2., 4., 8., 16.]) func3d = l2f.lut2func((lut3d,), (x0, x1, x2)) print(func3d([3.5, 5., 10.])) print(l2f.interpolators)
def generate_jacobian_lut(luts, axes): """ generates a jacobian-lut at exactly the same positions (axes points) as the original luts """ func = l2f.lut2func(luts, axes) if isinstance(luts, tuple): ny = len(luts) elif isinstance(luts, np.ndarray): ny = luts.shape[-1] else: print "Luts is not of right type" return None nx = len(axes) dimi = np.array([len(ax) for ax in (axes)]) dimj = [len(ax) for ax in (axes)] dimj.append(ny * nx) out = np.zeros(dimj) a = np.zeros(len(axes)) b = np.zeros(len(axes)) a = np.array([ax.min() for ax in axes]) b = np.array([ax.max() for ax in axes]) def njac(x): return numerical_jacoby(a, b, x, func, nx, ny, delta=0.01) print ("Filling the jacobian ...") progress = wln.wln(dimi.prod(), "Progress", modulo=10000) for i in np.arange(dimi.prod()): idx = np.unravel_index(i, dimi) wo = np.array([ax[i] for i, ax in zip(idx, axes)]) out[idx] = njac(wo).ravel() progress.event() print ("") print ("Done") return {"lut": out, "ny": ny, "nx": nx, "axes": axes}
def generate_jacobian_lut(luts, axes): ''' generates a jacobian-lut at exactly the same positions (axes points) as the original luts ''' func = l2f.lut2func(luts, axes) if isinstance(luts, tuple): ny = len(luts) elif isinstance(luts, np.ndarray): ny = luts.shape[-1] else: print 'Luts is not of right type' return None nx = len(axes) dimi = np.array([len(ax) for ax in (axes)]) dimj = [len(ax) for ax in (axes)] dimj.append(ny * nx) out = np.zeros(dimj) a = np.zeros(len(axes)) b = np.zeros(len(axes)) a = np.array([ax.min() for ax in axes]) b = np.array([ax.max() for ax in axes]) def njac(x): return numerical_jacoby(a, b, x, func, nx, ny, delta=0.01) print('Filling the jacobian ...') progress = wln.wln(dimi.prod(), 'Progress', modulo=10000) for i in np.arange(dimi.prod()): idx = np.unravel_index(i, dimi) wo = np.array([ax[i] for i, ax in zip(idx, axes)]) out[idx] = njac(wo).ravel() progress.event() print('') print('Done') return {'lut': out, 'ny': ny, 'nx': nx, 'axes': axes}
luta = np.arange(240000, dtype=np.float).reshape(6, 40000, order="C") lutb = np.arange(240000, dtype=np.float).reshape(6, 40000, order="F") ** 2 lutc = np.sqrt(np.arange(240000, dtype=np.float).reshape(6, 40000, order="F")) # either as Tuple luts1 = (luta, lutb, lutc) # or as single np.array luts2 = np.array((luta, lutb, lutc)).transpose([1, 2, 0]) # second create the axes xx1 = np.array([3.0, 4.0, 6.0, 7.0, 9.0, 15.0]) xx2 = np.linspace(-10.0, 30.0, 40000) # as a tuple axes = (xx1, xx2) # finaly create the func func1 = l2f.lut2func(luts1, axes) func2 = l2f.lut2func(luts2, axes) # for the jacobians # use the function luts jlut = generate_jacobian_lut(luts2, axes) # save the jacobians lut2ncdf(jlut, "jlut_test.nc") # and make a function from it jfun = ncdf2func("jlut_test.nc") # now test the jacobian function: print jfun([3.0, 10.0]) # and compare it with a numerical_jacoby a = np.array([3, 1]) b = np.array([15, 15])
def __init__(self,corefile=os.path.join(about_me()['cawadir'],'luts','cloud_core_meris.nc4'),used_ab=None): self.about_me=about_me() with Dataset(corefile,'r') as ncds: #get the full lut self.lut=np.array(ncds.variables['lut'][:],order='F') self.jlut=np.array(ncds.variables['jlut'][:],order='F') self.axes=tuple([np.array(ncds.variables[a][:]) for a in ncds.variables['lut'].dimensions[:-1]]) self.jaxes=tuple([np.array(ncds.variables[a][:]) for a in ncds.variables['jlut'].dimensions[:-1]]) self.ny_nx=np.array(ncds.variables['jaco'][:]) self.wb=ncds.getncattr('win_bnd').split(',') self.ab=ncds.getncattr('abs_bnd').split(',') self.cha={bb: {kk:ncds.groups['cha'].groups[bb].getncattr(kk) for kk in ('bwvl','cwvl')} for bb in self.wb+self.ab} if used_ab is None: self.ab_idx=None else: self.ab=[used_ab,] self.ab_idx=pos_in_list(ncds.getncattr('abs_bnd').split(','),used_ab)[0] #generic forward self._forward=lut2func.lut2func(self.lut,self.axes) #generic jacobian self._jacobian=lut2jacobian_lut.jlut2func({'lut':self.jlut ,'axes':self.jaxes ,'ny':self.ny_nx[0] ,'nx':self.ny_nx[1]}) #global predefinition of input for speed reasons self.xaa=np.zeros(2) self.par=np.zeros(5) self.mes=np.zeros(len(self.wb)+len(self.ab)) #local predefine inp for speed inp=np.zeros(7) def forward(woo,par): ''' Input: woo: state (ctp,cot) par: parameter (alb, sza, vza, azd, dwvl) aot is aot at winband [0] wvc is sqrt of wvc Output: normalized radiances at winbands and absbands ''' inp[:2],inp[2:]=woo,par if self.ab_idx is None: return self._forward(inp) else: return self._forward(inp)[((0,1+self.ab_idx),)] self.forward =forward def jforward(woo,geo): ''' as forward, but returns jacobian output must be limited to the first three elements (the state and not the geometry) ''' inp[:2],inp[2:]=woo,geo if self.ab_idx is None: return self._jacobian(inp)[:,:2] else: return self._jacobian(inp)[(0,1+self.ab_idx),:2] self.jforward=jforward #min_state a=np.array([self.axes[i].min() for i in range(2)]) #max_state b=np.array([self.axes[i].max() for i in range(2)]) self.inverter=oe.my_inverter(self.forward,a,b,jaco=self.jforward) #finaly preset SE sew=[0.0001 for i in self.wb] sea=[0.0001 for i in self.ab] self.SE=np.diag(sew+sea)
def __init__(self, corefile=os.path.join(about_me()['cawadir'], 'luts', 'cloud_core_meris.nc4'), used_ab=None): self.about_me = about_me() with Dataset(corefile, 'r') as ncds: #get the full lut self.lut = np.array(ncds.variables['lut'][:], order='F') self.jlut = np.array(ncds.variables['jlut'][:], order='F') self.axes = tuple([ np.array(ncds.variables[a][:]) for a in ncds.variables['lut'].dimensions[:-1] ]) self.jaxes = tuple([ np.array(ncds.variables[a][:]) for a in ncds.variables['jlut'].dimensions[:-1] ]) self.ny_nx = np.array(ncds.variables['jaco'][:]) self.wb = ncds.getncattr('win_bnd').split(',') self.ab = ncds.getncattr('abs_bnd').split(',') self.cha = { bb: { kk: ncds.groups['cha'].groups[bb].getncattr(kk) for kk in ('bwvl', 'cwvl') } for bb in self.wb + self.ab } if used_ab is None: self.ab_idx = None else: self.ab = [ used_ab, ] self.ab_idx = pos_in_list( ncds.getncattr('abs_bnd').split(','), used_ab)[0] #generic forward self._forward = lut2func.lut2func(self.lut, self.axes) #generic jacobian self._jacobian = lut2jacobian_lut.jlut2func({ 'lut': self.jlut, 'axes': self.jaxes, 'ny': self.ny_nx[0], 'nx': self.ny_nx[1] }) #global predefinition of input for speed reasons self.xaa = np.zeros(2) self.par = np.zeros(5) self.mes = np.zeros(len(self.wb) + len(self.ab)) #local predefine inp for speed inp = np.zeros(7) def forward(woo, par): ''' Input: woo: state (ctp,cot) par: parameter (alb, sza, vza, azd, dwvl) aot is aot at winband [0] wvc is sqrt of wvc Output: normalized radiances at winbands and absbands ''' inp[:2], inp[2:] = woo, par if self.ab_idx is None: return self._forward(inp) else: return self._forward(inp)[((0, 1 + self.ab_idx), )] self.forward = forward def jforward(woo, geo): ''' as forward, but returns jacobian output must be limited to the first three elements (the state and not the geometry) ''' inp[:2], inp[2:] = woo, geo if self.ab_idx is None: return self._jacobian(inp)[:, :2] else: return self._jacobian(inp)[(0, 1 + self.ab_idx), :2] self.jforward = jforward #min_state a = np.array([self.axes[i].min() for i in range(2)]) #max_state b = np.array([self.axes[i].max() for i in range(2)]) self.inverter = oe.my_inverter(self.forward, a, b, jaco=self.jforward) #finaly preset SE sew = [0.0001 for i in self.wb] sea = [0.0001 for i in self.ab] self.SE = np.diag(sew + sea)
def __init__(self, ocean_lut=os.path.join('.', 'luts', 'ocean', 'ocean_core_meris.nc4')): # sys.stderr.write('entering CawaTcwvOceanCore... \n') libdir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'libs') sys.path.append(libdir) with Dataset(ocean_lut, 'r') as ncds: # get the full lut self.lut = np.array(ncds.variables['lut'][:], order='F') self.jlut = np.array(ncds.variables['jlut'][:], order='F') self.axes = tuple([np.array(ncds.variables[a][:]) for a in ncds.variables['lut'].dimensions[:-1]]) self.jaxes = tuple([np.array(ncds.variables[a][:]) for a in ncds.variables['jlut'].dimensions[:-1]]) self.ny_nx = np.array(ncds.variables['jaco'][:]) self.wb = ncds.getncattr('win_bnd').split(',') self.ab = ncds.getncattr('abs_bnd').split(',') # generic forward self._forward = lut2func.lut2func(self.lut, self.axes) # generic jacobian self._jacobian = lut2jacobian_lut.jlut2func({'lut': self.jlut, 'axes': self.jaxes, 'ny': self.ny_nx[0], 'nx': self.ny_nx[1]}) # global predefinition of input for speed reasons self.xaa = np.zeros(3) self.par = np.zeros(3) self.mes = np.zeros(len(self.wb) + len(self.ab)) # local predefine inp for speed inp = np.zeros(6) def forward(woo, geo): """ Input: woo: state (wvc aot wsp) geo: azi vie suz aot is aot at winband [0] wvc is sqrt of wvc Output: normalized radiances at winbands -np.log(effective_transmission)/sqrt(amf) at absbands effective_transmission= L_toa/L_0 with L_0 is normalized radiance without water vapor """ inp[:3], inp[3:] = woo, geo return self._forward(inp) self.forward = forward def jforward(woo, geo): """ as forward, but returns jacobian output must be limited to the first three elements (the state and not the geometry) """ inp[:3], inp[3:] = woo, geo return self._jacobian(inp)[:, :3] self.jforward = jforward # min_state a = np.array([self.axes[i].min() for i in range(3)]) # max_state b = np.array([self.axes[i].max() for i in range(3)]) self.inverter = oe.my_inverter(self.forward, a, b, jaco=self.jforward) # finaly preset SE sew = [0.0001 for i in self.wb] sea = [0.001 for i in self.ab] self.SE = np.diag(sew + sea)
return out return function if __name__ == '__main__': from lut2func import lut2func # example for R^2-->R^3 luta = np.arange(24, dtype=np.float).reshape(6, 4, order='C') lutb = np.arange(24, dtype=np.float).reshape(6, 4, order='F') ** 2 lutc = np.sqrt(np.arange(24, dtype=np.float).reshape(6, 4, order='F')) luts = (luta, lutb, lutc) xx = np.array([3., 4., 6., 7., 9., 15.]) yy = np.array([1., 5., 10, 15])[::-1] axes = (xx, yy) funca = lut2func(luts, axes) luts = np.array((luta, lutb, lutc)).transpose([1, 2, 0]) funcb = lut2func(luts, axes) import time for ff in (funca, funcb): a = time.time() for i in range(1000): _ = ff(np.array([3.5, 11.])) print _, ': %i us' % ((time.time() - a) * 1000)
def __init__(self, ocean_lut=os.path.join('.', 'luts', 'ocean', 'ocean_core_meris.nc4')): # sys.stderr.write('entering CawaTcwvOceanCore... \n') libdir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'libs') sys.path.append(libdir) with Dataset(ocean_lut, 'r') as ncds: # get the full lut self.lut = np.array(ncds.variables['lut'][:], order='F') self.jlut = np.array(ncds.variables['jlut'][:], order='F') self.axes = tuple([ np.array(ncds.variables[a][:]) for a in ncds.variables['lut'].dimensions[:-1] ]) self.jaxes = tuple([ np.array(ncds.variables[a][:]) for a in ncds.variables['jlut'].dimensions[:-1] ]) self.ny_nx = np.array(ncds.variables['jaco'][:]) self.wb = ncds.getncattr('win_bnd').split(',') self.ab = ncds.getncattr('abs_bnd').split(',') # generic forward self._forward = lut2func.lut2func(self.lut, self.axes) # generic jacobian self._jacobian = lut2jacobian_lut.jlut2func({ 'lut': self.jlut, 'axes': self.jaxes, 'ny': self.ny_nx[0], 'nx': self.ny_nx[1] }) # global predefinition of input for speed reasons self.xaa = np.zeros(3) self.par = np.zeros(3) self.mes = np.zeros(len(self.wb) + len(self.ab)) # local predefine inp for speed inp = np.zeros(6) def forward(woo, geo): """ Input: woo: state (wvc aot wsp) geo: azi vie suz aot is aot at winband [0] wvc is sqrt of wvc Output: normalized radiances at winbands -np.log(effective_transmission)/sqrt(amf) at absbands effective_transmission= L_toa/L_0 with L_0 is normalized radiance without water vapor """ inp[:3], inp[3:] = woo, geo return self._forward(inp) self.forward = forward def jforward(woo, geo): """ as forward, but returns jacobian output must be limited to the first three elements (the state and not the geometry) """ inp[:3], inp[3:] = woo, geo return self._jacobian(inp)[:, :3] self.jforward = jforward # min_state a = np.array([self.axes[i].min() for i in range(3)]) # max_state b = np.array([self.axes[i].max() for i in range(3)]) self.inverter = oe.my_inverter(self.forward, a, b, jaco=self.jforward) # finaly preset SE sew = [0.0001 for i in self.wb] sea = [0.001 for i in self.ab] self.SE = np.diag(sew + sea)
def generate_jacobian_lut(luts, axes, xidx=None): ''' generates a jacobian-lut at exactly the same positions (axes points) as the original luts xidx gives the positions, where the jacobian will be calulated, the other positions are ignored (e.g. geometry in LUTs ) ''' func = l2f.lut2func(luts, axes) if isinstance(luts, tuple): ny = len(luts) dtype = luts[0].dtype elif isinstance(luts, np.ndarray): ny = luts.shape[-1] dtype = luts.dtype else: print('Lut is not of right type') return None if xidx is None: nx = len(axes) else: nx = len(xidx) dimi = np.array([len(ax) for ax in (axes)]) dimj = [len(ax) for ax in (axes)] dimj.append(ny * nx) out = np.zeros(dimj, dtype=dtype) a = np.zeros(len(axes)) b = np.zeros(len(axes)) a = np.array([ax.min() for ax in axes]) b = np.array([ax.max() for ax in axes]) def njac(x, dx=None): return numerical_jacoby(a, b, x, func, nx, ny, delta=0.01, xidx=xidx, dx=dx) def optimal_dx(i, ax): if i == 0: dx = np.abs(ax[1] - ax[0]) / 2. elif i == ax.size - 1: dx = np.abs(ax[-1] - ax[-2]) / 2. else: dx = min(np.abs(ax[i + 1] - ax[i]), np.abs(ax[i] - ax[i - 1])) / 2. return dx print('Filling the jacobian ...') progress = wln(dimi.prod(), 'Progress', modulo=10000) for i in np.arange(dimi.prod()): idx = np.unravel_index(i, dimi) wo = np.array([ax[i] for i, ax in zip(idx, axes)]) dx = np.array([optimal_dx(i, ax) for i, ax in zip(idx, axes)]) out[idx] = njac(wo, dx).ravel() progress.event() print('') print('Done') return {'lut': out, 'ny': ny, 'nx': nx, 'axes': axes, 'xidx': xidx}
# either as Tuple luts1 = (luta, lutb, lutc, lutd) # or as single np.array luts2 = np.array((luta, lutb, lutc, lutd)).transpose([1, 2, 3, 0]) print('R^3 --> R^4, LUT shape:', luts2.shape) #second create the axes xx1 = np.array([3., 4., 6., 7., 9., 15.], dtype=dtype) xx2 = np.linspace(-10., 30., 400, dtype=dtype) xx3 = np.linspace(-1., 3., 100, dtype=dtype) #as a tuple axes = (xx1, xx2, xx3) #finaly create the func func1 = l2f.lut2func(luts1, axes) func2 = l2f.lut2func(luts2, axes) #for the jacobians #use the function luts jlut = generate_jacobian_lut(luts2, axes, xidx=[0, 2]) #save the jacobians lut2ncdf(jlut, 'jlut3_test.nc') #and make a function from it jfun = ncdf2func('jlut3_test.nc') # now test the jacobian function: print('interploated:') print(jfun([3., 10., 1.])) #and compare it with a numerical_jacoby