def __init__(self,
land_lut=os.path.join('.', 'luts', 'land',
'land_core_modis_aqua.nc4'),
ocean_lut=os.path.join('.', 'luts', 'ocean',
'ocean_core_modis_aqua.nc4')):
self.cawa_land = cawa_land.CawaTcwvLandCore(land_lut)
self.cawa_ocean = cawa_ocean.CawaTcwvOceanCore(ocean_lut)
self.cawa_utils = cu.CawaUtils()
def initialize(self, operator):
"""
GPF initialize method
:param operator
:return:
"""
resource_root = os.path.dirname(__file__)
f = open(tempfile.gettempdir() + '/cava_tcwv_modis.log', 'w')
f.write('Python module location: ' + __file__ + '\n')
f.write('Python module location parent: ' + resource_root + '\n')
# get source product:
source_product = operator.getSourceProduct('sourceProduct')
if not source_product:
raise RuntimeError('No source product specified or product not found - cannot continue.')
# f.write('Start initialize: source product is' + source_product.getFileLocation().getAbsolutePath() + '\n')
f.write('Start initialize: source product is' + source_product.getName() + '\n')
# get parameters:
self.temperature = operator.getParameter('temperature') # todo: get temperature field from ERA-Interim
self.pressure = operator.getParameter('pressure') # todo: get pressure field from ERA-Interim
self.prior_aot = operator.getParameter('prior_aot') # todo: clarify if only one AOT is needed
if os.path.isdir(resource_root):
land_lut = os.path.join(resource_root, 'luts/land/land_core_modis_aqua.nc4')
ocean_lut = os.path.join(resource_root, 'luts/ocean/ocean_core_modis_aqua.nc4')
shared_libs_dir = resource_root
else:
with zipfile.ZipFile(resource_root) as zf:
auxpath = SystemUtils.getAuxDataPath()
f.write('auxpath: ' + str(auxpath) + '\n')
if not os.path.exists(os.path.join(str(auxpath), 'cawa/luts/land/land_core_modis_aqua.nc4')):
land_lut = zf.extract('luts/land/land_core_modis_aqua.nc4', os.path.join(str(auxpath), 'cawa'))
f.write('extracted LUT land: ' + land_lut + '\n')
else:
land_lut = os.path.join(str(auxpath), 'cawa/luts/land/land_core_modis_aqua.nc4')
f.write('existing LUT land: ' + land_lut + '\n')
if not os.path.exists(os.path.join(str(auxpath), 'cawa/luts/ocean/ocean_core_modis_aqua.nc4')):
ocean_lut = zf.extract('luts/ocean/ocean_core_modis_aqua.nc4', os.path.join(str(auxpath), 'cawa'))
f.write('extracted LUT ocean: ' + ocean_lut + '\n')
else:
ocean_lut = os.path.join(str(auxpath), 'cawa/luts/ocean/ocean_core_modis_aqua.nc4')
f.write('existing LUT ocean: ' + ocean_lut + '\n')
shared_libs_dir = tempfile.gettempdir()
if not os.path.exists(shared_libs_dir + '/lib-python'):
lib_interpolator = zf.extract('lib-python/interpolators.so', shared_libs_dir)
lib_nd_interpolator = zf.extract('lib-python/nd_interpolator.so', shared_libs_dir)
lib_oec = zf.extract('lib-python/optimal_estimation_core.so', shared_libs_dir)
else:
lib_interpolator = os.path.join(str(shared_libs_dir), 'lib-python/interpolators.so')
lib_nd_interpolator = os.path.join(str(shared_libs_dir), 'lib-python/nd_interpolator.so')
lib_oec = os.path.join(str(shared_libs_dir), 'lib-python/optimal_estimation_core.so')
f.write('LUT land: ' + land_lut + '\n')
f.write('LUT ocean: ' + ocean_lut + '\n')
f.write('shared_libs_dir = %s' % (shared_libs_dir + '/lib-python') + '\n')
# sys.path.append(shared_libs_dir + '/lib-python')
sys.path.append(shared_libs_dir + '/libs')
import cawa_tcwv_modis_core as cawa_core
import cawa_utils as cu
self.cawa = cawa_core.CawaTcwvModisCore(land_lut, ocean_lut)
self.cawa_utils = cu.CawaUtils()
width = source_product.getSceneRasterWidth()
height = source_product.getSceneRasterHeight()
f.write('Source product width, height = ...' + str(width) + ', ' + str(height) + '\n')
# get source bands:
self.rho_toa_2_band = self.get_band(source_product, 'EV_250_Aggr1km_RefSB_2') # RefSB is from Idepix product!
self.rho_toa_5_band = self.get_band(source_product, 'EV_500_Aggr1km_RefSB_5') # RefSB is from Idepix product!
self.rho_toa_17_band = self.get_band(source_product, 'EV_1KM_RefSB_17')
self.rho_toa_18_band = self.get_band(source_product, 'EV_1KM_RefSB_18')
self.rho_toa_19_band = self.get_band(source_product, 'EV_1KM_RefSB_19')
self.sza_band = self.get_band(source_product, 'SolarZenith')
self.vza_band = self.get_band(source_product, 'SensorZenith')
self.saa_band = self.get_band(source_product, 'SolarAzimuth')
self.vaa_band = self.get_band(source_product, 'SensorAzimuth')
self.prior_t2m_band = None
self.prior_msl_band = None
self.prior_tcwv_band = None
self.prior_wsp_band = None
if cu.CawaUtils.band_exists('t2m', source_product.getBandNames()):
self.prior_t2m_band = self.get_band(source_product, 't2m')
if cu.CawaUtils.band_exists('msl', source_product.getBandNames()):
self.prior_msl_band = self.get_band(source_product, 'msl')
if cu.CawaUtils.band_exists('tcwv', source_product.getBandNames()):
self.prior_tcwv_band = self.get_band(source_product, 'tcwv')
if cu.CawaUtils.band_exists('ws', source_product.getBandNames()):
self.prior_wsp_band = self.get_band(source_product, 'ws')
#self.classif_band = self.get_band(source_product, 'pixel_classif_flags')
self.classif_band = None
if cu.CawaUtils.band_exists('pixel_classif_flags', source_product.getBandNames()):
self.classif_band = self.get_band(source_product, 'pixel_classif_flags')
elif cu.CawaUtils.band_exists('cloud_classif_flags', source_product.getBandNames()):
self.classif_band = self.get_band(source_product, 'cloud_classif_flags')
# setup target product:
cawa_product = snappy.Product('pyCAWA', 'CAWA TCWV', width, height)
cawa_product.setDescription('CAWA TCWV product')
cawa_product.setStartTime(source_product.getStartTime())
cawa_product.setEndTime(source_product.getEndTime())
# setup target bands:
self.tcwv_band = cawa_product.addBand('tcwv', snappy.ProductData.TYPE_FLOAT32)
self.tcwv_band.setNoDataValue(TCWV_NODATA_VALUE)
self.tcwv_band.setNoDataValueUsed(True)
self.tcwv_band.setUnit('mm')
self.tcwv_band.setDescription('Total column of water vapour')
self.tcwv_flags_band = cawa_product.addBand('tcwv_flags', snappy.ProductData.TYPE_UINT8)
self.tcwv_flags_band.setUnit('dl')
self.tcwv_flags_band.setDescription('TCWV flags band')
# copy flag bands, tie points, geocoding:
snappy.ProductUtils.copyFlagBands(source_product, cawa_product, True)
# snappy.ProductUtils.copyFlagBands(classif_product, cawa_product, True)
# snappy.ProductUtils.copyTiePointGrids(source_product, cawa_product) # todo: wait for fix in SNAP
source_product.transferGeoCodingTo(cawa_product, None)
operator.setTargetProduct(cawa_product)
f.write('end initialize.')
f.close()
def initialize(self, operator):
"""
GPF initialize method
:param operator
:return:
"""
resource_root = os.path.dirname(__file__)
f = open(tempfile.gettempdir() + '/cava_ctp.log', 'w')
f.write('Python module location: ' + __file__ + '\n')
# print('Python module location: ' + __file__ + '\n')
f.write('Python module location parent: ' + resource_root + '\n')
# get source product:
source_product = operator.getSourceProduct('sourceProduct')
if not source_product:
raise RuntimeError('No source product specified or product not found - cannot continue.')
f.write('Start initialize: source product is ' + source_product.getName() + '\n')
print('Start initialize: source product is ' + source_product.getName() + '\n')
if os.path.isdir(resource_root):
f.write('resource_root is dir ' + '\n')
print('resource_root is dir ' + '\n')
cloud_lut = os.path.join(resource_root, 'luts/cloud_core_meris.nc4')
str_coeffs_lut = os.path.join(resource_root, 'luts/stray_coeff_potenz4.nc')
ws_alb_lut = os.path.join(resource_root, 'luts/ws_alb_10_2005.nc')
spectral_fluxes_input_path = os.path.join(resource_root, 'luts/meris_sun_spectral_flux_rr_10_11.txt')
shared_libs_dir = resource_root
else:
f.write('extracting resources... ' + '\n')
with zipfile.ZipFile(resource_root) as zf:
auxpath = SystemUtils.getAuxDataPath()
f.write('auxpath: ' + str(auxpath) + '\n')
if not os.path.exists(os.path.join(str(auxpath), 'cawa/luts/cloud_core_meris.nc4')):
cloud_lut = zf.extract('luts/cloud_core_meris.nc4', os.path.join(str(auxpath), 'cawa'))
f.write('extracted LUT cloud: ' + cloud_lut + '\n')
else:
cloud_lut = os.path.join(str(auxpath), 'cawa/luts/cloud_core_meris.nc4')
f.write('existing LUT cloud: ' + cloud_lut + '\n')
if not os.path.exists(os.path.join(str(auxpath), 'cawa/luts/stray_coeff_potenz4.nc')):
str_coeffs_lut = zf.extract('luts/stray_coeff_potenz4.nc', os.path.join(str(auxpath), 'cawa'))
f.write('extracted stray_coeff: ' + str_coeffs_lut + '\n')
else:
str_coeffs_lut = os.path.join(str(auxpath), 'cawa/luts/stray_coeff_potenz4.nc')
f.write('existing stray_coeff: ' + str_coeffs_lut + '\n')
if not os.path.exists(os.path.join(str(auxpath), 'cawa/luts/ws_alb_10_2005.nc')):
ws_alb_lut = zf.extract('luts/ws_alb_10_2005.nc', os.path.join(str(auxpath), 'cawa'))
f.write('extracted ws_alb_10: ' + ws_alb_lut + '\n')
else:
ws_alb_lut = os.path.join(str(auxpath), 'cawa/luts/ws_alb_10_2005.nc')
f.write('existing ws_alb_10: ' + ws_alb_lut + '\n')
if not os.path.exists(os.path.join(str(auxpath), 'cawa/luts/meris_sun_spectral_flux_rr_10_11.txt')):
spectral_fluxes_input_path = zf.extract('luts/meris_sun_spectral_flux_rr_10_11.txt', os.path.join(str(auxpath), 'cawa'))
f.write('extracted meris_sun_spectral_flux_rr_10_11: ' + spectral_fluxes_input_path + '\n')
else:
spectral_fluxes_input_path = os.path.join(str(auxpath), 'cawa/luts/meris_sun_spectral_flux_rr_10_11.txt')
f.write('existing meris_sun_spectral_flux_rr_10_11: ' + spectral_fluxes_input_path + '\n')
shared_libs_dir = tempfile.gettempdir()
if not os.path.exists(shared_libs_dir + '/lib-python'):
lib_interpolator = zf.extract('lib-python/interpolators.so', shared_libs_dir)
lib_nd_interpolator = zf.extract('lib-python/nd_interpolator.so', shared_libs_dir)
lib_oec = zf.extract('lib-python/optimal_estimation_core.so', shared_libs_dir)
else:
lib_interpolator = os.path.join(str(shared_libs_dir), 'lib-python/interpolators.so')
lib_nd_interpolator = os.path.join(str(shared_libs_dir), 'lib-python/nd_interpolator.so')
lib_oec = os.path.join(str(shared_libs_dir), 'lib-python/optimal_estimation_core.so')
f.write('shared_libs_dir = %s' % (shared_libs_dir + '/lib-python') + '\n')
sys.path.append(shared_libs_dir + '/lib-python')
import cawa_ctp_meris_core as cawa_core
import cawa_utils as cu
self.cawa = cawa_core.CawaCtpMerisCore(cloud_lut)
self.cawa_utils = cu.CawaUtils()
width = source_product.getSceneRasterWidth()
height = source_product.getSceneRasterHeight()
f.write('Source product width, height = ...' + str(width) + ', ' + str(height) + '\n')
# get source bands:
self.rad_10_band = self.get_band(source_product, 'radiance_10') # reflectance is from Idepix product!
self.rad_11_band = self.get_band(source_product, 'radiance_11')
self.detector_index_band = self.get_band(source_product, 'detector_index')
self.sza_band = self.get_band(source_product, 'sun_zenith')
self.vza_band = self.get_band(source_product, 'view_zenith')
self.saa_band = self.get_band(source_product, 'sun_azimuth')
self.vaa_band = self.get_band(source_product, 'view_azimuth')
self.lat_band = self.get_band(source_product, 'latitude')
self.lon_band = self.get_band(source_product, 'longitude')
self.alt_band = self.get_band(source_product, 'dem_alt')
self.l1_flag_band = self.get_band(source_product, 'l1_flags')
self.classif_band = None
if cu.CawaUtils.band_exists('pixel_classif_flags', source_product.getBandNames()):
self.classif_band = self.get_band(source_product, 'pixel_classif_flags')
elif cu.CawaUtils.band_exists('cloud_classif_flags', source_product.getBandNames()):
self.classif_band = self.get_band(source_product, 'cloud_classif_flags')
# setup target product:
cawa_product = snappy.Product('pyCAWA', 'CAWA CTP', width, height)
cawa_product.setDescription('CAWA CTP product')
cawa_product.setStartTime(source_product.getStartTime())
cawa_product.setEndTime(source_product.getEndTime())
# setup target bands:
self.ctp_band = cawa_product.addBand('ctp', snappy.ProductData.TYPE_FLOAT32)
self.ctp_band.setNoDataValue(CTP_NODATA_VALUE)
self.ctp_band.setNoDataValueUsed(True)
self.ctp_band.setUnit('hPa')
self.ctp_band.setDescription('Cloud Top Pressure')
self.ctp_flags_band = cawa_product.addBand('ctp_flags', snappy.ProductData.TYPE_UINT8)
self.ctp_flags_band.setUnit('dl')
self.ctp_flags_band.setDescription('CTP flags band')
# copy flag bands, tie points, geocoding:
snappy.ProductUtils.copyFlagBands(source_product, cawa_product, True)
source_product.transferGeoCodingTo(cawa_product, None)
with Dataset(str_coeffs_lut,'r') as stray_ncds:
#get the full stray coeffs
self.str_coeffs=np.array(stray_ncds.variables['STRAY'][:],order='F')
self.lmd=np.array(stray_ncds.variables['LAMBDA'][:],order='F')
# doy = 363
datestring = cu.CawaUtils.get_meris_rr_product_datestring(source_product.getName())
print('datestring: ' + datestring + '\n')
doy = int(cu.CawaUtils.get_doy_from_yyyymmdd(datestring))
print('doy: ' + str(doy) + '\n')
with Dataset(ws_alb_lut,'r') as wsalb_ncds:
#get the full stray coeffs
#get closest day of year
doy_idx=np.abs(wsalb_ncds.variables['time'][:]-doy).argmin()
self.alb = wsalb_ncds.variables['albedo'][doy_idx,:,:]
# spectral_fluxes_input_path = os.path.join(resource_root, 'luts/meris_sun_spectral_flux_rr_10_11.txt')
spectral_fluxes_table = cu.CawaUtils.get_table_from_csvfile(spectral_fluxes_input_path, ',', "")
self.spectral_flux_10 = np.array(spectral_fluxes_table['E0_band10'])
self.spectral_flux_11 = np.array(spectral_fluxes_table['E0_band11'])
operator.setTargetProduct(cawa_product)
f.write('end initialize.')
f.close()
def __init__(self, corefile=os.path.join('luts',
'wadamo_core_meris.json')):
'''
dgfra
'''
self.cawa_utils = cu.CawaUtils()
self.num_computations = 0
# 0 read LUT.
#self.lut = d2j.json2dict(corefile,order='F')
self.lut = memcached_j2d(corefile, order='F')
self.wb = self.lut['win_bnd'].tolist()
self.ab = self.lut['abs_bnd'].tolist()
self.fb = self.lut['fig_bnd'].tolist()
self.prf = self.lut['ckd'][self.ab[0]].keys()
self.prs = self.lut['ckd'][self.ab[0]][self.prf[0]]['prs']
self.min_prs = self.prs.min()
self.max_prs = self.prs.max()
# self.about_me=about_me()
self.nle = self.prs.shape[0]
print('1. interpolators for 1d-dimensions ...')
# 1. interpolators for 1d-dimensions
t1 = time.clock() * 1000
def make_1d_interpolators(xx):
out = {}
for cha in xx:
out[cha] = {}
for dim in xx[cha]['dimensions']:
out[cha][dim] = generate_interpol_to_index(xx[cha][dim])
return out
self.intp = {}
for lut in ('atc', 'scf_low', 'scf_hig'):
self.intp[lut] = make_1d_interpolators(self.lut[lut])
self.intp['prs'] = generate_interpol_to_index(self.prs)
t2 = time.clock() * 1000
print('init_1: ', (t2 - t1))
# 2. All watervapor-to-transmission operators
t1 = time.clock() * 1000
def make_transmission_operators(ch, poly=False):
out = {}
for iprf in self.prf:
dam = []
for iprs in range(self.nle):
if poly:
par = np.array(self.lut['ckd'][ch][iprf]
['wvc2trn_coeff'][iprs, :],
order='F')
dam.append(lambda x, a, par=par: wrap_wvc2trn_pol1(
x * a, par))
else:
nor = self.lut['ckd'][ch][iprf]['tcwv'][iprs]
tau = self.lut['ckd'][ch][iprf]['tau'][0:iprs +
1, :].sum(
axis=0)
wgt = self.lut['ckd'][ch][iprf]['wgt']
dam.append(lambda x, a, nor=nor, tau=tau, wgt=wgt: (
(np.exp(-tau * x * a / nor) * wgt).sum()).clip(
0., 1.))
out[iprf] = dam
return out
self.single_tro_poly = {
cha: make_transmission_operators(cha, poly=True)
for cha in self.wb + self.ab
}
self.single_tro_kdis = {
cha: make_transmission_operators(cha)
for cha in self.wb + self.ab
}
t2 = time.clock() * 1000
print('init_2: ', (t2 - t1))
# 3. window bands to absorption bands interpolators
# assuming that I always have two window bands
t1 = time.clock() * 1000
def abs_b_int(ch):
www= (self.lut['cha'][ch]['cwvl'] - self.lut['cha'][self.wb[0]]['cwvl']) \
/ (self.lut['cha'][self.wb[1]]['cwvl'] - self.lut['cha'][self.wb[0]]['cwvl'])
return lambda x, www=www: x[0] + (x[1] - x[0]) * www
self.win_to_abs_interpolator = {cha: abs_b_int(cha) for cha in self.ab}
t2 = time.clock() * 1000
print('init_3: ', (t2 - t1))
# 4. interpolators (6d) for scattering factor derivatives
t1 = time.clock() * 1000
self.d_sf = {
lut: {
what: {
ch:
generate_interpol_derivatives(self.lut[lut][ch][what], [
self.lut[lut][ch][dd]
for dd in self.lut[lut][ch]['dimensions']
])
for ch in self.ab
}
for what in ('d_sf__dwvc', 'd_sf__daot', 'd_sf__dalb')
}
for lut in ('scf_low', 'scf_hig')
}
t2 = time.clock() * 1000
print('init_4: ', (t2 - t1))
def initialize(self, operator):
source_product = operator.getSourceProduct('sourceProduct')
if not source_product:
raise RuntimeError(
'No source product specified or product not found - cannot continue.'
)
print('start initialize: source product is',
source_product.getFileLocation().getAbsolutePath())
# pixel classification from Idepix:
classif_product = operator.getSourceProduct('classifProduct')
if not classif_product:
raise RuntimeError(
'No pixel classification product specified or product not found - cannot continue.'
)
print('Python module location: ' + __file__)
resource_root = os.path.dirname(__file__)
print('Python module location parent: ' + resource_root)
print('get parameters ...')
self.temperature = operator.getParameter('temperature')
self.pressure = operator.getParameter('pressure')
self.aot13 = operator.getParameter('aot_13')
self.aot14 = operator.getParameter('aot_14')
self.aot15 = operator.getParameter('aot_15')
self.sig_aot13 = self.aot13
self.sig_aot14 = self.aot14
self.sig_aot15 = self.aot15
self.cawa_utils = cu.CawaUtils()
with zipfile.ZipFile(resource_root) as zf:
auxpath = SystemUtils.getAuxDataPath()
print('auxpath: ' + str(auxpath))
lut_json = zf.extract('luts/wadamo_core_meris.json',
os.path.join(str(auxpath), 'cawa'))
self.cawa = cawa_core.cawa_core(lut_json)
print('LUT json file: ' + lut_json)
width = source_product.getSceneRasterWidth()
height = source_product.getSceneRasterHeight()
print('width, height = ...', width, height)
print('get bands ...')
# todo: add something similar for MODIS input
self.rhoToa13Band = self.get_band(source_product, 'reflec_13')
self.rhoToa14Band = self.get_band(source_product, 'reflec_14')
self.rhoToa15Band = self.get_band(source_product, 'reflec_15')
self.szaBand = self.get_band(source_product, 'sun_zenith')
print('got band vza ...')
self.vzaBand = self.get_band(source_product, 'view_zenith')
self.vaaBand = self.get_band(source_product, 'view_azimuth')
self.l1_flag_band = self.get_band(source_product, 'l1_flags')
self.classif_band = self.get_band(classif_product,
'cloud_classif_flags')
print('setup target product...')
cawa_product = snappy.Product('pyCAWA', 'CAWA TCWV', width, height)
cawa_product.setDescription('CAWA TCWV product')
cawa_product.setStartTime(source_product.getStartTime())
cawa_product.setEndTime(source_product.getEndTime())
# cawa_product.setPreferredTileSize(width, 16)
# cawa_product.setPreferredTileSize(width, height) # todo: wadamo_core does not yet support multi-threading with smaller tiles
self.tcwvBand = cawa_product.addBand('tcwv',
snappy.ProductData.TYPE_FLOAT32)
self.tcwvBand.setNoDataValue(TCWV_NODATA_VALUE)
self.tcwvBand.setNoDataValueUsed(True)
self.tcwvBand.setUnit('mm')
self.tcwvBand.setDescription('Total column of water vapour')
# todo: flag band
self.tcwvFlagsBand = cawa_product.addBand(
'tcwv_flags', snappy.ProductData.TYPE_UINT8)
self.tcwvFlagsBand.setUnit('dl')
self.tcwvFlagsBand.setDescription('TCWV flags band')
lat_ac_band = self.copy_src_band(source_product, cawa_product,
'corr_latitude')
lat_ac_band.setNoDataValue(LAT_NODATA_VALUE)
lat_ac_band.setNoDataValueUsed(True)
lon_ac_band = self.copy_src_band(source_product, cawa_product,
'corr_longitude')
lat_ac_band.setNoDataValue(LON_NODATA_VALUE)
lon_ac_band.setNoDataValueUsed(True)
sza_ac_band = self.copy_src_band(source_product, cawa_product,
'sun_zenith')
vza_ac_band = self.copy_src_band(source_product, cawa_product,
'view_zenith')
vaa_ac_band = self.copy_src_band(source_product, cawa_product,
'sun_azimuth')
vaa_ac_band = self.copy_src_band(source_product, cawa_product,
'view_azimuth')
altitude_ac_band = self.copy_src_band(source_product, cawa_product,
'altitude')
snappy.ProductUtils.copyFlagBands(source_product, cawa_product, True)
snappy.ProductUtils.copyFlagBands(classif_product, cawa_product, True)
# copy geocoding:
source_product.transferGeoCodingTo(cawa_product, None)
print('set target product...')
operator.setTargetProduct(cawa_product)
print('end initialize.')
def __init__(self,
cloud_lut=os.path.join('.', 'luts', 'cloud_core_meris.nc4')):
self.cawa_ctp = cawa_ctp.CawaCtpCore(cloud_lut)
self.cawa_utils = cu.CawaUtils()