def read_dardar_cer(ifile, dataset):
imager = {}
dardar = {}
aux = {}
with h5py.File(ifile, 'r') as h5file:
# imager variables
imager['cer'] = h5file[dataset.lower()]['cpp_cer'][:]
imager['phase'] = get_imager_cph(h5file[dataset.lower()])
if dataset.lower() == 'cci':
imager['quality'] = h5file[
dataset.lower()]['cpp_quality'][:].astype(int)
# truth variables
dardar['cer'] = h5file['dardar']['effective_radius'][:] * 1E6
dardar['lat'] = h5file['dardar']['latitude'][:]
dardar['lon'] = h5file['dardar']['longitude'][:]
dardar['img_linnum_nneigh'] = h5file['dardar'][
'imager_linnum_nneigh'][:]
# aux variables
aux['sunz'] = h5file[dataset.lower()]['sunz'][:]
aux['satz'] = h5file[dataset.lower()]['satz'][:]
return _process_dardar_n_neighbours(imager, dardar, aux, 'cer', dataset)
def read_dardar_iwp(ifile, dataset):
imager = {}
dardar = {}
aux = {}
with h5py.File(ifile, 'r') as h5file:
# imager variables
imager['iwp'] = h5file[dataset.lower()]['cpp_iwp'][:]
imager['phase'] = get_imager_cph(h5file[dataset.lower()])
if dataset.lower() == 'cci':
imager['quality'] = h5file[
dataset.lower()]['cpp_quality'][:].astype(int)
# truth variables
dardar['iwp'] = h5file['dardar']['iwc'][:] / 1E2
dardar['lat'] = h5file['dardar']['latitude'][:]
dardar['lon'] = h5file['dardar']['longitude'][:]
dardar['mask'] = h5file['dardar'][
'DARMASK_Simplified_Categorization'][:]
dardar['img_linnum_nneigh'] = h5file['dardar'][
'imager_linnum_nneigh'][:]
# aux variables
aux['sunz'] = h5file[dataset.lower()]['sunz'][:]
aux['satz'] = h5file[dataset.lower()]['satz'][:]
return _process_dardar_n_neighbours(imager, dardar, aux, 'iwp', dataset)
def read_amsr_lwp(ifile, dataset):
amsr_key = 'amsr'
cci_key = dataset.lower()
with h5py.File(ifile, 'r') as h5file:
lwp_amsr = h5file[amsr_key]['lwp'][:]
lwp_cci = h5file[cci_key]['cpp_lwp'][:]
lat_amsr = h5file[amsr_key]['latitude'][:]
lon_amsr = h5file[amsr_key]['longitude'][:]
cph = get_imager_cph(h5file[cci_key])
lsm = h5file[cci_key]['fractionofland'][:]
sunz = h5file[cci_key]['sunz'][:]
satz = h5file[cci_key]['satz'][:]
img_linnum_nneigh = h5file[amsr_key]['imager_linnum_nneigh'][:]
qual_cci = h5file[cci_key]['cpp_quality'][:].astype(int)
cot = h5file[cci_key]['cpp_cot'][:]
# contains pixel_status, quality and surface_type if AMSR2 NSIDC
if 'pixel_status' in h5file[amsr_key].keys() and \
'quality' in h5file[amsr_key].keys() and \
'surface_type' in h5file[amsr_key].keys():
amsr_data_source = 'NSIDC'
quality_amsr = h5file[amsr_key]['quality'][:]
pixel_status_amsr = h5file[amsr_key]['pixel_status'][:]
surface_type_amsr = h5file[amsr_key]['surface_type'][:]
else:
amsr_data_source = 'JAXA'
N_NEIGHBOURS = lwp_cci.shape[-1] # number of neighbours
VALID_NEIGHBOURS_PERC_MAX = 0.50 # max percentage of invalid neighbours
# quality checking using cci qcflag
#bin(3) = 11000000, bit0 = Retrieval did not converge, bit1= Rettrieval cost > 100.
bad_quality_img = np.bitwise_and(qual_cci, 3) > 0
bad_quality_img = np.where(img_linnum_nneigh < 0, np.nan, bad_quality_img)
bad_quality_cnt_img = np.nansum(bad_quality_img, axis=-1)
use_quality_img = bad_quality_cnt_img < 1
# AMSR quality checking only available for NSIDC collocations
if amsr_data_source == 'NSIDC':
# only use amsr pixels where Quality Flag is 0
use_quality_amsr = quality_amsr == 0
# only use amsr pixels where pixel status is 0
use_quality_amsr = np.logical_and(use_quality_amsr,
pixel_status_amsr == 0)
# only use amsr pixels where surface type is ocean (1)
use_quality_amsr = np.logical_and(use_quality_amsr,
surface_type_amsr == 1)
# set clearsky pixels to 0
lwp_cci = np.where(np.isnan(cot), 0, lwp_cci)
# mask pixels within footprint where no neighbour is found
lwp_cci = np.where(img_linnum_nneigh <= 0, np.nan, lwp_cci)
# average imager LWP footprints
lwp_cci_mean = np.nanmean(lwp_cci, axis=-1)
# mask footprints where imager cfc < 0.5
#cma = np.where(cot > 0, 1, 0)
#cma = np.where(img_linnum_nneigh < 0, np.nan, cma)
#cfc = np.nanmean(cma, axis=-1)
#use_cfc = cfc > 0.2
# calculate ice cloud fraction (ICF) of footprint
cph = np.where(img_linnum_nneigh <= 0, np.nan, cph)
icf = np.nanmean(cph, axis=-1)
# mask footprints where ICF >= 10%
use_icf = icf < 0.1
# calculate landfraction of footprints
lsm = np.where(img_linnum_nneigh <= 0, np.nan, lsm)
landfrac = np.nanmean(lsm, axis=-1)
# mask pixels where land fraction > 0%
use_landfrac = landfrac < 0.00005
# mask night and twilight
use_sunz = sunz < 75
# mask high zenith angles
use_satz = satz < 70
# minimum number of found neighbours limit
mask_noneigh = img_linnum_nneigh < 0
use_nvalid_neighbours = np.sum(
mask_noneigh, axis=-1) / N_NEIGHBOURS < VALID_NEIGHBOURS_PERC_MAX
# select valid values
use_valid_values_amsr = np.logical_and(lwp_amsr >= 0, lwp_amsr < 170)
use_valid_values_cci = np.logical_and(lwp_cci_mean >= 0,
lwp_cci_mean < 500)
use_valid_values = np.logical_and(use_valid_values_amsr,
use_valid_values_cci)
selection = np.logical_and(use_icf, use_landfrac)
selection = np.logical_and(selection, use_sunz)
selection = np.logical_and(selection, use_satz)
selection = np.logical_and(selection, use_valid_values)
selection = np.logical_and(selection, use_nvalid_neighbours)
selection = np.logical_and(selection, use_quality_img)
#selection = np.logical_and(selection, use_cfc)
if amsr_data_source == 'NSIDC':
selection = np.logical_and(selection, use_quality_amsr)
return lwp_amsr[selection], lwp_cci_mean[selection], lat_amsr[
selection], lon_amsr[selection]
def _get_calipso_matchup_file_content(ipath,
chunksize,
dnt='ALL',
satz_lim=None,
dataset='CCI'):
""" Obtain variables from atrain_match HDF5 matchup file. """
file = h5py.File(ipath, 'r')
caliop = file['calipso']
if dataset == 'CCI':
imager = file['cci']
elif dataset == 'CLAAS':
imager = file['pps']
else:
raise Exception('Dataset {} not known!'.format(dataset))
# get CTH, CTT, CTP
sev_cth = da.from_array(gi.get_imager_cth(imager), chunks=chunksize)
cal_cth = da.from_array(gc.get_caliop_cth(caliop), chunks=chunksize)
sev_ctt = da.from_array(gi.get_imager_ctt(imager), chunks=chunksize)
cal_ctt = da.from_array(gc.get_caliop_ctt(caliop), chunks=chunksize)
cal_cflag = np.array(caliop['feature_classification_flags'][::, 0])
if filter_stratospheric:
tropo_height = da.from_array(gc.get_tropopause_height(caliop),
chunks=chunksize)
sev_ctp = da.from_array(gi.get_imager_ctp(imager), chunks=(chunksize))
cal_ctp = da.from_array(gc.get_caliop_ctp(caliop), chunks=(chunksize))
if dataset == 'CCI':
sev_quality = da.from_array(imager['cpp_quality'][:].astype(int),
chunks=chunksize)
bad_quality_imager = np.bitwise_and(sev_quality, 3) > 0
sev_cth[bad_quality_imager] = np.nan
sev_ctt[bad_quality_imager] = np.nan
# get CMA, CPH, VZA, SZA, LAT and LON
sev_cph = da.from_array(gi.get_imager_cph(imager), chunks=chunksize)
cal_cph = da.from_array(gc.get_caliop_cph(caliop), chunks=chunksize)
cal_cma = da.from_array(gc.get_caliop_cma(caliop), chunks=chunksize)
sev_cma = da.from_array(gi.get_imager_cma(imager), chunks=chunksize)
satz = da.from_array(imager['satz'], chunks=chunksize)
sunz = da.from_array(imager['sunz'], chunks=chunksize)
lat = da.from_array(imager['latitude'], chunks=chunksize)
lon = da.from_array(imager['longitude'], chunks=chunksize)
# mask fill values for angles
satz = np.where(satz == -9, np.nan, satz)
sunz = np.where(sunz == -9, np.nan, sunz)
# mask satellize zenith angle
if satz_lim is not None:
mask = satz > satz_lim
cal_cma = da.where(mask, np.nan, cal_cma)
sev_cma = da.where(mask, np.nan, sev_cma)
cal_cph = da.where(mask, np.nan, cal_cph)
sev_cph = da.where(mask, np.nan, sev_cph)
cal_cth = da.where(mask, np.nan, cal_cth)
sev_cth = da.where(mask, np.nan, sev_cth)
cal_ctt = da.where(mask, np.nan, cal_ctt)
sev_ctt = da.where(mask, np.nan, sev_ctt)
cal_ctp = da.where(mask, np.nan, cal_ctp)
sev_ctp = da.where(mask, np.nan, sev_ctp)
# mask all pixels except daytime
if dnt == 'DAY':
mask = sunz >= 80
# mask all pixels except nighttime
elif dnt == 'NIGHT':
mask = sunz <= 95
# mask all pixels except twilight
elif dnt == 'TWILIGHT':
mask = ~da.logical_and(sunz > 80, sunz < 95)
# no masking
elif dnt == 'ALL':
mask = None
else:
raise Exception('DNT option ', dnt, ' is invalid.')
# filter stratospheric clouds in CALIPSO
if filter_stratospheric:
strato_mask = cal_cth > tropo_height
cal_cma = da.where(strato_mask, np.nan, cal_cma)
cal_cph = da.where(strato_mask, np.nan, cal_cph)
cal_cth = da.where(strato_mask, np.nan, cal_cth)
cal_ctt = da.where(strato_mask, np.nan, cal_ctt)
cal_ctp = da.where(strato_mask, np.nan, cal_ctp)
# apply DNT masking
masked = _apply_dnt_mask(cal_cma, sev_cma, cal_cph, sev_cph, cal_cth,
sev_cth, cal_ctt, sev_ctt, cal_ctp, sev_ctp, mask)
data = {
'caliop_cma': masked[0],
'imager_cma': masked[1],
'caliop_cph': masked[2],
'imager_cph': masked[3],
'satz': satz,
'sunz': sunz,
'caliop_cth': masked[4],
'imager_cth': masked[5],
'caliop_ctt': masked[6],
'imager_ctt': masked[7],
'caliop_ctp': masked[8],
'imager_ctp': masked[9],
'caliop_cflag': cal_cflag,
}
latlon = {'lat': lat, 'lon': lon}
return data, latlon