def generic_addvar_reader(fname, vname):
'''
Reads the time stack of additional variables.
Parameters
----------
fname : str
filename of data file
vname : str
variable name
(variable should be contained in file)
Returns
--------
dset : dict
dataset dictionary containing georef and add data.
'''
dset = ncio.read_icon_4d_data(fname, [
vname,
], itime=None)
geo = ncio.read_icon_georef(fname)
geo['x'], geo['y'] = gi.ll2xyc(geo['lon'], geo['lat'])
dset.update(geo)
return dset
def xy_grid_for_icondata(dset):
'''
Calculates an x-y grid (rotated sinosoidal projection) for
simulation data stored on regular lon-lat grid.
Parameters
----------
dset : xr.Dataset
dataset containing lon / lat vectors
Returns
-------
d : xr.Dataset
copy of input data with added x, y grid
'''
d = dset.copy()
olon = xr.ones_like(d.lon)
olat = xr.ones_like(d.lat)
d['lat2d'] = d.lat * olon
d['lon2d'] = olat * d.lon
x, y = gi.ll2xyc(d['lon2d'], d['lat2d'])
d['x'] = x
d['y'] = y
return d
def read_hdcp2_data(fname):
'''
Reads BT10.8 data from files generated by the HDCP2 O module.
Parameters
----------
fname : str
file name
Returns
--------
dset : dict
dictionary of datasets
'''
# data fields ----------------------------------------------------
vlist = ['tb108', 'lon', 'lat', 'time']
dset = ncio.read_icon_4d_data(fname, vlist, itime=None)
b3d = dset.pop('tb108')
b3d = np.ma.masked_less(b3d, 100.)
# ================================================================
# geo ref --------------------------------------------------------
lon, lat = dset['lon'], dset['lat']
x, y = gi.ll2xyc(lon, lat, lon0=10, lat0=50)
area = np.abs(gi.simple_pixel_area(lon, lat))
# ================================================================
# time conversions -----------------------------------------------
abs_time = dset['time'] / (3600. * 24)
rel_time = np.mod(abs_time, 1) * 24.
ntime = len(rel_time)
index_time = np.arange(ntime)
# ================================================================
# prepare output .................................................
vnames = ['x', 'y', 'area', 'rel_time', 'abs_time', 'index_time']
vvec = [x, y, area, rel_time, abs_time, index_time]
for i, vname in enumerate(vnames):
dset[vname] = vvec[i]
dset['bt108'] = b3d
dset['lsm'] = np.ones_like(x)
dset['input_dir'] = os.path.dirname(fname)
# ================================================================
return dset
def read_icon_lem_data(fname):
'''
Reads BT10.8 data from files generated for ICON-LEM runs.
Parameters
----------
fname : str
file name
Returns
--------
dset : dict
dictionary of datasets
'''
# data fields ----------------------------------------------------
vlist = ['bt108', 'lon', 'lat', 'time']
dset = ncio.read_icon_4d_data(fname, vlist, itime=None)
b3d = dset.pop('bt108')
b3d = np.ma.masked_less(b3d, 100.)
# ================================================================
# geo ref --------------------------------------------------------
lon, lat = dset['lon'], dset['lat']
x, y = gi.ll2xyc(lon, lat, lon0=10, lat0=50)
area = np.abs(gi.simple_pixel_area(lon, lat))
# ================================================================
# time conversions -----------------------------------------------
rel_time = 24 * (dset['time'] - dset['time'][0])
ntime = len(rel_time)
index_time = np.arange(ntime)
t0 = datetime.datetime(1970, 1, 1)
abs_time = []
for t in dset['time']:
day = str(int(t))
subday = np.mod(t, 1)
tobj = datetime.datetime.strptime(day, '%Y%m%d')
tobj += datetime.timedelta(days=subday)
dt = (tobj - t0).total_seconds()
abs_time.append(dt / (24. * 3600.))
abs_time = np.array(abs_time)
# ================================================================
# prepare output .................................................
vnames = ['x', 'y', 'area', 'rel_time', 'abs_time', 'index_time']
vvec = [x, y, area, rel_time, abs_time, index_time]
for i, vname in enumerate(vnames):
dset[vname] = vvec[i]
dset['bt108'] = b3d
dset['lsm'] = np.ones_like(x)
dset['input_dir'] = os.path.dirname(fname)
# ================================================================
return dset
def read_narval_addvars(fname, vname, domain_center=None, region_slice=None):
'''
Reads the time stack of Narval data, either meteoat or synsat.
Parameters
----------
fname : str
filename of data file
vname : str
variable name
(variable should be contained in file)
domain_center : tuple of floats, optional, default = None
setting the projection center to (clon, clat)
if None: not used
region_slice : tuple of floats, optional, default = None
cutout of fields for form ((irow1, irow2), (icol1, icol2))
if None: not used
Returns
--------
dset : dict
dataset dictionary containing georef and bt108 data.
'''
# read land sea data ---------------------------------------------
narval_dir = '%s/icon/narval' % local_data_path
lsm_name = '%s/aux/narval_landsea_coast_mask.h5' % narval_dir
print '... read land-sea-mask from %s' % lsm_name
dset = hio.read_dict_from_hdf(lsm_name)
lsm = dset['mask50']
# ================================================================
# read bt108 -----------------------------------------------------
print '... read %s from %s' % (vname, fname)
basename, file_ext = os.path.splitext(os.path.basename(fname))
date = basename.split('_')[-1]
t0 = datetime.datetime.strptime(date, '%Y%m%d')
b3d = ncio.read_icon_4d_data(fname, [
vname,
], itime=None)[vname]
b3d = np.ma.masked_invalid(b3d)
ntime, nrow, ncol = b3d.shape
# ================================================================
# prepare time vector --------------------------------------------
rel_time = np.arange(1, ntime + 1)
index_time = np.arange(ntime)
day_shift = t0 - datetime.datetime(1970, 1, 1)
day_shift = day_shift.total_seconds() / (24. * 3600)
abs_time = day_shift + rel_time / 24.
# ================================================================
# get georef .....................................................
gfile = '%s/aux/target_grid_geo_reference_narval.h5' % narval_dir
geo = hio.read_dict_from_hdf(gfile)
lon, lat = geo['lon'], geo['lat']
if domain_center is not None:
mlon, mlat = domain_center
else:
mlon, mlat = None, None
x, y = gi.ll2xyc(lon, lat, mlon=mlon, mlat=mlat)
area = np.abs(gi.simple_pixel_area(x, y, xy=True))
# ================================================================
# prepare output .................................................
dset = {}
dset[vname] = b3d
addnames = [
'x', 'y', 'lon', 'lat', 'lsm', 'area', 'rel_time', 'abs_time',
'index_time'
]
vvec = [x, y, lon, lat, lsm, area, rel_time, abs_time, index_time]
for i, aname in enumerate(addnames):
dset[aname] = vvec[i]
dset['input_dir'] = os.path.dirname(fname)
# ================================================================
# do cutout if wanted --------------------------------------------
field_names = ['x', 'y', 'lon', 'lat', 'lsm', 'area', vname]
if region_slice is not None:
for name in field_names:
dset[name] = gi.cutout_fields(dset[name], region_slice, vaxis=0)
# ================================================================
return dset
def read_narval_data(fname):
'''
Reads the time stack of Narval data, either meteoat or synsat.
Parameters
----------
fname : str
filename of data file
Returns
--------
dset : dict
dataset dictionary containing georef and bt108 data.
'''
# read land sea data ---------------------------------------------
narval_dir = '%s/icon/narval' % local_data_path
lsm_name = '%s/aux/narval_landsea_coast_mask.h5' % narval_dir
print '... read land-sea-mask from %s' % lsm_name
dset = hio.read_dict_from_hdf(lsm_name)
lsm = dset['mask50']
# ================================================================
# read bt108 -----------------------------------------------------
print '... read BT10.8 from %s' % fname
basename, file_ext = os.path.splitext(os.path.basename(fname))
date = basename.split('_')[-1]
t0 = datetime.datetime.strptime(date, '%Y%m%d')
# check if its is obs or sim?
ftype = basename.split('_')[0]
if ftype in ['msevi', 'trans']:
subpath = None
elif ftype == 'synsat':
subpath = 'synsat_oper'
# read bt108 from hdf
if file_ext == '.h5':
b3d = hio.read_var_from_hdf(fname, 'IR_108', subpath=subpath) / 100.
elif file_ext == '.nc':
vname = 'bt108'
b3d = ncio.read_icon_4d_data(fname, vname, itime=None)[vname]
b3d = np.ma.masked_invalid(b3d)
b3d = np.ma.masked_less(b3d, 100.)
ntime, nrow, ncol = b3d.shape
# ================================================================
# prepare time vector --------------------------------------------
rel_time = np.arange(1, ntime + 1)
index_time = np.arange(ntime)
day_shift = t0 - datetime.datetime(1970, 1, 1)
day_shift = day_shift.total_seconds() / (24. * 3600)
abs_time = day_shift + rel_time / 24.
# ================================================================
# get georef .....................................................
gfile = '%s/aux/target_grid_geo_reference_narval.h5' % narval_dir
geo = hio.read_dict_from_hdf(gfile)
lon, lat = geo['lon'], geo['lat']
# centered sinusoidal
x, y = gi.ll2xyc(lon, lat)
area = np.abs(gi.simple_pixel_area(lon, lat))
# ================================================================
# prepare output .................................................
dset = {}
vnames = [
'x', 'y', 'lon', 'lat', 'lsm', 'area', 'rel_time', 'abs_time',
'index_time'
]
vvec = [x, y, lon, lat, lsm, area, rel_time, abs_time, index_time]
for i, vname in enumerate(vnames):
dset[vname] = vvec[i]
dset['bt108'] = b3d
dset['input_dir'] = narval_dir
# ================================================================
return dset
def collect_data4cre_sim(radname, itime):
'''
Collects a set of simulated data fields for cloud-radiative effect analysis.
Parameters
----------
radname : str
name of toa allsky radiation file
itime : int
time index of data fields ('swf_net' and 'lwf') in radname
Returns
--------
dset : dict
dataset dict containing swf, lwf and ct fields
'''
# set filenames
clearname = radname.replace('toa_', 'toa_clear_')
ctname = radname2ctname(radname, datatype='sim')
# read radiation data for allsky
dset = {}
for vname in ['lwf', 'swf_net', 'swf_up']:
radset = read_data_field(radname, itime, vname, region='atlantic')
dset[vname] = radset[vname]
dset['swf_down'] = dset['swf_net'] - dset['swf_up']
# read radiation data for clearsky
for vname in ['lwf', 'swf_net']:
clearset = read_data_field(clearname, itime, vname, region='atlantic')
dset['%s_clear' % vname] = clearset[vname]
# calculate SWF up (clearsky) from allsky downwelling (downwelling is the same...)
dset['swf_up_clear'] = dset['swf_net_clear'] - dset['swf_down']
ctset = read_data_field(ctname,
radset['time_obj'],
'CT',
region='atlantic')
dset.update(ctset)
# select region mask
region_mask = dset['mask']
# possible extension (get away from coast)
nedge = 11
region_mask = scipy.ndimage.minimum_filter(region_mask, nedge)
mlon = dset['lon'][region_mask].mean()
mlat = dset['lat'][region_mask].mean()
x, y = gi.ll2xyc(dset['lon'], dset['lat'], mlon=mlon, mlat=mlat)
a = gi.simple_pixel_area(x, y, xy=True)
# update mask and area
dset['mask'] = region_mask
dset['area'] = a
return dset
def collect_data4cre_obs(radname,
itime,
filepart='-scaled',
lwf_clear_offset=-2.):
'''
Collects a set of observed data fields for cloud-radiative effect analysis.
Parameters
----------
radname : str
name of toa allsky radiation file
itime : int
time index of data fields ('swf_net' and 'lwf') in radname
filepart : str, optional, default = '-scaled'
part in the file that gives information about scaling of clear-sky fields
either '-scaled' or '-not_scaled'
lwf_clear_offset : float, optional, default = 2.
due to the bias in the simulated LWF, we might use an predefined offset
to correct this issue
i.e. LWF_clear_simulated += lwf_clear_offset
Returns
--------
dset : dict
dataset dict containing swf, lwf and ct fields
'''
# set filenames
# ==============
clearname = radname.replace('toa_', 'toa_clear_')
ctname = radname2ctname(radname, datatype='obs')
# read allsky data
# =================
dset = {}
for vname in ['lwf', 'swf_net', 'swf_up']:
radset = read_data_field(radname, itime, vname, region='atlantic')
dset[vname] = radset[vname]
dset['swf_down'] = dset['swf_net'] - dset['swf_up']
# find the right short-wave clear file
# ===================================
tobj = radset['time_obj']
filemap = selector.make_filetime_index(
'swf_net',
tobj,
filepart=filepart,
subdirs=['retrieved_clearsky_netswf'])
# print filemap
# input swf clear
# ===============
clearname = filemap[tobj][0]
clearset = read_data_field(clearname, tobj, 'swf_net', region='atlantic')
dset['swf_net_clear'] = clearset['swf_net']
dset['swf_up_clear'] = dset['swf_net_clear'] - dset['swf_down']
# long-wave filename
# ====================
lwfclearname = clearname.replace(
'retrieved_clearsky_netswf/clearsky_netswf-',
'sim-toarad/toa_clear_radflux-')
lwfclearname = lwfclearname.replace(filepart, '')
print((radname, clearname, lwfclearname))
# input lwf clear data
# ====================
if filepart == '-not_scaled':
lwf_clear_offset = 0
lwfclearset = read_data_field(lwfclearname, tobj, 'lwf', region='atlantic')
dset['lwf_clear'] = lwfclearset['lwf'] + lwf_clear_offset
# input cloud type
# ====================
ctset = read_data_field(ctname, tobj, 'CT', region='atlantic')
dset.update(ctset)
# select and modify region mask
# ==============================
region_mask = dset['mask']
# possible extension (get away from coast)
nedge = 11
region_mask = scipy.ndimage.minimum_filter(region_mask, nedge)
# finally prepare georef
# =======================
mlon = dset['lon'][region_mask].mean()
mlat = dset['lat'][region_mask].mean()
x, y = gi.ll2xyc(dset['lon'], dset['lat'], mlon=mlon, mlat=mlat)
a = gi.simple_pixel_area(x, y, xy=True)
# update mask and area
dset['mask'] = region_mask
dset['area'] = a
return dset