def generic_addvar_reader(fname, vname):
'''
Reads the time stack of additional variables.
Parameters
----------
fname : str
filename of data file
vname : str
variable name
(variable should be contained in file)
Returns
--------
dset : dict
dataset dictionary containing georef and add data.
'''
dset = ncio.read_icon_4d_data(fname, [
vname,
], itime=None)
geo = ncio.read_icon_georef(fname)
geo['x'], geo['y'] = gi.ll2xyc(geo['lon'], geo['lat'])
dset.update(geo)
return dset
def read_iconvar_vector( fname, vlist ):
'''
Input of ICON variable vector (given at original ICON grid and with applied
zenith angle mask).
Parameters
----------
fname : str
name of ICON file (should be netcdf file)
vname : list of str
list of variable names
Returns
-------
outset : dict of numpy arrays
set of synsat and georef vectors
'''
# check if vlist is list
# ====================
if not type(vlist) == type([]):
vlist = [vlist,]
# read variables
# =============================
print(('... read data from ', fname))
dset = ncio.read_icon_4d_data(fname, vlist, itime = None)
# read georeference
# ==================
subdir = subdir_from_fname( fname )
geo = read_georef( subdir, mask_with_zen = False )
# get zenith angle mask
# =====================
mask = get_zen_mask( subdir, geo = geo )
# prepare dict output
# ====================
outset = dset.copy()
outset.update( geo )
for vname in list(outset.keys()):
outset[vname] = outset[vname].squeeze()[mask]
return outset
def read_data_field(fname, time, varname, region='full_region'):
'''
Reads "level2" data for analysis and plotting.
Parameters
----------
fname : str
input data file name
time : int or datetime object
time index OR datetime object for which data is read
varname : str
name of the product read
region : str, optional, default = 'full_region'
region for which mask is input
Returns
--------
dset : dict
dataset dictionary
'''
# read bt variables
# dset = ncio.read_icon_4d_data(fname, [varname], itime = itime)
xset = xr.open_dataset(fname)
# time is given as index
if type(time) == type(10):
itime = time
var = np.ma.masked_invalid(xset.isel(time=itime)[varname].data)
time_obj = ncio.read_icon_time(fname, itime=itime)
# time is given as datetime object
elif type(time) == datetime.datetime:
tfloat = convert_time(time)
var = np.ma.masked_invalid(
xset.sel(time=tfloat, method='nearest')[varname].data)
time_obj = time
dset = {varname: var}
# read geo-ref
geo = ncio.read_icon_4d_data(fname, ['lon', 'lat'], itime=None)
dset.update(geo)
# also get mask
dset.update(read_mask(region=region))
dset['time_obj'] = time_obj
dset['time_str'] = dset['time_obj'].strftime('%Y-%m-%d %H:%M UTC')
return dset
def read_hdcp2_data(fname):
'''
Reads BT10.8 data from files generated by the HDCP2 O module.
Parameters
----------
fname : str
file name
Returns
--------
dset : dict
dictionary of datasets
'''
# data fields ----------------------------------------------------
vlist = ['tb108', 'lon', 'lat', 'time']
dset = ncio.read_icon_4d_data(fname, vlist, itime=None)
b3d = dset.pop('tb108')
b3d = np.ma.masked_less(b3d, 100.)
# ================================================================
# geo ref --------------------------------------------------------
lon, lat = dset['lon'], dset['lat']
x, y = gi.ll2xyc(lon, lat, lon0=10, lat0=50)
area = np.abs(gi.simple_pixel_area(lon, lat))
# ================================================================
# time conversions -----------------------------------------------
abs_time = dset['time'] / (3600. * 24)
rel_time = np.mod(abs_time, 1) * 24.
ntime = len(rel_time)
index_time = np.arange(ntime)
# ================================================================
# prepare output .................................................
vnames = ['x', 'y', 'area', 'rel_time', 'abs_time', 'index_time']
vvec = [x, y, area, rel_time, abs_time, index_time]
for i, vname in enumerate(vnames):
dset[vname] = vvec[i]
dset['bt108'] = b3d
dset['lsm'] = np.ones_like(x)
dset['input_dir'] = os.path.dirname(fname)
# ================================================================
return dset
def read_georef( expname, mask_with_zen = True, zen_max = 75. ):
'''
Reads geo reference of simulation.
Parameters
----------
expname : str
this is the experiment name which should be equal to the subdirectory
it is allowed to also pass the georef filename directly through this agrument
mask_with_zen : bool, optional, default = True
if zen mask should be applied
zen_max : float, optional, default = 75
maximum in satellite zenith angle (if mask_with_zen = True)
Returns
-------
geo : dict of numpy arrays
'''
# get gridfile name
if os.path.isfile( expname ):
gridfile = expname
else:
gridfile = get_grid_filename( expname )
# lon/lat input
geo = ncio.read_icon_4d_data(gridfile, ['clon', 'clat'], itime = None)
# calculate zenith angle
clon, clat = geo['clon'], geo['clat']
clon, clat = np.rad2deg( clon ), np.rad2deg( clat )
geo['lon'], geo['lat'] = clon, clat
geo['azi'], geo['zen'] = lonlat2azizen(clon, clat)
# do masking with satellite zenith angle
if mask_with_zen:
mask = (geo['zen'] <= zen_max)
for vname in list(geo.keys()):
geo[vname] = geo[vname][mask]
return geo
def read_nc(self, time):
'''
Reads Radolan data from HDCP2 project.
'''
# get name of radolan file
fname = radoname_from_time(time,
rproduct = 'rx_hdcp2')
# determine itime from time object
# TODO: SIMPLE METHOD TO BE IMPROVED
itime = 12 * time.hour + time.minute / 5
vname = 'dbz'
d = ncio.read_icon_4d_data(fname, [vname], itime = itime)[vname]
d = np.ma.masked_invalid( d )
self.data = np.ma.masked_greater_equal( d, 92)
def read_cluster(fname, cname='iclust'):
'''
Reads Cluster Data from netcdf files.
Parameters
----------
fname : str
file name
cname : str, optional, default = 'iclust'
name of cluster index variable
Returns
--------
c : numpy array, int
cluster index field
'''
c = ncio.read_icon_4d_data(fname, cname, itime=None)[cname]
return c.astype(np.int)
def read_radiation_flux_tstack(date,
fdir = gerb_like_dir,
georef_file = None,
ntimes = 24,
do_cutout = True):
'''
Reads and scales radiation flux data based on GERB-like SEVIRI retrievals.
Parameters
----------
date : string
date string as %Y%m%d
fdir : str, optional, default = gerb_like_dir
file directory name
georef_file : str, optional, default = None
filename where lon and lat can be found
ntimes : int, optional, default = 24
number of time step included (starting at mid-night)
do_cutout : bool, optional, default = True
if SEVIRI cutout is applied
Returns
-------
dset : dict of numpy arrays, 3dim
set of fields incl. long- and short-wave radiaton
'''
# set time ranges
t1 = datetime.datetime.strptime( date, '%Y%m%d')
t2 = t1 + datetime.timedelta( hours = ntimes - 1 )
dt = datetime.timedelta( hours = 1 )
# stack data in time loop
dset = dict( time = [] )
t = copy.copy(t1)
n = 0
while t <= t2:
lwf, swf_net, swf_up = read_radiation_fluxes(t,
fdir = fdir,
do_cutout = do_cutout)
# initialize -------------------------------------------------
if n == 0:
nrows, ncols = lwf.shape
lwf_stack = np.ma.zeros( (ntimes, nrows, ncols) )
swf_net_stack = np.ma.zeros( (ntimes, nrows, ncols) )
swf_up_stack = np.ma.zeros( (ntimes, nrows, ncols) )
# ============================================================
lwf_stack[n] = lwf[:]
swf_net_stack[n] = swf_net[:]
swf_up_stack[n] = swf_up[:]
dset['time'] += [ copy.copy( t ) ]
t += dt
n += 1
dset['lwf'] = lwf_stack
dset['swf_net'] = swf_net_stack
dset['swf_up'] = swf_up_stack
# ================================================================
# read georef ----------------------------------------------------
if georef_file is None:
# hope that meteosat file is there
georef_file = meteosat_georef_file
georef = ncio.read_icon_4d_data( georef_file, ['lon', 'lat'], itime = None)
dset.update( georef )
# ================================================================
return dset
except:
pvec = np.nan * np.ones(nperc)
pc.append(pvec)
return pc
######################################################################
######################################################################
if __name__ == '__main__':
# input test field
fname = '/vols/talos/home/fabian/data/icon/narval/msevi_narval_DOM01_20160801.nc'
b = ncio.read_icon_4d_data(fname, ['bt108'], itime=None)['bt108']
geo = ncio.read_icon_georef(fname)
itime = 15
ns = 2
da, bperc = get_area_rate(geo['clon'],
geo['clat'],
-b[itime - 1],
-b[itime],
-230,
vmin=-260,
vmax=-180,
cluster_method='watershed_merge',
ctype=4,
filter_method='curve',
marker_field='dist',
def read_icon_lem_data(fname):
'''
Reads BT10.8 data from files generated for ICON-LEM runs.
Parameters
----------
fname : str
file name
Returns
--------
dset : dict
dictionary of datasets
'''
# data fields ----------------------------------------------------
vlist = ['bt108', 'lon', 'lat', 'time']
dset = ncio.read_icon_4d_data(fname, vlist, itime=None)
b3d = dset.pop('bt108')
b3d = np.ma.masked_less(b3d, 100.)
# ================================================================
# geo ref --------------------------------------------------------
lon, lat = dset['lon'], dset['lat']
x, y = gi.ll2xyc(lon, lat, lon0=10, lat0=50)
area = np.abs(gi.simple_pixel_area(lon, lat))
# ================================================================
# time conversions -----------------------------------------------
rel_time = 24 * (dset['time'] - dset['time'][0])
ntime = len(rel_time)
index_time = np.arange(ntime)
t0 = datetime.datetime(1970, 1, 1)
abs_time = []
for t in dset['time']:
day = str(int(t))
subday = np.mod(t, 1)
tobj = datetime.datetime.strptime(day, '%Y%m%d')
tobj += datetime.timedelta(days=subday)
dt = (tobj - t0).total_seconds()
abs_time.append(dt / (24. * 3600.))
abs_time = np.array(abs_time)
# ================================================================
# prepare output .................................................
vnames = ['x', 'y', 'area', 'rel_time', 'abs_time', 'index_time']
vvec = [x, y, area, rel_time, abs_time, index_time]
for i, vname in enumerate(vnames):
dset[vname] = vvec[i]
dset['bt108'] = b3d
dset['lsm'] = np.ones_like(x)
dset['input_dir'] = os.path.dirname(fname)
# ================================================================
return dset
def read_narval_addvars(fname, vname, domain_center=None, region_slice=None):
'''
Reads the time stack of Narval data, either meteoat or synsat.
Parameters
----------
fname : str
filename of data file
vname : str
variable name
(variable should be contained in file)
domain_center : tuple of floats, optional, default = None
setting the projection center to (clon, clat)
if None: not used
region_slice : tuple of floats, optional, default = None
cutout of fields for form ((irow1, irow2), (icol1, icol2))
if None: not used
Returns
--------
dset : dict
dataset dictionary containing georef and bt108 data.
'''
# read land sea data ---------------------------------------------
narval_dir = '%s/icon/narval' % local_data_path
lsm_name = '%s/aux/narval_landsea_coast_mask.h5' % narval_dir
print '... read land-sea-mask from %s' % lsm_name
dset = hio.read_dict_from_hdf(lsm_name)
lsm = dset['mask50']
# ================================================================
# read bt108 -----------------------------------------------------
print '... read %s from %s' % (vname, fname)
basename, file_ext = os.path.splitext(os.path.basename(fname))
date = basename.split('_')[-1]
t0 = datetime.datetime.strptime(date, '%Y%m%d')
b3d = ncio.read_icon_4d_data(fname, [
vname,
], itime=None)[vname]
b3d = np.ma.masked_invalid(b3d)
ntime, nrow, ncol = b3d.shape
# ================================================================
# prepare time vector --------------------------------------------
rel_time = np.arange(1, ntime + 1)
index_time = np.arange(ntime)
day_shift = t0 - datetime.datetime(1970, 1, 1)
day_shift = day_shift.total_seconds() / (24. * 3600)
abs_time = day_shift + rel_time / 24.
# ================================================================
# get georef .....................................................
gfile = '%s/aux/target_grid_geo_reference_narval.h5' % narval_dir
geo = hio.read_dict_from_hdf(gfile)
lon, lat = geo['lon'], geo['lat']
if domain_center is not None:
mlon, mlat = domain_center
else:
mlon, mlat = None, None
x, y = gi.ll2xyc(lon, lat, mlon=mlon, mlat=mlat)
area = np.abs(gi.simple_pixel_area(x, y, xy=True))
# ================================================================
# prepare output .................................................
dset = {}
dset[vname] = b3d
addnames = [
'x', 'y', 'lon', 'lat', 'lsm', 'area', 'rel_time', 'abs_time',
'index_time'
]
vvec = [x, y, lon, lat, lsm, area, rel_time, abs_time, index_time]
for i, aname in enumerate(addnames):
dset[aname] = vvec[i]
dset['input_dir'] = os.path.dirname(fname)
# ================================================================
# do cutout if wanted --------------------------------------------
field_names = ['x', 'y', 'lon', 'lat', 'lsm', 'area', vname]
if region_slice is not None:
for name in field_names:
dset[name] = gi.cutout_fields(dset[name], region_slice, vaxis=0)
# ================================================================
return dset
def read_narval_data(fname):
'''
Reads the time stack of Narval data, either meteoat or synsat.
Parameters
----------
fname : str
filename of data file
Returns
--------
dset : dict
dataset dictionary containing georef and bt108 data.
'''
# read land sea data ---------------------------------------------
narval_dir = '%s/icon/narval' % local_data_path
lsm_name = '%s/aux/narval_landsea_coast_mask.h5' % narval_dir
print '... read land-sea-mask from %s' % lsm_name
dset = hio.read_dict_from_hdf(lsm_name)
lsm = dset['mask50']
# ================================================================
# read bt108 -----------------------------------------------------
print '... read BT10.8 from %s' % fname
basename, file_ext = os.path.splitext(os.path.basename(fname))
date = basename.split('_')[-1]
t0 = datetime.datetime.strptime(date, '%Y%m%d')
# check if its is obs or sim?
ftype = basename.split('_')[0]
if ftype in ['msevi', 'trans']:
subpath = None
elif ftype == 'synsat':
subpath = 'synsat_oper'
# read bt108 from hdf
if file_ext == '.h5':
b3d = hio.read_var_from_hdf(fname, 'IR_108', subpath=subpath) / 100.
elif file_ext == '.nc':
vname = 'bt108'
b3d = ncio.read_icon_4d_data(fname, vname, itime=None)[vname]
b3d = np.ma.masked_invalid(b3d)
b3d = np.ma.masked_less(b3d, 100.)
ntime, nrow, ncol = b3d.shape
# ================================================================
# prepare time vector --------------------------------------------
rel_time = np.arange(1, ntime + 1)
index_time = np.arange(ntime)
day_shift = t0 - datetime.datetime(1970, 1, 1)
day_shift = day_shift.total_seconds() / (24. * 3600)
abs_time = day_shift + rel_time / 24.
# ================================================================
# get georef .....................................................
gfile = '%s/aux/target_grid_geo_reference_narval.h5' % narval_dir
geo = hio.read_dict_from_hdf(gfile)
lon, lat = geo['lon'], geo['lat']
# centered sinusoidal
x, y = gi.ll2xyc(lon, lat)
area = np.abs(gi.simple_pixel_area(lon, lat))
# ================================================================
# prepare output .................................................
dset = {}
vnames = [
'x', 'y', 'lon', 'lat', 'lsm', 'area', 'rel_time', 'abs_time',
'index_time'
]
vvec = [x, y, lon, lat, lsm, area, rel_time, abs_time, index_time]
for i, vname in enumerate(vnames):
dset[vname] = vvec[i]
dset['bt108'] = b3d
dset['input_dir'] = narval_dir
# ================================================================
return dset
def get_highres_cma(t, region='de', scan_type='rss', arch="none"):
'''
Reads product of High-Res Cloud Mask product (Bley et al., 2013) for a given date.
Parameters
----------
t : datetime object
day and time of MSG time slot
region : str, optional, default = 'de'
string that defines regional cutout
scan_type : str, optional, default = 'rss'
arch : str, optional, default = "none"
archive directory where NWC-SAF files are saved (OPTIONAL)
Returns
--------
var : numpy array
Cma product as field
'''
# set date and time strings --------------------------------------
date_string = t.strftime('%Y/%m/%d')
tstr = t.strftime('%Y%m%dt%H%Mz')
time_string = tstr
# ================================================================
# check scan_type ................................................
scan = scan_type.lower()
if not scan in ('rss', 'pzs'):
print('ERROR: scan_type %s not available')
print(' use either "rss" or "pzs"')
return None
# ================================================================
# set archive directory ------------------------------------------
base_dir = "/vols/altair/datasets/eumcst/"
if arch == 'none':
arch_dir = "%s/msevi_%s/l2_hrv_products/%s/%s" % (base_dir, scan,
region, date_string)
else:
arch_dir = arch
# ================================================================
# set file name --------------------------------------------------
prod_file = arch_dir + "/msg?-sevi-%s-l2-hrvcma-%s-%s.c2.nc" % (
time_string, scan, region)
# get filename while MSG number is unspecified ...................
pfile_list = glob.glob(prod_file)
if pfile_list:
prod_file = pfile_list[0]
else:
raise Exception('ERROR: ', prod_file, ' does not exist!')
# ================================================================
# read variable --------------------------------------------------
# TODO function call should updated !!!
prod = 'clm'
var = ncio.read_icon_4d_data(prod_file, prod, itime=None)
# ================================================================
return var[prod]