def generic_addvar_reader(fname, vname): ''' Reads the time stack of additional variables. Parameters ---------- fname : str filename of data file vname : str variable name (variable should be contained in file) Returns -------- dset : dict dataset dictionary containing georef and add data. ''' dset = ncio.read_icon_4d_data(fname, [ vname, ], itime=None) geo = ncio.read_icon_georef(fname) geo['x'], geo['y'] = gi.ll2xyc(geo['lon'], geo['lat']) dset.update(geo) return dset
def xy_grid_for_icondata(dset): ''' Calculates an x-y grid (rotated sinosoidal projection) for simulation data stored on regular lon-lat grid. Parameters ---------- dset : xr.Dataset dataset containing lon / lat vectors Returns ------- d : xr.Dataset copy of input data with added x, y grid ''' d = dset.copy() olon = xr.ones_like(d.lon) olat = xr.ones_like(d.lat) d['lat2d'] = d.lat * olon d['lon2d'] = olat * d.lon x, y = gi.ll2xyc(d['lon2d'], d['lat2d']) d['x'] = x d['y'] = y return d
def read_hdcp2_data(fname): ''' Reads BT10.8 data from files generated by the HDCP2 O module. Parameters ---------- fname : str file name Returns -------- dset : dict dictionary of datasets ''' # data fields ---------------------------------------------------- vlist = ['tb108', 'lon', 'lat', 'time'] dset = ncio.read_icon_4d_data(fname, vlist, itime=None) b3d = dset.pop('tb108') b3d = np.ma.masked_less(b3d, 100.) # ================================================================ # geo ref -------------------------------------------------------- lon, lat = dset['lon'], dset['lat'] x, y = gi.ll2xyc(lon, lat, lon0=10, lat0=50) area = np.abs(gi.simple_pixel_area(lon, lat)) # ================================================================ # time conversions ----------------------------------------------- abs_time = dset['time'] / (3600. * 24) rel_time = np.mod(abs_time, 1) * 24. ntime = len(rel_time) index_time = np.arange(ntime) # ================================================================ # prepare output ................................................. vnames = ['x', 'y', 'area', 'rel_time', 'abs_time', 'index_time'] vvec = [x, y, area, rel_time, abs_time, index_time] for i, vname in enumerate(vnames): dset[vname] = vvec[i] dset['bt108'] = b3d dset['lsm'] = np.ones_like(x) dset['input_dir'] = os.path.dirname(fname) # ================================================================ return dset
def read_icon_lem_data(fname): ''' Reads BT10.8 data from files generated for ICON-LEM runs. Parameters ---------- fname : str file name Returns -------- dset : dict dictionary of datasets ''' # data fields ---------------------------------------------------- vlist = ['bt108', 'lon', 'lat', 'time'] dset = ncio.read_icon_4d_data(fname, vlist, itime=None) b3d = dset.pop('bt108') b3d = np.ma.masked_less(b3d, 100.) # ================================================================ # geo ref -------------------------------------------------------- lon, lat = dset['lon'], dset['lat'] x, y = gi.ll2xyc(lon, lat, lon0=10, lat0=50) area = np.abs(gi.simple_pixel_area(lon, lat)) # ================================================================ # time conversions ----------------------------------------------- rel_time = 24 * (dset['time'] - dset['time'][0]) ntime = len(rel_time) index_time = np.arange(ntime) t0 = datetime.datetime(1970, 1, 1) abs_time = [] for t in dset['time']: day = str(int(t)) subday = np.mod(t, 1) tobj = datetime.datetime.strptime(day, '%Y%m%d') tobj += datetime.timedelta(days=subday) dt = (tobj - t0).total_seconds() abs_time.append(dt / (24. * 3600.)) abs_time = np.array(abs_time) # ================================================================ # prepare output ................................................. vnames = ['x', 'y', 'area', 'rel_time', 'abs_time', 'index_time'] vvec = [x, y, area, rel_time, abs_time, index_time] for i, vname in enumerate(vnames): dset[vname] = vvec[i] dset['bt108'] = b3d dset['lsm'] = np.ones_like(x) dset['input_dir'] = os.path.dirname(fname) # ================================================================ return dset
def read_narval_addvars(fname, vname, domain_center=None, region_slice=None): ''' Reads the time stack of Narval data, either meteoat or synsat. Parameters ---------- fname : str filename of data file vname : str variable name (variable should be contained in file) domain_center : tuple of floats, optional, default = None setting the projection center to (clon, clat) if None: not used region_slice : tuple of floats, optional, default = None cutout of fields for form ((irow1, irow2), (icol1, icol2)) if None: not used Returns -------- dset : dict dataset dictionary containing georef and bt108 data. ''' # read land sea data --------------------------------------------- narval_dir = '%s/icon/narval' % local_data_path lsm_name = '%s/aux/narval_landsea_coast_mask.h5' % narval_dir print '... read land-sea-mask from %s' % lsm_name dset = hio.read_dict_from_hdf(lsm_name) lsm = dset['mask50'] # ================================================================ # read bt108 ----------------------------------------------------- print '... read %s from %s' % (vname, fname) basename, file_ext = os.path.splitext(os.path.basename(fname)) date = basename.split('_')[-1] t0 = datetime.datetime.strptime(date, '%Y%m%d') b3d = ncio.read_icon_4d_data(fname, [ vname, ], itime=None)[vname] b3d = np.ma.masked_invalid(b3d) ntime, nrow, ncol = b3d.shape # ================================================================ # prepare time vector -------------------------------------------- rel_time = np.arange(1, ntime + 1) index_time = np.arange(ntime) day_shift = t0 - datetime.datetime(1970, 1, 1) day_shift = day_shift.total_seconds() / (24. * 3600) abs_time = day_shift + rel_time / 24. # ================================================================ # get georef ..................................................... gfile = '%s/aux/target_grid_geo_reference_narval.h5' % narval_dir geo = hio.read_dict_from_hdf(gfile) lon, lat = geo['lon'], geo['lat'] if domain_center is not None: mlon, mlat = domain_center else: mlon, mlat = None, None x, y = gi.ll2xyc(lon, lat, mlon=mlon, mlat=mlat) area = np.abs(gi.simple_pixel_area(x, y, xy=True)) # ================================================================ # prepare output ................................................. dset = {} dset[vname] = b3d addnames = [ 'x', 'y', 'lon', 'lat', 'lsm', 'area', 'rel_time', 'abs_time', 'index_time' ] vvec = [x, y, lon, lat, lsm, area, rel_time, abs_time, index_time] for i, aname in enumerate(addnames): dset[aname] = vvec[i] dset['input_dir'] = os.path.dirname(fname) # ================================================================ # do cutout if wanted -------------------------------------------- field_names = ['x', 'y', 'lon', 'lat', 'lsm', 'area', vname] if region_slice is not None: for name in field_names: dset[name] = gi.cutout_fields(dset[name], region_slice, vaxis=0) # ================================================================ return dset
def read_narval_data(fname): ''' Reads the time stack of Narval data, either meteoat or synsat. Parameters ---------- fname : str filename of data file Returns -------- dset : dict dataset dictionary containing georef and bt108 data. ''' # read land sea data --------------------------------------------- narval_dir = '%s/icon/narval' % local_data_path lsm_name = '%s/aux/narval_landsea_coast_mask.h5' % narval_dir print '... read land-sea-mask from %s' % lsm_name dset = hio.read_dict_from_hdf(lsm_name) lsm = dset['mask50'] # ================================================================ # read bt108 ----------------------------------------------------- print '... read BT10.8 from %s' % fname basename, file_ext = os.path.splitext(os.path.basename(fname)) date = basename.split('_')[-1] t0 = datetime.datetime.strptime(date, '%Y%m%d') # check if its is obs or sim? ftype = basename.split('_')[0] if ftype in ['msevi', 'trans']: subpath = None elif ftype == 'synsat': subpath = 'synsat_oper' # read bt108 from hdf if file_ext == '.h5': b3d = hio.read_var_from_hdf(fname, 'IR_108', subpath=subpath) / 100. elif file_ext == '.nc': vname = 'bt108' b3d = ncio.read_icon_4d_data(fname, vname, itime=None)[vname] b3d = np.ma.masked_invalid(b3d) b3d = np.ma.masked_less(b3d, 100.) ntime, nrow, ncol = b3d.shape # ================================================================ # prepare time vector -------------------------------------------- rel_time = np.arange(1, ntime + 1) index_time = np.arange(ntime) day_shift = t0 - datetime.datetime(1970, 1, 1) day_shift = day_shift.total_seconds() / (24. * 3600) abs_time = day_shift + rel_time / 24. # ================================================================ # get georef ..................................................... gfile = '%s/aux/target_grid_geo_reference_narval.h5' % narval_dir geo = hio.read_dict_from_hdf(gfile) lon, lat = geo['lon'], geo['lat'] # centered sinusoidal x, y = gi.ll2xyc(lon, lat) area = np.abs(gi.simple_pixel_area(lon, lat)) # ================================================================ # prepare output ................................................. dset = {} vnames = [ 'x', 'y', 'lon', 'lat', 'lsm', 'area', 'rel_time', 'abs_time', 'index_time' ] vvec = [x, y, lon, lat, lsm, area, rel_time, abs_time, index_time] for i, vname in enumerate(vnames): dset[vname] = vvec[i] dset['bt108'] = b3d dset['input_dir'] = narval_dir # ================================================================ return dset
def collect_data4cre_sim(radname, itime): ''' Collects a set of simulated data fields for cloud-radiative effect analysis. Parameters ---------- radname : str name of toa allsky radiation file itime : int time index of data fields ('swf_net' and 'lwf') in radname Returns -------- dset : dict dataset dict containing swf, lwf and ct fields ''' # set filenames clearname = radname.replace('toa_', 'toa_clear_') ctname = radname2ctname(radname, datatype='sim') # read radiation data for allsky dset = {} for vname in ['lwf', 'swf_net', 'swf_up']: radset = read_data_field(radname, itime, vname, region='atlantic') dset[vname] = radset[vname] dset['swf_down'] = dset['swf_net'] - dset['swf_up'] # read radiation data for clearsky for vname in ['lwf', 'swf_net']: clearset = read_data_field(clearname, itime, vname, region='atlantic') dset['%s_clear' % vname] = clearset[vname] # calculate SWF up (clearsky) from allsky downwelling (downwelling is the same...) dset['swf_up_clear'] = dset['swf_net_clear'] - dset['swf_down'] ctset = read_data_field(ctname, radset['time_obj'], 'CT', region='atlantic') dset.update(ctset) # select region mask region_mask = dset['mask'] # possible extension (get away from coast) nedge = 11 region_mask = scipy.ndimage.minimum_filter(region_mask, nedge) mlon = dset['lon'][region_mask].mean() mlat = dset['lat'][region_mask].mean() x, y = gi.ll2xyc(dset['lon'], dset['lat'], mlon=mlon, mlat=mlat) a = gi.simple_pixel_area(x, y, xy=True) # update mask and area dset['mask'] = region_mask dset['area'] = a return dset
def collect_data4cre_obs(radname, itime, filepart='-scaled', lwf_clear_offset=-2.): ''' Collects a set of observed data fields for cloud-radiative effect analysis. Parameters ---------- radname : str name of toa allsky radiation file itime : int time index of data fields ('swf_net' and 'lwf') in radname filepart : str, optional, default = '-scaled' part in the file that gives information about scaling of clear-sky fields either '-scaled' or '-not_scaled' lwf_clear_offset : float, optional, default = 2. due to the bias in the simulated LWF, we might use an predefined offset to correct this issue i.e. LWF_clear_simulated += lwf_clear_offset Returns -------- dset : dict dataset dict containing swf, lwf and ct fields ''' # set filenames # ============== clearname = radname.replace('toa_', 'toa_clear_') ctname = radname2ctname(radname, datatype='obs') # read allsky data # ================= dset = {} for vname in ['lwf', 'swf_net', 'swf_up']: radset = read_data_field(radname, itime, vname, region='atlantic') dset[vname] = radset[vname] dset['swf_down'] = dset['swf_net'] - dset['swf_up'] # find the right short-wave clear file # =================================== tobj = radset['time_obj'] filemap = selector.make_filetime_index( 'swf_net', tobj, filepart=filepart, subdirs=['retrieved_clearsky_netswf']) # print filemap # input swf clear # =============== clearname = filemap[tobj][0] clearset = read_data_field(clearname, tobj, 'swf_net', region='atlantic') dset['swf_net_clear'] = clearset['swf_net'] dset['swf_up_clear'] = dset['swf_net_clear'] - dset['swf_down'] # long-wave filename # ==================== lwfclearname = clearname.replace( 'retrieved_clearsky_netswf/clearsky_netswf-', 'sim-toarad/toa_clear_radflux-') lwfclearname = lwfclearname.replace(filepart, '') print((radname, clearname, lwfclearname)) # input lwf clear data # ==================== if filepart == '-not_scaled': lwf_clear_offset = 0 lwfclearset = read_data_field(lwfclearname, tobj, 'lwf', region='atlantic') dset['lwf_clear'] = lwfclearset['lwf'] + lwf_clear_offset # input cloud type # ==================== ctset = read_data_field(ctname, tobj, 'CT', region='atlantic') dset.update(ctset) # select and modify region mask # ============================== region_mask = dset['mask'] # possible extension (get away from coast) nedge = 11 region_mask = scipy.ndimage.minimum_filter(region_mask, nedge) # finally prepare georef # ======================= mlon = dset['lon'][region_mask].mean() mlat = dset['lat'][region_mask].mean() x, y = gi.ll2xyc(dset['lon'], dset['lat'], mlon=mlon, mlat=mlat) a = gi.simple_pixel_area(x, y, xy=True) # update mask and area dset['mask'] = region_mask dset['area'] = a return dset