def read(filename, time): d = ds.read(filename, ['time']) i = np.argmin(np.abs(d['time'] - time)) dt = np.abs(d['time'][i] - time) if dt <= 1 / 24.: return ds.read(filename, VARS, sel={'time': i}) else: return None
def cat(vars_, *input_, **opts): if not isinstance(vars_, list): vars_ = [vars_] for filename in input_: d = ds.read( filename, vars_, full=False, jd=(opts.get('jd') or opts.get('h')), ) varsx = [x for x in d.keys() if not x.startswith('.')] if len(varsx) == 0: return dims0 = d['.'][varsx[0]]['.dims'] for var in varsx: dims = d['.'][var]['.dims'] if dims != dims0: raise ValueError('incompatible dimensions') n = d[vars_[0]].flatten().shape[0] for i in range(n): if opts.get('h'): s = ','.join([ pretty(d[var].flatten()[i], d['.'][var]) for var in vars_ ]) else: s = ','.join([str(d[var].flatten()[i]) for var in vars_]) print(s)
def read(filename, vars, altitude=None, **kwargs): d = ds.read(filename, VARIABLES, jd=True) mask = d['elevation_angle'] == 0.0 dx = {} n, m = d['nrb_copol'].shape if altitude is None: altitude = d['gps_altitude'] else: altitude = np.full(n, altitude, np.float64) if 'time' in vars: dx['time'] = d['time'] if 'zfull' in vars: dx['zfull'] = np.full((n, m), np.nan, np.float64) for i in range(n): range_ = 0.5 * d['bin_time'][i] * d['c'] * (np.arange(m) + 0.5) dx['zfull'][i, :] = range_ * np.sin( d['elevation_angle'][i] / 180.0 * np.pi) dx['zfull'][i, :] += altitude[i] if 'backscatter' in vars: dx['backscatter'] = (d['nrb_copol'] + 2. * d['nrb_crosspol']) * CALIBRATION_COEFF if 'altitude' in vars: dx['altitude'] = altitude dx['backscatter'] = dx['backscatter'][:, 10:] dx['zfull'] = dx['zfull'][:, 10:] dx['.'] = META dx['.'] = {x: dx['.'][x] for x in vars if x in dx['.']} return dx
def stats(*args, **opts): if len(args) < 2: raise TypeError('Usage: stats <var> <input>...') var = args[0] input_ = args[1:] for filename in input_: d = ds.read(filename, [var]) x = d[var].flatten() count = len(x) min_ = np.min(x) max_ = np.max(x) mean = np.mean(x) median = np.median(x) j = json.dumps( { 'count': count, 'min': min_, 'max': max_, 'mean': mean, 'median': median, }, cls=NumpyEncoder, sort_keys=True, indent=4) print(j)
def readdir(dirname, variables=None, merge=None, warnings=[], **kwargs): l = sorted(os.listdir(dirname)) dd = [] for name in l: filename = os.path.join(dirname, name) try: d = ds.read(filename, variables=variables, **kwargs) except Exception as e: warnings.append(( '%s: %s' % (filename, e), tb.format_exc() )) continue d['filename'] = filename d['.']['filename'] = { '.dims': [], } dd.append(d) if merge is None: return dd else: n = 0 for n, d in enumerate(dd): m = ds.get_dims(d)[merge] d['n'] = np.full(m, n) d['i'] = np.arange(m) d['.']['n'] = { '.dims': [merge], } d['.']['i'] = { '.dims': [merge], } d = ds.op.merge(dd, merge, new='n') return d
def meta(*args, **opts): input_ = args for filename in input_: d = ds.read(filename, [], full=True) info = d['.'] j = json.dumps(info, sort_keys=True, indent=4, cls=ds.cmd.NumpyEncoder) print(j)
def read(dirname, track, warnings=[], step=1. / 24.): dd_index = ds.readdir(dirname, variables=['time0', 'latitude', 'longitude'], jd=True) start_time = track['time'][0] end_time = track['time'][1] dd = [] for d_index in dd_index: time = d_index['time0'] lon = d_index['longitude'] lon = np.where(lon < 0., 360. + lon, lon) lat = d_index['latitude'] filename = d_index['filename'] ii = np.where((time >= start_time - GRACE_TIME) & (time <= end_time + GRACE_TIME))[0] for i in ii: i2 = np.argmin(np.abs(track['time'] - time[i])) lon0 = track['lon'][i2] lat0 = track['lat'][i2] l = np.argmin((lon - lon0)**2 + (lat - lat0)**2) j, k = np.unravel_index(l, lon.shape) # print('<- %s' % filename) d = ds.read(filename, variables=VARIABLES, sel={ 'time0': i, 'rlat': j, 'rlon': k }) if not set(VARIABLES).issubset(d.keys()): continue clw = d['model_qcl'] cli = d['model_qcf'] cl = 100. * np.ones(len(clw), dtype=np.float64) ps = 2 * d['model_press'][0] - d['model_press'][1] orog = max(0., 2 * d['hybridt32'][0] - d['hybridt32'][1]) pfull = d['model_press'] zfull = d['hybridt32'] ta = d['theta_lev_temp'] newshape4 = (1, len(clw)) newshape3 = (1, ) d_new = { 'clw': clw.reshape(newshape4), 'cli': cli.reshape(newshape4), 'ta': ta.reshape(newshape4), 'cl': cl.reshape(newshape4), 'pfull': pfull.reshape(newshape4), 'zfull': zfull.reshape(newshape4), 'ps': [ps], 'orog': [orog], 'lon': np.array([lon[j, k]]), 'lat': np.array([lat[j, k]]), 'time': np.array([time[i]]), '.': META, } dd.append(d_new) d = ds.op.merge(dd, 'time') if 'time' in d: d['time_bnds'] = misc.time_bnds(d['time'], step) return d
def read(filename, vars, altitude=None, lon=None, lat=None, **kwargs): dep_vars = list(set([y for x in vars if x in VARS for y in VARS[x]])) d = ds.read(filename, dep_vars) dx = {} if 'time' in vars: n = len(d['time']) dx['time'] = d['time']/86400. + 2440587.5 dx['time_bnds'] = misc.time_bnds(dx['time'], dx['time'][1] - dx['time'][0]) if 'altitude' in vars: dx['altitude'] = d['altitude'][:,0].astype(np.float64).filled(np.nan) if 'zfull' in vars: range_ = d['range'].astype(np.float64).filled(np.nan) dx['zfull'] = np.tile(range_, (n, 1)) dx['zfull'] = (dx['zfull'].T + dx['altitude']).T if 'lon' in vars: dx['lon'] = np.full(n, lon, np.float64) if 'lat' in vars: dx['lat'] = np.full(n, lat, np.float64) if 'backscatter' in vars: profile_data = d['profile_data'].astype(np.float64).filled(np.nan) dx['backscatter'] = profile_data*CALIBRATION_COEFF n, m = dx['backscatter'].shape j = np.max(np.nonzero(np.any(np.isfinite(dx['backscatter']), axis=0))) dx['backscatter'] = dx['backscatter'][:,:(j + 1)] if 'zfull' in vars: dx['zfull'] = dx['zfull'][:,:(j + 1)] dx['.'] = META dx['.'] = { x: dx['.'][x] for x in vars if x in VARS } return dx
def merge(dim, *args, **opts): input_ = args[:-1] output = args[-1] dd = [] for filename in input_: d = ds.read(filename) dd.append(d) d = ds.op.merge(dd, dim, variables=opts.get('variables')) ds.write(output, d)
def read(dirname, track, warnings=[], step=3. / 24.): dd_index = ds.readdir(dirname, variables=['time', 'lat', 'lon'], jd=True) start_time = track['time'][0] end_time = track['time'][-1] dd = [] for d_index in dd_index: time = d_index['time'] lat = d_index['lat'] lon = d_index['lon'] lon = np.where(lon < 0., 360. + lon, lon) filename = d_index['filename'] ii = np.nonzero((time >= start_time) & (time < end_time))[0] for i in ii: t = time[i] i2 = np.argmin(np.abs(track['time'] - time[i])) lat0 = track['lat'][i2] lon0 = track['lon'][i2] j = np.argmin(np.abs(lat - lat0)) k = np.argmin(np.abs(lon - lon0)) d = ds.read(filename, variables=VARIABLES, sel={ 'time': i, 'lat': j, 'lon': k }) clw = d['QL'][::-1] cli = d['QI'][::-1] cl = d['CLOUD'][::-1] * 100. ps = d['PS'] orog = d['PHIS'] / 9.80665 pfull = d['PL'][::-1] zfull = d['H'][::-1] ta = d['T'][::-1] nlev = len(clw) newshape4 = (1, nlev) newshape3 = (1, ) d_new = { 'clw': clw.reshape(newshape4), 'cli': cli.reshape(newshape4), 'ta': ta.reshape(newshape4), 'cl': cl.reshape(newshape4), 'pfull': pfull.reshape(newshape4), 'zfull': zfull.reshape(newshape4), 'ps': ps.reshape(newshape3), 'orog': orog.reshape(newshape3), 'lat': np.array([lat[j]]), 'lon': np.array([lon[k]]), 'time': np.array([t]), '.': META, } dd.append(d_new) d = ds.op.merge(dd, 'time') if 'time' in d: d['time_bnds'] = misc.time_bnds(d['time'], step) d['.'] = META return d
def read(dirname, track, warnings=[], step=3. / 24.): dd_index = ds.readdir(dirname, variables=['XTIME'], jd=True) start_time = track['time'][0] end_time = track['time'][-1] dd = [] for d_index in dd_index: time = d_index['XTIME'][0] time0 = d_index['.']['.']['SIMULATION_START_DATE'] time0 = aq.from_iso(time0.replace('_', 'T')) if time < 2000000.: time = time0 + time / (24. * 60.) filename = d_index['filename'] if (time >= start_time - step * 0.5) & (time < end_time + step * 0.5): k = np.argmin(np.abs(track['time'] - time)) lon0 = track['lon'][k] lat0 = track['lat'][k] d = ds.read(filename, variables=VARS, sel={'Time': 0}) lon = np.where(d['XLONG'] < 0., 360. + d['XLONG'], d['XLONG']) lat = d['XLAT'] l = np.argmin((lon - lon0)**2 + (lat - lat0)**2) i, j = np.unravel_index(l, lon.shape) clw = d['QCLOUD'][:, i, j] cli = d['QICE'][:, i, j] cl = 100. * np.ones(len(clw), dtype=np.float64) ps = d['PSFC'][i, j] orog = d['HGT'][i, j] pfull = d['PB'][:, i, j] + d['P'][:, i, j] zfull = (d['PHB'][:, i, j] + d['PH'][:, i, j]) / 9.81 zfull = 0.5 * (zfull[1:] + zfull[:-1]) theta = d['T'][:, i, j] + d['T00'] ta = theta * (pfull / ps)**KAPPA newshape3 = [1] + list(clw.shape) newshape2 = [1] + list(ps.shape) d_new = { 'clw': clw.reshape(newshape3), 'cli': cli.reshape(newshape3), 'ta': ta.reshape(newshape3), 'cl': cl.reshape(newshape3), 'pfull': pfull.reshape(newshape3), 'zfull': zfull.reshape(newshape3), 'ps': ps.reshape(newshape2), 'orog': orog.reshape(newshape2), 'lon': np.array([lon[i, j]]), 'lat': np.array([lat[i, j]]), 'time': np.array([time]), '.': META, } dd.append(d_new) d = ds.op.merge(dd, 'time') if 'time' in d: d['time_bnds'] = misc.time_bnds(d['time'], step, start_time, end_time) d['time'] = np.mean(d['time_bnds'], axis=1) d['.'] = META return d
def dims(*args, **opts): if len(args) == 0: raise TypeError('Usage: dims <input>...') input_ = args for filename in input_: d = ds.read(filename, []) dims = ds.get_dims(d) for dim in dims: print(dim) out = {x: for x in dims} j = json.dumps(out, indent=4) print(j)
def couple(d, d_idx): dims = d['backscatter'].shape n = dims[0] l = dims[2] if len(dims) == 3 else 0 couple_bsd = False couple_bmol = False if 'backscatter_sd' not in d: couple_bsd = True d['backscatter_sd'] = np.full(dims, np.nan, np.float64) d['.']['backscatter_sd'] = { '.dims': ['time', 'range', 'column'] \ if len(dims) == 3 \ else ['time', 'range'], 'long_name': 'total_attenuated_backscatter_coefficient_standard_deviation', 'units': 'm-1 sr-1', } if 'backsatter_mol' not in d: couple_bmol = True d['backscatter_mol'] = np.full(dims, np.nan, np.float64) d['.']['backscatter_mol'] = { '.dims': ['time', 'range', 'column'] \ if len(dims) == 3 \ else ['time', 'range'], 'long_name': 'total_attenuated_molecular_backscatter_coefficient', 'units': 'm-1 sr-1', } for i in range(n): t = d['time'][i] j = np.argmin(np.abs(d_idx['time'] - t)) n1 = d_idx['n'][j] filename = d_idx['filename'][n1] i1 = d_idx['i'][j] d1 = ds.read(filename, VARIABLES, {'time': i1}) zhalf1 = misc.half(d1['zfull']) zhalf = misc.half(d['zfull'][i,:]) \ if d['zfull'].ndim == 2 \ else misc.half(d['zfull']) if couple_bsd and 'backscatter_sd' in d1: b_sd = interp(zhalf1, d1['backscatter_sd'], zhalf) if len(dims) == 3: for k in range(l): d['backscatter_sd'][i, :, k] = b_sd else: d['backscatter_sd'][i, :] = b_sd if couple_bmol and 'backscatter_mol' in d1: b_mol = interp(zhalf1, d1['backscatter_mol'], zhalf) if len(dims) == 3: for k in range(l): d['backscatter_mol'][i, :, k] = b_mol else: d['backscatter_mol'][i, :] = b_mol
def ls(*args, **opts): input_ = args for filename in input_: d = ds.read(filename, [], full=True) vars_ = sorted([x for x in d['.'].keys() if not x.startswith('.')]) if opts.get('l'): for x in vars_: dims = d['.'][x]['.dims'] size = d['.'][x]['.size'] dims_s = ','.join( ['%s=%d' % (dim, size0) for dim, size0 in zip(dims, size)]) print('%s(%s)' % (x, dims_s)) else: print('\n'.join(vars_))
def index(dirname, variables=None, warnings=[], **kwargs): l = sorted(os.listdir(dirname)) dd = [] for name in l: filename = os.path.join(dirname, name) try: d = ds.read(filename, variables=variables, **kwargs) except Exception as e: warnings.append(( '%s: %s' % (filename, e), tb.format_exc() )) continue d['filename'] = filename d['.']['filename'] = { '.dims': [], } dd.append(d) return dd
def couple(d, d_idx): n, m, l = d['backscatter'].shape d['backscatter_sd'] = np.full((n, m, l), np.nan, np.float64) d['.']['backscatter_sd'] = { '.dims': ['time', 'range', 'column'], 'long_name': 'total_attenuated_backscatter_coefficient_standard_deviation', 'units': 'm-1 sr-1', } for i in range(n): t = d['time'][i] j = np.argmin(np.abs(d_idx['time'] - t)) n1 = d_idx['n'][j] filename = d_idx['filename'][n1] i1 = d_idx['i'][j] d1 = ds.read(filename, ['zfull', 'backscatter_sd'], {'time': i1}) zhalf1 = misc.half(d1['zfull']) zhalf = misc.half(d['zfull'][i, :]) b_sd = interp(zhalf1, d1['backscatter_sd'], zhalf) for k in range(l): d['backscatter_sd'][i, :, k] = b_sd
def read(filename, vars, altitude=None, lon=None, lat=None, calibration_coeff=CALIBRATION_COEFF, fix_cl_range=False, cl_crit_range=6000, **kwargs): dep_vars = list(set([y for x in vars if x in VARS for y in VARS[x]])) required_vars = dep_vars + DEFAULT_VARS d = ds.read(filename, required_vars, jd=True) dx = {} dx['time'] = d['time'] dx['time_bnds'] = misc.time_bnds(dx['time'], dx['time'][1] - dx['time'][0]) n = len(dx['time']) range_ = d['vertical_resolution'][0] * d['level'] if 'zfull' in vars: zfull1 = range_ dx['zfull'] = np.tile(zfull1, (n, 1)) if altitude is not None: dx['zfull'] += altitude if 'backscatter' in vars: dx['backscatter'] = d['backscatter'] * calibration_coeff mask = range_ > 6000 if fix_cl_range is True: for i in range(n): if d['detection_status'][i] == b'0': dx['backscatter'][i, mask] *= (range_[mask] / 6000)**2 if 'altitude' in vars: dx['altitude'] = np.full(n, altitude, np.float64) if 'lon' in vars: dx['lon'] = np.full(n, lon, np.float64) if 'lat' in vars: dx['lat'] = np.full(n, lat, np.float64) dx['.'] = META dx['.'] = {x: dx['.'][x] for x in vars if x in dx['.']} return dx
def read(filename, vars, altitude=None, lon=None, lat=None, **kwargs): dep_vars = list(set([y for x in vars if x in VARS for y in VARS[x]])) d = ds.read(filename, dep_vars) dx = {} if 'time' in vars: n = len(d['time']) dx['time'] = d['time'] / 86400. + 2440587.5 dx['time_bnds'] = misc.time_bnds(dx['time'], dx['time'][1] - dx['time'][0]) if 'altitude' in vars: dx['altitude'] = d['elevation'] if 'zfull' in vars: dx['zfull'] = np.tile(d['range'], (n, 1)) dx['zfull'] = (dx['zfull'].T + dx['altitude']).T if 'lon' in vars: dx['lon'] = np.full(n, lon, np.float64) if 'lat' in vars: dx['lat'] = np.full(n, lat, np.float64) if 'backscatter' in vars: dx['backscatter'] = d['beta_att'] * CALIBRATION_COEFF dx['.'] = META dx['.'] = {x: dx['.'][x] for x in vars if x in VARS} return dx
def read(filename, vars, altitude=None, lon=None, lat=None, **kwargs): d = ds.read(filename, VARIABLES, jd=True) mask = d['elevation_angle'] == 0.0 dx = {} n, m = d['nrb_copol'].shape altitude = d['gps_altitude'] if altitude is None else \ np.full(n, altitude, np.float64) lon = d['gps_longitude'] if lon is None else \ np.full(n, lon, np.float64) lat = d['gps_latitude'] if lat is None else \ np.full(n, lat, np.float64) if 'time' in vars: dx['time'] = d['time'] dx['time_bnds'] = misc.time_bnds(dx['time'], dx['time'][1] - dx['time'][0]) if 'zfull' in vars: dx['zfull'] = np.full((n, m), np.nan, np.float64) for i in range(n): range_ = 0.5 * d['bin_time'][i] * d['c'] * (np.arange(m) + 0.5) dx['zfull'][i, :] = range_ * np.sin( d['elevation_angle'][i] / 180.0 * np.pi) dx['zfull'][i, :] += altitude[i] if 'backscatter' in vars: dx['backscatter'] = (d['nrb_copol'] + 2. * d['nrb_crosspol']) * CALIBRATION_COEFF if 'altitude' in vars: dx['altitude'] = altitude if 'lon' in vars: dx['lon'] = lon if 'lat' in vars: dx['lat'] = lat dx['.'] = META dx['.'] = {x: dx['.'][x] for x in vars if x in dx['.']} return dx
def select(input_, output, variables=None, sel=None): sel = sel[0] if sel is not None and len(sel) > 0 else None d = ds.read(input_, variables, sel) ds.write(output, d)
def run(plot_type, *args, lr=True, subcolumn=0, width=None, height=None, dpi=300, grid=False, colors=COLORS, linestyle=LINESTYLE, lw=1, labels=None, xlim=None, zlim=None, vlim=None, vlog=None, sigma=5., remove_bmol=True, cloud_mask=True, title=None, zres=50, **kwargs): """ alcf plot - plot lidar data Usage: `alcf plot <plot_type> <input> <output> [<options>] [<plot_options>]` Arguments: - `plot_type`: plot type (see Plot types below) - `input`: input filename or directory - `output`: output filename or directory - `options`: see Options below - `plot_options`: Plot type specific options. See Plot options below. Plot types: - `backscatter`: plot backscatter - `backscatter_hist`: plot backscatter histogram - `backscatter_sd_hist`: plot backscatter standard deviation histogram - `cl`: plot model cloud area fraction - `cli`: plot model mass fraction of cloud ice - `cloud_occurrence`: plot cloud occurrence - `clw`: plot model mass fraction of cloud liquid water - `clw+cli`: plot model mass fraction of cloud liquid water and ice Options: - `dpi: <value>`: Resolution in dots per inch (DPI). Default: `300`. - `--grid`: plot grid - `height: <value>`: Plot height (inches). Default: `5` if `plot_type` is `cloud_occurrence` or `backscatter_hist` else `4`. - `subcolumn: <value>`: Model subcolumn to plot. Default: `0`. - `title: <value>`: Plot title. - `width: <value>`: Plot width (inches). Default: `5` if `plot_type` is `cloud_occurrence` or `backscatter_hist` else `10`. Plot command options: - `backscatter`: - `lr: <value>`: Plot effective lidar ratio. Default: `true`. - `cloud_mask: <value>`: plot cloud mask. Default: `true`. - `sigma: <value>`: Remove of number of standard deviations of backscatter from the mean backscatter (real). Default: `5`. - `remove_bmol: <value>`: Remove molecular backscatter (observed data have to be coupled with model data via the `couple` option of `alcf lidar`). Default: `true`. - `vlim: { <min> <max }`. Value limits (10^6 m-1.sr-1). Default: `{ 0.1 200 }`. - `vlog: <value>`: Plot values on logarithmic scale: `true` of `false`. Default: `true`. - `backscatter_hist`: - `vlim: { <min> <max> }`. Value limits (%) or `none` for auto. If `none` and `vlog` is `none`, `min` is set to 1e-3 if less or equal to zero. Default: `none`. - `--vlog`: use logarithmic scale for values - `xlim: { <min> <max> }`. x axis limits (10^6 m-1.sr-1) or `none` for automatic. Default: `none`. - `zlim: { <min> <max> }`. z axis limits (m) or `none` for automatic. Default: `none`. - `backscatter_sd_hist`: - `xlim: { <min> <max> }`. x axis limits (10^6 m-1.sr-1) or `none` for automatic. Default: `none`. - `zlim: { <min> <max> }`. z axis limits (%) or `none` for automatic. Default: `none`. - `cl`: - `vlim: { <min> <max> }`. Value limits (%). Default: `{ 0 100 }`. - `vlog: <value>`: Plot values on logarithmic scale: `true` of `false`. Default: `false`. - `zres: <zres>`: Height resolution (m). Default: `50`. - `cli`, `clw`, `clw+cli`: - `vlim: { <min> <max> }`. Value limits (g/kg). Default: `{ 1e-3 1 }`. - `vlog: <value>`: Plot values on logarithmic scale: `true` of `false`. Default: `true`. - `zres: <zres>`: Height resolution (m). Default: `50`. - `cloud_occurrence`: - `colors: { <value>... }`: Line colors. Default: `{ #0084c8 #dc0000 #009100 #ffc022 #ba00ff }` - `linestyle: { <value> ... }`: Line style (`solid`, `dashed`, `dotted`). Default: `solid`. - `labels: { <value>... }`: Line labels. Default: `none`. - `lw: <value>`: Line width. Default: `1`. - `xlim: { <min> <max> }`: x axis limits (%). Default: `{ 0 100 }`. - `zlim: { <min> <max> }`: z axis limits (m). Default: `{ 0 15 }`. """ input_ = args[:-1] output = args[-1] if plot_type in ('backscatter_hist', 'backscatter_sd_hist', 'cloud_occurrence'): width = width if width is not None else 5 height = height if height is not None else 5 else: width = width if width is not None else 10 height = height if height is not None else 6 opts = { 'lr': lr, 'subcolumn': subcolumn, 'grid': grid, 'colors': colors, 'linestyle': linestyle, 'lw': lw, 'labels': labels, 'sigma': sigma, 'remove_bmol': remove_bmol, 'title': title, 'cloud_mask': cloud_mask, 'width': width, 'height': height, } if xlim is not None: opts['xlim'] = xlim if zlim is not None: opts['zlim'] = zlim if vlim is not None: opts['vlim'] = vlim if vlog is not None: opts['vlog'] = vlog if zres is not None: opts['zres'] = zres state = {} if plot_type in ('cloud_occurrence', 'backscatter_sd_hist'): dd = [] for file in input_: print('<- %s' % file) dd += [ds.read(file, VARIABLES)] plot(plot_type, dd, output, **opts) print('-> %s' % output) elif plot_type == 'backscatter_hist': print('<- %s' % input_[0]) d = ds.read(input_[0], VARIABLES) plot(plot_type, d, output, **opts) print('-> %s' % output) elif plot_type in ('backscatter', 'clw', 'cli', 'clw+cli', 'cl'): for input1 in input_: if os.path.isdir(input1): for file_ in sorted(os.listdir(input1)): filename = os.path.join(input1, file_) output_filename = os.path.join( output, os.path.splitext(file_)[0] + '.png') try: print('<- %s' % filename) d = ds.read(filename, VARIABLES) except SystemExit: raise except SystemError: raise except: logging.warning(traceback.format_exc()) try: plot(plot_type, d, output_filename, **opts) print('-> %s' % output_filename) except SystemExit: raise except SystemError: sys.exit(1) except: logging.warning(traceback.format_exc()) else: print('<- %s' % input1) d = ds.read(input1, VARIABLES) try: plot(plot_type, d, output, **opts) except SystemExit: raise except SystemError: sys.exit(1) else: raise ValueError('Invalid plot type "%s"' % plot_type)
def read0(type_, dirname, track, warnings=[], step=1. / 24.): dd_idx = ds.readdir( dirname, variables=['time', 'latitude', 'longitude'], jd=True, full=True, warnings=warnings, ) start_time = track['time'][0] end_time = track['time'][-1] vars = { 'surf': VARS_SURF, 'plev': VARS_PLEV, }[type_] trans = { 'surf': TRANS_SURF, 'plev': TRANS_PLEV, }[type_] dd = [] for d_idx in dd_idx: time = d_idx['time'] lat = d_idx['latitude'] lon = d_idx['longitude'] filename = d_idx['filename'] ii = np.nonzero((time >= start_time - step * 0.5) & (time < end_time + step * 0.5))[0] for i in ii: t = time[i] i2 = np.argmin(np.abs(track['time'] - time[i])) lat0 = track['lat'][i2] lon0 = track['lon'][i2] j = np.argmin(np.abs(lat - lat0)) k = np.argmin(np.abs(lon - lon0)) d = ds.read( filename, vars, sel={ 'time': [i], 'latitude': j, 'longitude': k }, jd=True, ) for a, b in trans.items(): if a in d.keys(): ds.rename(d, a, b) d['lat'] = np.array([d['lat']]) d['lon'] = np.array([d['lon']]) d['.']['lat']['.dims'] = ['time'] d['.']['lon']['.dims'] = ['time'] if type_ == 'plev': d['pfull'] = d['pfull'].reshape([1, len(d['pfull'])]) d['.']['pfull']['.dims'] = ['time', 'level'] d['cl'] = d['cl'][:, ::-1] d['clw'] = d['clw'][:, ::-1] d['cli'] = d['cli'][:, ::-1] d['ta'] = d['ta'][:, ::-1] d['zfull'] = d['zfull'][:, ::-1] d['pfull'] = d['pfull'][:, ::-1] dd.append(d) d = ds.op.merge(dd, 'time') if 'time' in d: d['time_bnds'] = misc.time_bnds(d['time'], step, start_time, end_time) d['time'] = np.mean(d['time_bnds'], axis=1) if 'pfull' in d: d['pfull'] = 1e2 * d['pfull'] if 'zfull' in d: d['zfull'] /= 9.80665 if 'orog' in d: d['orog'] /= 9.80665 if 'cl' in d: d['cl'] *= 100. d['.'] = META return d
def read(dirname, track, warnings=[], step=1./24.): d_orog = ds.read(os.path.join(dirname, 'qrparm.orog.nc'), [ 'latitude', 'longitude', 'surface_altitude', ]) dd_idx = ds.readdir(dirname, variables=['TALLTS', 'latitude_t', 'longitude_t', 'DALLTH_zsea_theta'], jd=True, full=True, warnings=warnings, ) start_time = track['time'][0] end_time = track['time'][-1] dd = [] for d_idx in dd_idx: if 'TALLTS' not in d_idx: continue time = d_idx['TALLTS'] lat = d_idx['latitude_t'] lon = d_idx['longitude_t'] filename = d_idx['filename'] ii = np.nonzero( (time >= start_time - step*0.5) & (time < end_time + step*0.5) )[0] for i in ii: t = time[i] i2 = np.argmin(np.abs(track['time'] - time[i])) lat0 = track['lat'][i2] lon0 = track['lon'][i2] j = np.argmin(np.abs(lat - lat0)) k = np.argmin(np.abs(lon - lon0)) d = ds.read(filename, VARS, sel={'TALLTS': [i], 'latitude_t': j, 'longitude_t': k}, jd=True, ) for a, b in TRANS.items(): if a in d.keys(): ds.rename(d, a, b) d['lat'] = np.array([d['lat']]) d['lon'] = np.array([d['lon']]) d['.']['lat']['.dims'] = ['time'] d['.']['lon']['.dims'] = ['time'] orog = d_orog['surface_altitude'][j,k] d['zfull'] = d['eta']*85000. + orog*(1. - d['eta']/d['eta'][51])**2 d['zfull'] = d['zfull'].reshape([1, len(d['zfull'])]) d['.']['zfull'] = {'.dims': ['time', 'level']} d['orog'] = np.array([orog], np.float64) d['.']['orog'] = {'.dims': ['time']} del d['eta'] dd.append(d) d = ds.op.merge(dd, 'time') d['cl'] *= 100. if 'time' in d: d['time_bnds'] = misc.time_bnds(d['time'], step, start_time, end_time) d['time'] = np.mean(d['time_bnds'], axis=1) d['.'] = META return d
def run(type_, time_periods, input_, output, eta=0.7): """ alcf calibrate - calibrate ALC backscatter Calibration based the O'Connor et al. (2004) method of lidar ratio (LR) in fully opaque stratocumulus clouds. Usage: `alcf calibrate <type> <time_periods> <input> <output> [<options>]` - `type`: lidar type (see Types below) - `time_periods`: file containing calibration time periods of stratocumulus clouds in the backscatter profiles (see Time periods below) - `input`: input directory containing NetCDF files (the output of `alcf lidar`) - `output`: output calibration file - `options`: see Options below. Types: - `chm15k`: Lufft CHM 15k - `cl31`: Vaisala CL31 - `cl51`: Vaisala CL51 - `minimpl`: Sigma Space MiniMPL - `mpl`: Sigma Space MPL Time periods file: A text file containting start and end time (see Time format below) of a time period separated by whitespace, one time period per line: ``` <start> <end> <start> <end> ... ``` where `start` is the start time and `end` is the end time. Time format: "YYYY-MM-DD[THH:MM[:SS]]", where YYYY is year, MM is month, DD is day, HH is hour, MM is minute, SS is second. Example: 2000-01-01T00:00:00. Examples: `alcf calibrate time_periods.txt lidar calibration.txt` Read time periods from `time_periods.txt`, lidar profiles from the directory `lidar` and write the calibration coefficient to `calibration.txt`. """ lidar = LIDARS.get(type_) tp = read_time_periods(time_periods) files = sorted(os.listdir(input_)) lr = [] for file_ in files: filename = os.path.join(input_, file_) print('<- %s' % filename) d = ds.read(filename, ['time']) mask = np.zeros(len(d['time']), dtype=np.bool) for period in tp: mask |= (d['time'] >= period[0]) & \ (d['time'] < period[1]) d = ds.read(filename, ['lr'], {'time': mask}) lr.append(d['lr']) lr = np.hstack(lr) lr_median = np.median(lr) calibration_ceoff = lidar.CALIBRATION_COEFF * lr_median / lidar.SC_LR print('-> %s' % output) with open(output, 'w') as f: f.write( 'lidar: %s wavelength: %d calibration_coeff: %f lr_median: %f\n' % ( type_, lidar.WAVELENGTH, calibration_ceoff, lr_median, ))
def read(d): print('<- %s' % d['filename']) if d is not None: d0 = ds.read(d['filename'], VARIABLES) d.update(d0)
def read(dirname, track, warnings=[], step=6. / 24.): dd_index = ds.readdir( dirname, variables=['time', 'latitude', 'longitude', 'level_height'], jd=True, full=True, warnings=warnings, ) start_time = track['time'][0] end_time = track['time'][-1] d_var = {} for var in VARIABLES: dd = [] for d_index in dd_index: if var not in d_index['.']: continue time = d_index['time'] time_half = misc.half(time) lat = d_index['latitude'] lon = d_index['longitude'] level_height = d_index['level_height'] filename = d_index['filename'] ii = np.nonzero((time_half[1:] >= start_time) & (time_half[:-1] < end_time))[0] for i in ii: t = time[i] i2 = np.argmin(np.abs(track['time'] - time[i])) lat0 = track['lat'][i2] lon0 = track['lon'][i2] j = np.argmin(np.abs(lat - lat0)) k = np.argmin(np.abs(lon - lon0)) d = ds.read(filename, variables=[var], sel={ 'time': i, 'latitude': j, 'longitude': k }) d_new = { 'lat': np.array([lat[j]]), 'lon': np.array([lon[k]]), 'time': np.array([t]), 'level_height': np.array([level_height]), '.': META, } d_new['.']['level_height'] = {'.dims': ['level']} d_new[TRANS[var]] = d[var].reshape([1] + list(d[var].shape)) if TRANS[var] == 'cl': d_new[TRANS[var]] *= 100. dd.append(d_new) if len(dd) > 0: d_var[TRANS[var]] = ds.op.merge(dd, 'time') time_list = [set(d_var[var]['time']) for var in d_var.keys()] time = time_list[0].intersection(*time_list[1:]) \ if len(time_list) > 0 \ else set() d = {} d['.'] = {} if len(time) == 0: return None for var in d_var.keys(): idx = [i for i, t in enumerate(d_var[var]['time']) if t in time] ds.select(d_var[var], {'time': idx}) d[var] = d_var[var][var] d['lat'] = d_var[var]['lat'] d['lon'] = d_var[var]['lon'] d['time'] = d_var[var]['time'] d['level_height'] = d_var[var]['level_height'] if 'ps' not in d: n, m = d['pfull'].shape d['ps'] = np.full(n, np.nan, dtype=np.float64) for i in range(n): d['ps'][i] = 2 * d['pfull'][i, 0] - d['pfull'][i, 1] if 'zfull' not in d: n, m = d['pfull'].shape d['zfull'] = np.full((n, m), np.nan, dtype=np.float64) for i in range(n): d['zfull'][i, :] = d['level_height'] del d['level_height'] if 'orog' not in d: n, m = d['zfull'].shape d['orog'] = np.full(n, np.nan, dtype=np.float64) for i in range(n): d['orog'][i] = 2 * d['zfull'][i, 0] - d['zfull'][i, 1] if 'time' in d: d['time_bnds'] = misc.time_bnds(d['time'], step) d['.'] = META return d
def run(type_, input_, output, point=None, time=None, track=None, track_override_year=None, track_lon_180=False, **kwargs ): """ alcf model - extract model data at a point or along a track Usage: alcf model <type> point: { <lon> <lat> } time: { <start> <end> } <input> <output> [options] alcf model <type> track: <track> <input> <output> Arguments: - `type`: input data type (see Types below) - `input`: input directory - `output`: output directory - `lon`: point longitude - `lat`: point latitutde - `start`: start time (see Time format below) - `end`: end time (see Time format below) - `track`: track NetCDF file (see Track below) - `options`: see Options below Options: - `track_override_year: <year>`: Override year in track. Use if comparing observations with a model statistically. Default: `none`. - `--track_lon_180`: expect track longitude between -180 and 180 degrees Types: - `amps`: Antarctic Mesoscale Prediction System (AMPS) - `era5`: ERA5 - `jra55`: JRA-55 - `merra2`: Modern-Era Retrospective Analysis for Research and Applications, Version 2 (MERRA-2) - `nzcsm`: New Zealand Convection Scale Model (NZCSM) - `nzesm`: New Zealand Earth System Model (NZESM) (experimental) - `um`: UK Met Office Unified Model (UM) Time format: "YYYY-MM-DD[THH:MM[:SS]]", where YYYY is year, MM is month, DD is day, HH is hour, MM is minute, SS is second. Example: 2000-01-01T00:00:00. Track: Track file is a NetCDF file containing 1D variables `lon`, `lat`, and `time`. `time` is time in format conforming with the NetCDF standard, `lon` is longitude between 0 and 360 degrees and `lat` is latitude between -90 and 90 degrees. """ time1 = None track1 = None if track is not None: track1 = ds.read(track) if track_override_year is not None: date = aq.to_date(track1['time']) date[1][:] = track_override_year track1['time'] = aq.from_date(date) if track_lon_180: track1['lon'] = np.where( track1['lon'] > 0, track1['lon'], 360. + track1['lon'] ) time1 = track1['time'][0], track1['time'][-1] elif point is not None and time is not None: pass else: raise ValueError('Point and time or track is required') if time is not None: time1 = [None, None] for i in 0, 1: time1[i] = aq.from_iso(time[i]) if time1[i] is None: raise ValueError('Invalid time format: %s' % time[i]) # if os.path.isdir(output): t1, t2 = time1[0], time1[1] for t in np.arange(np.floor(t1 - 0.5), np.ceil(t2 - 0.5)) + 0.5: output_filename = os.path.join(output, '%s.nc' % \ aq.to_iso(t).replace(':', '')) d = model(type_, input_, point, time=[t, t + 1.], track=track1) if d is not None: ds.write(output_filename, d) print('-> %s' % output_filename)
def main_(input_type, output_type, input_, output, surf=None): d_raw = None d_pts = None d_prof = None d_prof_desc = None d_surf = None desc = False not_supported_msg = 'input or output type not supported' if input_type.startswith('raw:'): name = input_type[(input_type.index(':') + 1):] drv = get_driver(name) d_raw = ds.read(input_) elif input_type == 'pts': d_pts = ds.read(input_) elif input_type == 'prof': d_prof = ds.read(input_) else: drv = get_driver(input_type) #if output_type == 'prof' and hasattr(drv, 'read_prof'): # d_prof = drv.read_prof(input_) if hasattr(drv, 'read'): d_raw = drv.read(input_) if d_pts is None and d_raw is not None and hasattr(drv, 'pts'): d_pts = drv.pts(d_raw) if d_prof is None and d_pts is not None: d_prof = prof(d_pts) d_prof_desc = prof(d_pts, desc=True) if d_prof is not None and surf is not None: drv = rstoollib.drivers.surf d_surf = drv.read(surf, d_prof['time'][0]) if d_surf is not None: for k, v in d_surf.items(): if k != '.': d_prof[k] = d_surf[k] if d_prof is not None: postprocess(d_prof) if d_prof_desc is not None: postprocess(d_prof_desc) if output_type == 'prof': if d_prof is None: raise ValueError(not_supported_msg) d = d_prof elif output_type == 'prof:desc': if d_prof_desc is None: raise ValueError(not_supported_msg) d = d_prof_desc elif output_type == 'pts': if d_pts is None: raise ValueError(not_supported_msg) d = d_pts elif output_type == 'raw': if d_raw is None: raise ValueError(not_supported_msg) d = d_raw else: raise ValueError(not_supported_msg) d['.'] = d.get('.', {}) d['.']['.'] = d['.'].get('.', {}) d['.']['.'].update({ 'software': 'rstool ' + VERSION + \ ' (https://github.com/peterkuma/rstool)', 'created': aq.to_iso(aq.from_datetime(dt.datetime.utcnow())), }) ds.write(output, d)
def run(input_, output, tlim=None, blim=[5., 200.], bres=5., bsd_lim=[0.001, 10.], bsd_log=True, bsd_res=0.001, bsd_z=8000., filter=None, zlim=[0., 15000.], zres=100., **kwargs): """ alcf stats - calculate cloud occurrence statistics Usage: `alcf stats <input> <output> [<options>]` Arguments: - `input`: input filename or directory - `output`: output filename or directory Options: - `blim: <value>`: backscatter histogram limits (1e-6 m-1.sr-1). Default: `{ 5 200 }`. - `bres: <value>`: backscatter histogram resolution (1e-6 m-1.sr-1). Default: `10`. - `filter: <value>`: filter profiles by condition: `cloudy` for cloudy profiles only, `clear` for clear sky profiles only, `none` for all profiles. Default: `none`. - `tlim: { <start> <end> }`: Time limits (see Time format below). Default: `none`. - `zlim: { <low> <high> }`: Height limits (m). Default: `{ 0 15000 }`. - `zres: <value>`: Height resolution (m). Default: `50`. - `bsd_lim: { <low> <high> }`: backscatter standard deviation histogram limits (1e-6 m-1.sr-1). Default: `{ 0.001 10 }`. - `bsd_log: <value>`: enable/disable logarithmic scale of the backscatter standard deviation histogram (`true` or `false`). Default: `true`. - `bsd_res: <value>`: backscatter standard deviation histogram resolution (1e-6 m-1.sr-1). Default: `0.001`. - `bsd_z: <value>`: backscatter standard deviation histogram height (m). Default: `8000`. Time format: "YYYY-MM-DD[THH:MM[:SS]]", where YYYY is year, MM is month, DD is day, HH is hour, MM is minute, SS is second. Example: 2000-01-01T00:00:00. """ tlim_jd = parse_time(tlim) if tlim is not None else None state = {} options = { 'tlim': tlim_jd, 'blim': np.array(blim, dtype=np.float64) * 1e-6, 'bres': bres * 1e-6, 'bsd_lim': np.array(bsd_lim, dtype=np.float64) * 1e-6, 'bsd_log': bsd_log, 'bsd_res': bsd_res * 1e-6, 'bsd_z': bsd_z, 'filter': filter, 'zlim': zlim, 'zres': zres, } if os.path.isdir(input_): files = os.listdir(input_) for file in sorted(files): filename = os.path.join(input_, file) if not os.path.isfile(filename): continue d = ds.read(filename, VARIABLES) print('<- %s' % filename) dd = stats.stream([d], state, **options) else: d = ds.read(input_, VARIABLES) print('<- %s' % input_) dd = stats.stream([d], state, **options) dd = stats.stream([None], state, **options) print('-> %s' % output) ds.to_netcdf(output, dd[0])
def run(input_, output, tlim=None, blim=[5., 200.], bres=5., bsd_lim=[0.001, 10.], bsd_log=True, bsd_res=0.001, bsd_z=8000., filter=None, zlim=[0., 15000.], zres=100., **kwargs): ''' alcf-stats -- Calculate cloud occurrence statistics. ========== Synopsis -------- alcf stats <input> <output> [<options>] Arguments --------- - `input`: Input filename or directory. - `output`: Output filename or directory. Options ------- - `blim: <value>`: Backscatter histogram limits (1e-6 m-1.sr-1). Default: `{ 5 200 }`. - `bres: <value>`: Backscatter histogram resolution (1e-6 m-1.sr-1). Default: `10`. - `bsd_lim: { <low> <high> }`: Backscatter standard deviation histogram limits (1e-6 m-1.sr-1). Default: `{ 0.001 10 }`. - `bsd_log: <value>`: Enable/disable logarithmic scale of the backscatter standard deviation histogram (`true` or `false`). Default: `true`. - `bsd_res: <value>`: Backscatter standard deviation histogram resolution (1e-6 m-1.sr-1). Default: `0.001`. - `bsd_z: <value>`: Backscatter standard deviation histogram height (m). Default: `8000`. - `filter: <value> | { <value> ... }`: Filter profiles by condition: `cloudy` for cloudy profiles only, `clear` for clear sky profiles only, `night` for nighttime profiles, `day` for daytime profiles, `none` for all profiles. If an array of values is supplied, all conditions must be true. For `night` and `day`, lidar profiles must contain valid longitude and latitude fields set via the `lon` and `lat` arguments of `alcf lidar` or read implicitly from raw lidar data files if available (mpl, mpl2nc). Default: `none`. - `tlim: { <start> <end> }`: Time limits (see Time format below). Default: `none`. - `zlim: { <low> <high> }`: Height limits (m). Default: `{ 0 15000 }`. - `zres: <value>`: Height resolution (m). Default: `50`. Time format ----------- `YYYY-MM-DD[THH:MM[:SS]]`, where `YYYY` is year, `MM` is month, `DD` is day, `HH` is hour, `MM` is minute, `SS` is second. Example: `2000-01-01T00:00:00`. Examples -------- Calculate statistics from processed lidar data in `alcf_cl51_lidar` and store the output in `alcf_cl51_stats.nc`. alcf stats alcf_cl51_lidar alcf_cl51_stats.nc ''' tlim_jd = parse_time(tlim) if tlim is not None else None state = {} options = { 'tlim': tlim_jd, 'blim': np.array(blim, dtype=np.float64) * 1e-6, 'bres': bres * 1e-6, 'bsd_lim': np.array(bsd_lim, dtype=np.float64) * 1e-6, 'bsd_log': bsd_log, 'bsd_res': bsd_res * 1e-6, 'bsd_z': bsd_z, 'filter': filter if type(filter) is list else [filter], 'zlim': zlim, 'zres': zres, } if os.path.isdir(input_): files = sorted(os.listdir(input_)) for file_ in files: filename = os.path.join(input_, file_) if not os.path.isfile(filename): continue d = ds.read(filename, VARIABLES) print('<- %s' % filename) dd = stats.stream([d], state, **options) else: d = ds.read(input_, VARIABLES) print('<- %s' % input_) dd = stats.stream([d], state, **options) dd = stats.stream([None], state, **options) print('-> %s' % output) ds.write(output, dd[0])