def read(filename, time):
d = ds.read(filename, ['time'])
i = np.argmin(np.abs(d['time'] - time))
dt = np.abs(d['time'][i] - time)
if dt <= 1 / 24.:
return ds.read(filename, VARS, sel={'time': i})
else:
return None
def cat(vars_, *input_, **opts):
if not isinstance(vars_, list):
vars_ = [vars_]
for filename in input_:
d = ds.read(
filename,
vars_,
full=False,
jd=(opts.get('jd') or opts.get('h')),
)
varsx = [x for x in d.keys() if not x.startswith('.')]
if len(varsx) == 0:
return
dims0 = d['.'][varsx[0]]['.dims']
for var in varsx:
dims = d['.'][var]['.dims']
if dims != dims0:
raise ValueError('incompatible dimensions')
n = d[vars_[0]].flatten().shape[0]
for i in range(n):
if opts.get('h'):
s = ','.join([
pretty(d[var].flatten()[i], d['.'][var]) for var in vars_
])
else:
s = ','.join([str(d[var].flatten()[i]) for var in vars_])
print(s)
def read(filename, vars, altitude=None, **kwargs):
d = ds.read(filename, VARIABLES, jd=True)
mask = d['elevation_angle'] == 0.0
dx = {}
n, m = d['nrb_copol'].shape
if altitude is None:
altitude = d['gps_altitude']
else:
altitude = np.full(n, altitude, np.float64)
if 'time' in vars:
dx['time'] = d['time']
if 'zfull' in vars:
dx['zfull'] = np.full((n, m), np.nan, np.float64)
for i in range(n):
range_ = 0.5 * d['bin_time'][i] * d['c'] * (np.arange(m) + 0.5)
dx['zfull'][i, :] = range_ * np.sin(
d['elevation_angle'][i] / 180.0 * np.pi)
dx['zfull'][i, :] += altitude[i]
if 'backscatter' in vars:
dx['backscatter'] = (d['nrb_copol'] +
2. * d['nrb_crosspol']) * CALIBRATION_COEFF
if 'altitude' in vars:
dx['altitude'] = altitude
dx['backscatter'] = dx['backscatter'][:, 10:]
dx['zfull'] = dx['zfull'][:, 10:]
dx['.'] = META
dx['.'] = {x: dx['.'][x] for x in vars if x in dx['.']}
return dx
def stats(*args, **opts):
if len(args) < 2:
raise TypeError('Usage: stats <var> <input>...')
var = args[0]
input_ = args[1:]
for filename in input_:
d = ds.read(filename, [var])
x = d[var].flatten()
count = len(x)
min_ = np.min(x)
max_ = np.max(x)
mean = np.mean(x)
median = np.median(x)
j = json.dumps(
{
'count': count,
'min': min_,
'max': max_,
'mean': mean,
'median': median,
},
cls=NumpyEncoder,
sort_keys=True,
indent=4)
print(j)
def readdir(dirname, variables=None, merge=None, warnings=[], **kwargs):
l = sorted(os.listdir(dirname))
dd = []
for name in l:
filename = os.path.join(dirname, name)
try: d = ds.read(filename, variables=variables, **kwargs)
except Exception as e:
warnings.append((
'%s: %s' % (filename, e),
tb.format_exc()
))
continue
d['filename'] = filename
d['.']['filename'] = {
'.dims': [],
}
dd.append(d)
if merge is None:
return dd
else:
n = 0
for n, d in enumerate(dd):
m = ds.get_dims(d)[merge]
d['n'] = np.full(m, n)
d['i'] = np.arange(m)
d['.']['n'] = {
'.dims': [merge],
}
d['.']['i'] = {
'.dims': [merge],
}
d = ds.op.merge(dd, merge, new='n')
return d
def meta(*args, **opts):
input_ = args
for filename in input_:
d = ds.read(filename, [], full=True)
info = d['.']
j = json.dumps(info, sort_keys=True, indent=4, cls=ds.cmd.NumpyEncoder)
print(j)
def read(dirname, track, warnings=[], step=1. / 24.):
dd_index = ds.readdir(dirname,
variables=['time0', 'latitude', 'longitude'],
jd=True)
start_time = track['time'][0]
end_time = track['time'][1]
dd = []
for d_index in dd_index:
time = d_index['time0']
lon = d_index['longitude']
lon = np.where(lon < 0., 360. + lon, lon)
lat = d_index['latitude']
filename = d_index['filename']
ii = np.where((time >= start_time - GRACE_TIME)
& (time <= end_time + GRACE_TIME))[0]
for i in ii:
i2 = np.argmin(np.abs(track['time'] - time[i]))
lon0 = track['lon'][i2]
lat0 = track['lat'][i2]
l = np.argmin((lon - lon0)**2 + (lat - lat0)**2)
j, k = np.unravel_index(l, lon.shape)
# print('<- %s' % filename)
d = ds.read(filename,
variables=VARIABLES,
sel={
'time0': i,
'rlat': j,
'rlon': k
})
if not set(VARIABLES).issubset(d.keys()):
continue
clw = d['model_qcl']
cli = d['model_qcf']
cl = 100. * np.ones(len(clw), dtype=np.float64)
ps = 2 * d['model_press'][0] - d['model_press'][1]
orog = max(0., 2 * d['hybridt32'][0] - d['hybridt32'][1])
pfull = d['model_press']
zfull = d['hybridt32']
ta = d['theta_lev_temp']
newshape4 = (1, len(clw))
newshape3 = (1, )
d_new = {
'clw': clw.reshape(newshape4),
'cli': cli.reshape(newshape4),
'ta': ta.reshape(newshape4),
'cl': cl.reshape(newshape4),
'pfull': pfull.reshape(newshape4),
'zfull': zfull.reshape(newshape4),
'ps': [ps],
'orog': [orog],
'lon': np.array([lon[j, k]]),
'lat': np.array([lat[j, k]]),
'time': np.array([time[i]]),
'.': META,
}
dd.append(d_new)
d = ds.op.merge(dd, 'time')
if 'time' in d:
d['time_bnds'] = misc.time_bnds(d['time'], step)
return d
def read(filename, vars, altitude=None, lon=None, lat=None, **kwargs):
dep_vars = list(set([y for x in vars if x in VARS for y in VARS[x]]))
d = ds.read(filename, dep_vars)
dx = {}
if 'time' in vars:
n = len(d['time'])
dx['time'] = d['time']/86400. + 2440587.5
dx['time_bnds'] = misc.time_bnds(dx['time'], dx['time'][1] - dx['time'][0])
if 'altitude' in vars:
dx['altitude'] = d['altitude'][:,0].astype(np.float64).filled(np.nan)
if 'zfull' in vars:
range_ = d['range'].astype(np.float64).filled(np.nan)
dx['zfull'] = np.tile(range_, (n, 1))
dx['zfull'] = (dx['zfull'].T + dx['altitude']).T
if 'lon' in vars:
dx['lon'] = np.full(n, lon, np.float64)
if 'lat' in vars:
dx['lat'] = np.full(n, lat, np.float64)
if 'backscatter' in vars:
profile_data = d['profile_data'].astype(np.float64).filled(np.nan)
dx['backscatter'] = profile_data*CALIBRATION_COEFF
n, m = dx['backscatter'].shape
j = np.max(np.nonzero(np.any(np.isfinite(dx['backscatter']), axis=0)))
dx['backscatter'] = dx['backscatter'][:,:(j + 1)]
if 'zfull' in vars:
dx['zfull'] = dx['zfull'][:,:(j + 1)]
dx['.'] = META
dx['.'] = {
x: dx['.'][x]
for x in vars
if x in VARS
}
return dx
def merge(dim, *args, **opts):
input_ = args[:-1]
output = args[-1]
dd = []
for filename in input_:
d = ds.read(filename)
dd.append(d)
d = ds.op.merge(dd, dim, variables=opts.get('variables'))
ds.write(output, d)
def read(dirname, track, warnings=[], step=3. / 24.):
dd_index = ds.readdir(dirname, variables=['time', 'lat', 'lon'], jd=True)
start_time = track['time'][0]
end_time = track['time'][-1]
dd = []
for d_index in dd_index:
time = d_index['time']
lat = d_index['lat']
lon = d_index['lon']
lon = np.where(lon < 0., 360. + lon, lon)
filename = d_index['filename']
ii = np.nonzero((time >= start_time) & (time < end_time))[0]
for i in ii:
t = time[i]
i2 = np.argmin(np.abs(track['time'] - time[i]))
lat0 = track['lat'][i2]
lon0 = track['lon'][i2]
j = np.argmin(np.abs(lat - lat0))
k = np.argmin(np.abs(lon - lon0))
d = ds.read(filename,
variables=VARIABLES,
sel={
'time': i,
'lat': j,
'lon': k
})
clw = d['QL'][::-1]
cli = d['QI'][::-1]
cl = d['CLOUD'][::-1] * 100.
ps = d['PS']
orog = d['PHIS'] / 9.80665
pfull = d['PL'][::-1]
zfull = d['H'][::-1]
ta = d['T'][::-1]
nlev = len(clw)
newshape4 = (1, nlev)
newshape3 = (1, )
d_new = {
'clw': clw.reshape(newshape4),
'cli': cli.reshape(newshape4),
'ta': ta.reshape(newshape4),
'cl': cl.reshape(newshape4),
'pfull': pfull.reshape(newshape4),
'zfull': zfull.reshape(newshape4),
'ps': ps.reshape(newshape3),
'orog': orog.reshape(newshape3),
'lat': np.array([lat[j]]),
'lon': np.array([lon[k]]),
'time': np.array([t]),
'.': META,
}
dd.append(d_new)
d = ds.op.merge(dd, 'time')
if 'time' in d:
d['time_bnds'] = misc.time_bnds(d['time'], step)
d['.'] = META
return d
def read(dirname, track, warnings=[], step=3. / 24.):
dd_index = ds.readdir(dirname, variables=['XTIME'], jd=True)
start_time = track['time'][0]
end_time = track['time'][-1]
dd = []
for d_index in dd_index:
time = d_index['XTIME'][0]
time0 = d_index['.']['.']['SIMULATION_START_DATE']
time0 = aq.from_iso(time0.replace('_', 'T'))
if time < 2000000.:
time = time0 + time / (24. * 60.)
filename = d_index['filename']
if (time >= start_time - step * 0.5) & (time < end_time + step * 0.5):
k = np.argmin(np.abs(track['time'] - time))
lon0 = track['lon'][k]
lat0 = track['lat'][k]
d = ds.read(filename, variables=VARS, sel={'Time': 0})
lon = np.where(d['XLONG'] < 0., 360. + d['XLONG'], d['XLONG'])
lat = d['XLAT']
l = np.argmin((lon - lon0)**2 + (lat - lat0)**2)
i, j = np.unravel_index(l, lon.shape)
clw = d['QCLOUD'][:, i, j]
cli = d['QICE'][:, i, j]
cl = 100. * np.ones(len(clw), dtype=np.float64)
ps = d['PSFC'][i, j]
orog = d['HGT'][i, j]
pfull = d['PB'][:, i, j] + d['P'][:, i, j]
zfull = (d['PHB'][:, i, j] + d['PH'][:, i, j]) / 9.81
zfull = 0.5 * (zfull[1:] + zfull[:-1])
theta = d['T'][:, i, j] + d['T00']
ta = theta * (pfull / ps)**KAPPA
newshape3 = [1] + list(clw.shape)
newshape2 = [1] + list(ps.shape)
d_new = {
'clw': clw.reshape(newshape3),
'cli': cli.reshape(newshape3),
'ta': ta.reshape(newshape3),
'cl': cl.reshape(newshape3),
'pfull': pfull.reshape(newshape3),
'zfull': zfull.reshape(newshape3),
'ps': ps.reshape(newshape2),
'orog': orog.reshape(newshape2),
'lon': np.array([lon[i, j]]),
'lat': np.array([lat[i, j]]),
'time': np.array([time]),
'.': META,
}
dd.append(d_new)
d = ds.op.merge(dd, 'time')
if 'time' in d:
d['time_bnds'] = misc.time_bnds(d['time'], step, start_time, end_time)
d['time'] = np.mean(d['time_bnds'], axis=1)
d['.'] = META
return d
def dims(*args, **opts):
if len(args) == 0:
raise TypeError('Usage: dims <input>...')
input_ = args
for filename in input_:
d = ds.read(filename, [])
dims = ds.get_dims(d)
for dim in dims:
print(dim)
out = {x: for x in dims}
j = json.dumps(out, indent=4)
print(j)
def couple(d, d_idx):
dims = d['backscatter'].shape
n = dims[0]
l = dims[2] if len(dims) == 3 else 0
couple_bsd = False
couple_bmol = False
if 'backscatter_sd' not in d:
couple_bsd = True
d['backscatter_sd'] = np.full(dims, np.nan, np.float64)
d['.']['backscatter_sd'] = {
'.dims': ['time', 'range', 'column'] \
if len(dims) == 3 \
else ['time', 'range'],
'long_name': 'total_attenuated_backscatter_coefficient_standard_deviation',
'units': 'm-1 sr-1',
}
if 'backsatter_mol' not in d:
couple_bmol = True
d['backscatter_mol'] = np.full(dims, np.nan, np.float64)
d['.']['backscatter_mol'] = {
'.dims': ['time', 'range', 'column'] \
if len(dims) == 3 \
else ['time', 'range'],
'long_name': 'total_attenuated_molecular_backscatter_coefficient',
'units': 'm-1 sr-1',
}
for i in range(n):
t = d['time'][i]
j = np.argmin(np.abs(d_idx['time'] - t))
n1 = d_idx['n'][j]
filename = d_idx['filename'][n1]
i1 = d_idx['i'][j]
d1 = ds.read(filename, VARIABLES, {'time': i1})
zhalf1 = misc.half(d1['zfull'])
zhalf = misc.half(d['zfull'][i,:]) \
if d['zfull'].ndim == 2 \
else misc.half(d['zfull'])
if couple_bsd and 'backscatter_sd' in d1:
b_sd = interp(zhalf1, d1['backscatter_sd'], zhalf)
if len(dims) == 3:
for k in range(l):
d['backscatter_sd'][i, :, k] = b_sd
else:
d['backscatter_sd'][i, :] = b_sd
if couple_bmol and 'backscatter_mol' in d1:
b_mol = interp(zhalf1, d1['backscatter_mol'], zhalf)
if len(dims) == 3:
for k in range(l):
d['backscatter_mol'][i, :, k] = b_mol
else:
d['backscatter_mol'][i, :] = b_mol
def ls(*args, **opts):
input_ = args
for filename in input_:
d = ds.read(filename, [], full=True)
vars_ = sorted([x for x in d['.'].keys() if not x.startswith('.')])
if opts.get('l'):
for x in vars_:
dims = d['.'][x]['.dims']
size = d['.'][x]['.size']
dims_s = ','.join(
['%s=%d' % (dim, size0) for dim, size0 in zip(dims, size)])
print('%s(%s)' % (x, dims_s))
else:
print('\n'.join(vars_))
def index(dirname, variables=None, warnings=[], **kwargs):
l = sorted(os.listdir(dirname))
dd = []
for name in l:
filename = os.path.join(dirname, name)
try: d = ds.read(filename, variables=variables, **kwargs)
except Exception as e:
warnings.append((
'%s: %s' % (filename, e),
tb.format_exc()
))
continue
d['filename'] = filename
d['.']['filename'] = {
'.dims': [],
}
dd.append(d)
return dd
def couple(d, d_idx):
n, m, l = d['backscatter'].shape
d['backscatter_sd'] = np.full((n, m, l), np.nan, np.float64)
d['.']['backscatter_sd'] = {
'.dims': ['time', 'range', 'column'],
'long_name':
'total_attenuated_backscatter_coefficient_standard_deviation',
'units': 'm-1 sr-1',
}
for i in range(n):
t = d['time'][i]
j = np.argmin(np.abs(d_idx['time'] - t))
n1 = d_idx['n'][j]
filename = d_idx['filename'][n1]
i1 = d_idx['i'][j]
d1 = ds.read(filename, ['zfull', 'backscatter_sd'], {'time': i1})
zhalf1 = misc.half(d1['zfull'])
zhalf = misc.half(d['zfull'][i, :])
b_sd = interp(zhalf1, d1['backscatter_sd'], zhalf)
for k in range(l):
d['backscatter_sd'][i, :, k] = b_sd
def read(filename,
vars,
altitude=None,
lon=None,
lat=None,
calibration_coeff=CALIBRATION_COEFF,
fix_cl_range=False,
cl_crit_range=6000,
**kwargs):
dep_vars = list(set([y for x in vars if x in VARS for y in VARS[x]]))
required_vars = dep_vars + DEFAULT_VARS
d = ds.read(filename, required_vars, jd=True)
dx = {}
dx['time'] = d['time']
dx['time_bnds'] = misc.time_bnds(dx['time'], dx['time'][1] - dx['time'][0])
n = len(dx['time'])
range_ = d['vertical_resolution'][0] * d['level']
if 'zfull' in vars:
zfull1 = range_
dx['zfull'] = np.tile(zfull1, (n, 1))
if altitude is not None:
dx['zfull'] += altitude
if 'backscatter' in vars:
dx['backscatter'] = d['backscatter'] * calibration_coeff
mask = range_ > 6000
if fix_cl_range is True:
for i in range(n):
if d['detection_status'][i] == b'0':
dx['backscatter'][i, mask] *= (range_[mask] / 6000)**2
if 'altitude' in vars:
dx['altitude'] = np.full(n, altitude, np.float64)
if 'lon' in vars:
dx['lon'] = np.full(n, lon, np.float64)
if 'lat' in vars:
dx['lat'] = np.full(n, lat, np.float64)
dx['.'] = META
dx['.'] = {x: dx['.'][x] for x in vars if x in dx['.']}
return dx
def read(filename, vars, altitude=None, lon=None, lat=None, **kwargs):
dep_vars = list(set([y for x in vars if x in VARS for y in VARS[x]]))
d = ds.read(filename, dep_vars)
dx = {}
if 'time' in vars:
n = len(d['time'])
dx['time'] = d['time'] / 86400. + 2440587.5
dx['time_bnds'] = misc.time_bnds(dx['time'],
dx['time'][1] - dx['time'][0])
if 'altitude' in vars:
dx['altitude'] = d['elevation']
if 'zfull' in vars:
dx['zfull'] = np.tile(d['range'], (n, 1))
dx['zfull'] = (dx['zfull'].T + dx['altitude']).T
if 'lon' in vars:
dx['lon'] = np.full(n, lon, np.float64)
if 'lat' in vars:
dx['lat'] = np.full(n, lat, np.float64)
if 'backscatter' in vars:
dx['backscatter'] = d['beta_att'] * CALIBRATION_COEFF
dx['.'] = META
dx['.'] = {x: dx['.'][x] for x in vars if x in VARS}
return dx
def read(filename, vars, altitude=None, lon=None, lat=None, **kwargs):
d = ds.read(filename, VARIABLES, jd=True)
mask = d['elevation_angle'] == 0.0
dx = {}
n, m = d['nrb_copol'].shape
altitude = d['gps_altitude'] if altitude is None else \
np.full(n, altitude, np.float64)
lon = d['gps_longitude'] if lon is None else \
np.full(n, lon, np.float64)
lat = d['gps_latitude'] if lat is None else \
np.full(n, lat, np.float64)
if 'time' in vars:
dx['time'] = d['time']
dx['time_bnds'] = misc.time_bnds(dx['time'],
dx['time'][1] - dx['time'][0])
if 'zfull' in vars:
dx['zfull'] = np.full((n, m), np.nan, np.float64)
for i in range(n):
range_ = 0.5 * d['bin_time'][i] * d['c'] * (np.arange(m) + 0.5)
dx['zfull'][i, :] = range_ * np.sin(
d['elevation_angle'][i] / 180.0 * np.pi)
dx['zfull'][i, :] += altitude[i]
if 'backscatter' in vars:
dx['backscatter'] = (d['nrb_copol'] +
2. * d['nrb_crosspol']) * CALIBRATION_COEFF
if 'altitude' in vars:
dx['altitude'] = altitude
if 'lon' in vars:
dx['lon'] = lon
if 'lat' in vars:
dx['lat'] = lat
dx['.'] = META
dx['.'] = {x: dx['.'][x] for x in vars if x in dx['.']}
return dx
def select(input_, output, variables=None, sel=None):
sel = sel[0] if sel is not None and len(sel) > 0 else None
d = ds.read(input_, variables, sel)
ds.write(output, d)
def run(plot_type,
*args,
lr=True,
subcolumn=0,
width=None,
height=None,
dpi=300,
grid=False,
colors=COLORS,
linestyle=LINESTYLE,
lw=1,
labels=None,
xlim=None,
zlim=None,
vlim=None,
vlog=None,
sigma=5.,
remove_bmol=True,
cloud_mask=True,
title=None,
zres=50,
**kwargs):
"""
alcf plot - plot lidar data
Usage: `alcf plot <plot_type> <input> <output> [<options>] [<plot_options>]`
Arguments:
- `plot_type`: plot type (see Plot types below)
- `input`: input filename or directory
- `output`: output filename or directory
- `options`: see Options below
- `plot_options`: Plot type specific options. See Plot options below.
Plot types:
- `backscatter`: plot backscatter
- `backscatter_hist`: plot backscatter histogram
- `backscatter_sd_hist`: plot backscatter standard deviation histogram
- `cl`: plot model cloud area fraction
- `cli`: plot model mass fraction of cloud ice
- `cloud_occurrence`: plot cloud occurrence
- `clw`: plot model mass fraction of cloud liquid water
- `clw+cli`: plot model mass fraction of cloud liquid water and ice
Options:
- `dpi: <value>`: Resolution in dots per inch (DPI). Default: `300`.
- `--grid`: plot grid
- `height: <value>`: Plot height (inches).
Default: `5` if `plot_type` is `cloud_occurrence` or `backscatter_hist`
else `4`.
- `subcolumn: <value>`: Model subcolumn to plot. Default: `0`.
- `title: <value>`: Plot title.
- `width: <value>`: Plot width (inches).
Default: `5` if `plot_type` is `cloud_occurrence` or `backscatter_hist`
else `10`.
Plot command options:
- `backscatter`:
- `lr: <value>`: Plot effective lidar ratio. Default: `true`.
- `cloud_mask: <value>`: plot cloud mask. Default: `true`.
- `sigma: <value>`: Remove of number of standard deviations of backscatter
from the mean backscatter (real). Default: `5`.
- `remove_bmol: <value>`: Remove molecular backscatter (observed data have
to be coupled with model data via the `couple` option of `alcf lidar`).
Default: `true`.
- `vlim: { <min> <max }`. Value limits (10^6 m-1.sr-1).
Default: `{ 0.1 200 }`.
- `vlog: <value>`: Plot values on logarithmic scale: `true` of `false`.
Default: `true`.
- `backscatter_hist`:
- `vlim: { <min> <max> }`. Value limits (%) or `none` for auto. If `none`
and `vlog` is `none`, `min` is set to 1e-3 if less or equal to zero.
Default: `none`.
- `--vlog`: use logarithmic scale for values
- `xlim: { <min> <max> }`. x axis limits (10^6 m-1.sr-1) or `none` for
automatic. Default: `none`.
- `zlim: { <min> <max> }`. z axis limits (m) or `none` for automatic.
Default: `none`.
- `backscatter_sd_hist`:
- `xlim: { <min> <max> }`. x axis limits (10^6 m-1.sr-1) or `none` for
automatic. Default: `none`.
- `zlim: { <min> <max> }`. z axis limits (%) or `none` for
automatic. Default: `none`.
- `cl`:
- `vlim: { <min> <max> }`. Value limits (%).
Default: `{ 0 100 }`.
- `vlog: <value>`: Plot values on logarithmic scale: `true` of `false`.
Default: `false`.
- `zres: <zres>`: Height resolution (m). Default: `50`.
- `cli`, `clw`, `clw+cli`:
- `vlim: { <min> <max> }`. Value limits (g/kg).
Default: `{ 1e-3 1 }`.
- `vlog: <value>`: Plot values on logarithmic scale: `true` of `false`.
Default: `true`.
- `zres: <zres>`: Height resolution (m). Default: `50`.
- `cloud_occurrence`:
- `colors: { <value>... }`: Line colors.
Default: `{ #0084c8 #dc0000 #009100 #ffc022 #ba00ff }`
- `linestyle: { <value> ... }`: Line style (`solid`, `dashed`, `dotted`).
Default: `solid`.
- `labels: { <value>... }`: Line labels. Default: `none`.
- `lw: <value>`: Line width. Default: `1`.
- `xlim: { <min> <max> }`: x axis limits (%). Default: `{ 0 100 }`.
- `zlim: { <min> <max> }`: z axis limits (m). Default: `{ 0 15 }`.
"""
input_ = args[:-1]
output = args[-1]
if plot_type in ('backscatter_hist', 'backscatter_sd_hist',
'cloud_occurrence'):
width = width if width is not None else 5
height = height if height is not None else 5
else:
width = width if width is not None else 10
height = height if height is not None else 6
opts = {
'lr': lr,
'subcolumn': subcolumn,
'grid': grid,
'colors': colors,
'linestyle': linestyle,
'lw': lw,
'labels': labels,
'sigma': sigma,
'remove_bmol': remove_bmol,
'title': title,
'cloud_mask': cloud_mask,
'width': width,
'height': height,
}
if xlim is not None: opts['xlim'] = xlim
if zlim is not None: opts['zlim'] = zlim
if vlim is not None: opts['vlim'] = vlim
if vlog is not None: opts['vlog'] = vlog
if zres is not None: opts['zres'] = zres
state = {}
if plot_type in ('cloud_occurrence', 'backscatter_sd_hist'):
dd = []
for file in input_:
print('<- %s' % file)
dd += [ds.read(file, VARIABLES)]
plot(plot_type, dd, output, **opts)
print('-> %s' % output)
elif plot_type == 'backscatter_hist':
print('<- %s' % input_[0])
d = ds.read(input_[0], VARIABLES)
plot(plot_type, d, output, **opts)
print('-> %s' % output)
elif plot_type in ('backscatter', 'clw', 'cli', 'clw+cli', 'cl'):
for input1 in input_:
if os.path.isdir(input1):
for file_ in sorted(os.listdir(input1)):
filename = os.path.join(input1, file_)
output_filename = os.path.join(
output,
os.path.splitext(file_)[0] + '.png')
try:
print('<- %s' % filename)
d = ds.read(filename, VARIABLES)
except SystemExit:
raise
except SystemError:
raise
except:
logging.warning(traceback.format_exc())
try:
plot(plot_type, d, output_filename, **opts)
print('-> %s' % output_filename)
except SystemExit:
raise
except SystemError:
sys.exit(1)
except:
logging.warning(traceback.format_exc())
else:
print('<- %s' % input1)
d = ds.read(input1, VARIABLES)
try:
plot(plot_type, d, output, **opts)
except SystemExit:
raise
except SystemError:
sys.exit(1)
else:
raise ValueError('Invalid plot type "%s"' % plot_type)
def read0(type_, dirname, track, warnings=[], step=1. / 24.):
dd_idx = ds.readdir(
dirname,
variables=['time', 'latitude', 'longitude'],
jd=True,
full=True,
warnings=warnings,
)
start_time = track['time'][0]
end_time = track['time'][-1]
vars = {
'surf': VARS_SURF,
'plev': VARS_PLEV,
}[type_]
trans = {
'surf': TRANS_SURF,
'plev': TRANS_PLEV,
}[type_]
dd = []
for d_idx in dd_idx:
time = d_idx['time']
lat = d_idx['latitude']
lon = d_idx['longitude']
filename = d_idx['filename']
ii = np.nonzero((time >= start_time - step * 0.5)
& (time < end_time + step * 0.5))[0]
for i in ii:
t = time[i]
i2 = np.argmin(np.abs(track['time'] - time[i]))
lat0 = track['lat'][i2]
lon0 = track['lon'][i2]
j = np.argmin(np.abs(lat - lat0))
k = np.argmin(np.abs(lon - lon0))
d = ds.read(
filename,
vars,
sel={
'time': [i],
'latitude': j,
'longitude': k
},
jd=True,
)
for a, b in trans.items():
if a in d.keys():
ds.rename(d, a, b)
d['lat'] = np.array([d['lat']])
d['lon'] = np.array([d['lon']])
d['.']['lat']['.dims'] = ['time']
d['.']['lon']['.dims'] = ['time']
if type_ == 'plev':
d['pfull'] = d['pfull'].reshape([1, len(d['pfull'])])
d['.']['pfull']['.dims'] = ['time', 'level']
d['cl'] = d['cl'][:, ::-1]
d['clw'] = d['clw'][:, ::-1]
d['cli'] = d['cli'][:, ::-1]
d['ta'] = d['ta'][:, ::-1]
d['zfull'] = d['zfull'][:, ::-1]
d['pfull'] = d['pfull'][:, ::-1]
dd.append(d)
d = ds.op.merge(dd, 'time')
if 'time' in d:
d['time_bnds'] = misc.time_bnds(d['time'], step, start_time, end_time)
d['time'] = np.mean(d['time_bnds'], axis=1)
if 'pfull' in d:
d['pfull'] = 1e2 * d['pfull']
if 'zfull' in d:
d['zfull'] /= 9.80665
if 'orog' in d:
d['orog'] /= 9.80665
if 'cl' in d:
d['cl'] *= 100.
d['.'] = META
return d
def read(dirname, track, warnings=[], step=1./24.):
d_orog = ds.read(os.path.join(dirname, 'qrparm.orog.nc'), [
'latitude',
'longitude',
'surface_altitude',
])
dd_idx = ds.readdir(dirname,
variables=['TALLTS', 'latitude_t', 'longitude_t', 'DALLTH_zsea_theta'],
jd=True,
full=True,
warnings=warnings,
)
start_time = track['time'][0]
end_time = track['time'][-1]
dd = []
for d_idx in dd_idx:
if 'TALLTS' not in d_idx:
continue
time = d_idx['TALLTS']
lat = d_idx['latitude_t']
lon = d_idx['longitude_t']
filename = d_idx['filename']
ii = np.nonzero(
(time >= start_time - step*0.5) &
(time < end_time + step*0.5)
)[0]
for i in ii:
t = time[i]
i2 = np.argmin(np.abs(track['time'] - time[i]))
lat0 = track['lat'][i2]
lon0 = track['lon'][i2]
j = np.argmin(np.abs(lat - lat0))
k = np.argmin(np.abs(lon - lon0))
d = ds.read(filename, VARS,
sel={'TALLTS': [i], 'latitude_t': j, 'longitude_t': k},
jd=True,
)
for a, b in TRANS.items():
if a in d.keys():
ds.rename(d, a, b)
d['lat'] = np.array([d['lat']])
d['lon'] = np.array([d['lon']])
d['.']['lat']['.dims'] = ['time']
d['.']['lon']['.dims'] = ['time']
orog = d_orog['surface_altitude'][j,k]
d['zfull'] = d['eta']*85000. + orog*(1. - d['eta']/d['eta'][51])**2
d['zfull'] = d['zfull'].reshape([1, len(d['zfull'])])
d['.']['zfull'] = {'.dims': ['time', 'level']}
d['orog'] = np.array([orog], np.float64)
d['.']['orog'] = {'.dims': ['time']}
del d['eta']
dd.append(d)
d = ds.op.merge(dd, 'time')
d['cl'] *= 100.
if 'time' in d:
d['time_bnds'] = misc.time_bnds(d['time'], step, start_time, end_time)
d['time'] = np.mean(d['time_bnds'], axis=1)
d['.'] = META
return d
def run(type_, time_periods, input_, output, eta=0.7):
"""
alcf calibrate - calibrate ALC backscatter
Calibration based the O'Connor et al. (2004) method of lidar ratio (LR) in fully
opaque stratocumulus clouds.
Usage: `alcf calibrate <type> <time_periods> <input> <output> [<options>]`
- `type`: lidar type (see Types below)
- `time_periods`: file containing calibration time periods of stratocumulus
clouds in the backscatter profiles (see Time periods below)
- `input`: input directory containing NetCDF files (the output of `alcf lidar`)
- `output`: output calibration file
- `options`: see Options below.
Types:
- `chm15k`: Lufft CHM 15k
- `cl31`: Vaisala CL31
- `cl51`: Vaisala CL51
- `minimpl`: Sigma Space MiniMPL
- `mpl`: Sigma Space MPL
Time periods file:
A text file containting start and end time (see Time format below) of a time
period separated by whitespace, one time period per line:
```
<start> <end>
<start> <end>
...
```
where `start` is the start time and `end` is the end time.
Time format:
"YYYY-MM-DD[THH:MM[:SS]]", where YYYY is year, MM is month, DD is day,
HH is hour, MM is minute, SS is second. Example: 2000-01-01T00:00:00.
Examples:
`alcf calibrate time_periods.txt lidar calibration.txt`
Read time periods from `time_periods.txt`, lidar profiles from the directory
`lidar` and write the calibration coefficient to `calibration.txt`.
"""
lidar = LIDARS.get(type_)
tp = read_time_periods(time_periods)
files = sorted(os.listdir(input_))
lr = []
for file_ in files:
filename = os.path.join(input_, file_)
print('<- %s' % filename)
d = ds.read(filename, ['time'])
mask = np.zeros(len(d['time']), dtype=np.bool)
for period in tp:
mask |= (d['time'] >= period[0]) & \
(d['time'] < period[1])
d = ds.read(filename, ['lr'], {'time': mask})
lr.append(d['lr'])
lr = np.hstack(lr)
lr_median = np.median(lr)
calibration_ceoff = lidar.CALIBRATION_COEFF * lr_median / lidar.SC_LR
print('-> %s' % output)
with open(output, 'w') as f:
f.write(
'lidar: %s wavelength: %d calibration_coeff: %f lr_median: %f\n' %
(
type_,
lidar.WAVELENGTH,
calibration_ceoff,
lr_median,
))
def read(d):
print('<- %s' % d['filename'])
if d is not None:
d0 = ds.read(d['filename'], VARIABLES)
d.update(d0)
def read(dirname, track, warnings=[], step=6. / 24.):
dd_index = ds.readdir(
dirname,
variables=['time', 'latitude', 'longitude', 'level_height'],
jd=True,
full=True,
warnings=warnings,
)
start_time = track['time'][0]
end_time = track['time'][-1]
d_var = {}
for var in VARIABLES:
dd = []
for d_index in dd_index:
if var not in d_index['.']:
continue
time = d_index['time']
time_half = misc.half(time)
lat = d_index['latitude']
lon = d_index['longitude']
level_height = d_index['level_height']
filename = d_index['filename']
ii = np.nonzero((time_half[1:] >= start_time)
& (time_half[:-1] < end_time))[0]
for i in ii:
t = time[i]
i2 = np.argmin(np.abs(track['time'] - time[i]))
lat0 = track['lat'][i2]
lon0 = track['lon'][i2]
j = np.argmin(np.abs(lat - lat0))
k = np.argmin(np.abs(lon - lon0))
d = ds.read(filename,
variables=[var],
sel={
'time': i,
'latitude': j,
'longitude': k
})
d_new = {
'lat': np.array([lat[j]]),
'lon': np.array([lon[k]]),
'time': np.array([t]),
'level_height': np.array([level_height]),
'.': META,
}
d_new['.']['level_height'] = {'.dims': ['level']}
d_new[TRANS[var]] = d[var].reshape([1] + list(d[var].shape))
if TRANS[var] == 'cl':
d_new[TRANS[var]] *= 100.
dd.append(d_new)
if len(dd) > 0:
d_var[TRANS[var]] = ds.op.merge(dd, 'time')
time_list = [set(d_var[var]['time']) for var in d_var.keys()]
time = time_list[0].intersection(*time_list[1:]) \
if len(time_list) > 0 \
else set()
d = {}
d['.'] = {}
if len(time) == 0:
return None
for var in d_var.keys():
idx = [i for i, t in enumerate(d_var[var]['time']) if t in time]
ds.select(d_var[var], {'time': idx})
d[var] = d_var[var][var]
d['lat'] = d_var[var]['lat']
d['lon'] = d_var[var]['lon']
d['time'] = d_var[var]['time']
d['level_height'] = d_var[var]['level_height']
if 'ps' not in d:
n, m = d['pfull'].shape
d['ps'] = np.full(n, np.nan, dtype=np.float64)
for i in range(n):
d['ps'][i] = 2 * d['pfull'][i, 0] - d['pfull'][i, 1]
if 'zfull' not in d:
n, m = d['pfull'].shape
d['zfull'] = np.full((n, m), np.nan, dtype=np.float64)
for i in range(n):
d['zfull'][i, :] = d['level_height']
del d['level_height']
if 'orog' not in d:
n, m = d['zfull'].shape
d['orog'] = np.full(n, np.nan, dtype=np.float64)
for i in range(n):
d['orog'][i] = 2 * d['zfull'][i, 0] - d['zfull'][i, 1]
if 'time' in d:
d['time_bnds'] = misc.time_bnds(d['time'], step)
d['.'] = META
return d
def run(type_, input_, output,
point=None,
time=None,
track=None,
track_override_year=None,
track_lon_180=False,
**kwargs
):
"""
alcf model - extract model data at a point or along a track
Usage:
alcf model <type> point: { <lon> <lat> } time: { <start> <end> } <input>
<output> [options]
alcf model <type> track: <track> <input> <output>
Arguments:
- `type`: input data type (see Types below)
- `input`: input directory
- `output`: output directory
- `lon`: point longitude
- `lat`: point latitutde
- `start`: start time (see Time format below)
- `end`: end time (see Time format below)
- `track`: track NetCDF file (see Track below)
- `options`: see Options below
Options:
- `track_override_year: <year>`: Override year in track.
Use if comparing observations with a model statistically. Default: `none`.
- `--track_lon_180`: expect track longitude between -180 and 180 degrees
Types:
- `amps`: Antarctic Mesoscale Prediction System (AMPS)
- `era5`: ERA5
- `jra55`: JRA-55
- `merra2`: Modern-Era Retrospective Analysis for Research and Applications,
Version 2 (MERRA-2)
- `nzcsm`: New Zealand Convection Scale Model (NZCSM)
- `nzesm`: New Zealand Earth System Model (NZESM) (experimental)
- `um`: UK Met Office Unified Model (UM)
Time format:
"YYYY-MM-DD[THH:MM[:SS]]", where YYYY is year, MM is month, DD is day,
HH is hour, MM is minute, SS is second. Example: 2000-01-01T00:00:00.
Track:
Track file is a NetCDF file containing 1D variables `lon`, `lat`, and `time`.
`time` is time in format conforming with the NetCDF standard,
`lon` is longitude between 0 and 360 degrees and `lat` is latitude between
-90 and 90 degrees.
"""
time1 = None
track1 = None
if track is not None:
track1 = ds.read(track)
if track_override_year is not None:
date = aq.to_date(track1['time'])
date[1][:] = track_override_year
track1['time'] = aq.from_date(date)
if track_lon_180:
track1['lon'] = np.where(
track1['lon'] > 0,
track1['lon'],
360. + track1['lon']
)
time1 = track1['time'][0], track1['time'][-1]
elif point is not None and time is not None:
pass
else:
raise ValueError('Point and time or track is required')
if time is not None:
time1 = [None, None]
for i in 0, 1:
time1[i] = aq.from_iso(time[i])
if time1[i] is None:
raise ValueError('Invalid time format: %s' % time[i])
# if os.path.isdir(output):
t1, t2 = time1[0], time1[1]
for t in np.arange(np.floor(t1 - 0.5), np.ceil(t2 - 0.5)) + 0.5:
output_filename = os.path.join(output, '%s.nc' % \
aq.to_iso(t).replace(':', ''))
d = model(type_, input_, point, time=[t, t + 1.], track=track1)
if d is not None:
ds.write(output_filename, d)
print('-> %s' % output_filename)
def main_(input_type, output_type, input_, output, surf=None):
d_raw = None
d_pts = None
d_prof = None
d_prof_desc = None
d_surf = None
desc = False
not_supported_msg = 'input or output type not supported'
if input_type.startswith('raw:'):
name = input_type[(input_type.index(':') + 1):]
drv = get_driver(name)
d_raw = ds.read(input_)
elif input_type == 'pts':
d_pts = ds.read(input_)
elif input_type == 'prof':
d_prof = ds.read(input_)
else:
drv = get_driver(input_type)
#if output_type == 'prof' and hasattr(drv, 'read_prof'):
# d_prof = drv.read_prof(input_)
if hasattr(drv, 'read'):
d_raw = drv.read(input_)
if d_pts is None and d_raw is not None and hasattr(drv, 'pts'):
d_pts = drv.pts(d_raw)
if d_prof is None and d_pts is not None:
d_prof = prof(d_pts)
d_prof_desc = prof(d_pts, desc=True)
if d_prof is not None and surf is not None:
drv = rstoollib.drivers.surf
d_surf = drv.read(surf, d_prof['time'][0])
if d_surf is not None:
for k, v in d_surf.items():
if k != '.':
d_prof[k] = d_surf[k]
if d_prof is not None:
postprocess(d_prof)
if d_prof_desc is not None:
postprocess(d_prof_desc)
if output_type == 'prof':
if d_prof is None:
raise ValueError(not_supported_msg)
d = d_prof
elif output_type == 'prof:desc':
if d_prof_desc is None:
raise ValueError(not_supported_msg)
d = d_prof_desc
elif output_type == 'pts':
if d_pts is None:
raise ValueError(not_supported_msg)
d = d_pts
elif output_type == 'raw':
if d_raw is None:
raise ValueError(not_supported_msg)
d = d_raw
else:
raise ValueError(not_supported_msg)
d['.'] = d.get('.', {})
d['.']['.'] = d['.'].get('.', {})
d['.']['.'].update({
'software': 'rstool ' + VERSION + \
' (https://github.com/peterkuma/rstool)',
'created': aq.to_iso(aq.from_datetime(dt.datetime.utcnow())),
})
ds.write(output, d)
def run(input_,
output,
tlim=None,
blim=[5., 200.],
bres=5.,
bsd_lim=[0.001, 10.],
bsd_log=True,
bsd_res=0.001,
bsd_z=8000.,
filter=None,
zlim=[0., 15000.],
zres=100.,
**kwargs):
"""
alcf stats - calculate cloud occurrence statistics
Usage: `alcf stats <input> <output> [<options>]`
Arguments:
- `input`: input filename or directory
- `output`: output filename or directory
Options:
- `blim: <value>`: backscatter histogram limits (1e-6 m-1.sr-1).
Default: `{ 5 200 }`.
- `bres: <value>`: backscatter histogram resolution (1e-6 m-1.sr-1).
Default: `10`.
- `filter: <value>`: filter profiles by condition: `cloudy` for cloudy profiles
only, `clear` for clear sky profiles only, `none` for all profiles.
Default: `none`.
- `tlim: { <start> <end> }`: Time limits (see Time format below).
Default: `none`.
- `zlim: { <low> <high> }`: Height limits (m). Default: `{ 0 15000 }`.
- `zres: <value>`: Height resolution (m). Default: `50`.
- `bsd_lim: { <low> <high> }`: backscatter standard deviation histogram limits
(1e-6 m-1.sr-1). Default: `{ 0.001 10 }`.
- `bsd_log: <value>`: enable/disable logarithmic scale of the backscatter
standard deviation histogram (`true` or `false`). Default: `true`.
- `bsd_res: <value>`: backscatter standard deviation histogram resolution
(1e-6 m-1.sr-1). Default: `0.001`.
- `bsd_z: <value>`: backscatter standard deviation histogram height (m).
Default: `8000`.
Time format:
"YYYY-MM-DD[THH:MM[:SS]]", where YYYY is year, MM is month, DD is day,
HH is hour, MM is minute, SS is second. Example: 2000-01-01T00:00:00.
"""
tlim_jd = parse_time(tlim) if tlim is not None else None
state = {}
options = {
'tlim': tlim_jd,
'blim': np.array(blim, dtype=np.float64) * 1e-6,
'bres': bres * 1e-6,
'bsd_lim': np.array(bsd_lim, dtype=np.float64) * 1e-6,
'bsd_log': bsd_log,
'bsd_res': bsd_res * 1e-6,
'bsd_z': bsd_z,
'filter': filter,
'zlim': zlim,
'zres': zres,
}
if os.path.isdir(input_):
files = os.listdir(input_)
for file in sorted(files):
filename = os.path.join(input_, file)
if not os.path.isfile(filename):
continue
d = ds.read(filename, VARIABLES)
print('<- %s' % filename)
dd = stats.stream([d], state, **options)
else:
d = ds.read(input_, VARIABLES)
print('<- %s' % input_)
dd = stats.stream([d], state, **options)
dd = stats.stream([None], state, **options)
print('-> %s' % output)
ds.to_netcdf(output, dd[0])
def run(input_,
output,
tlim=None,
blim=[5., 200.],
bres=5.,
bsd_lim=[0.001, 10.],
bsd_log=True,
bsd_res=0.001,
bsd_z=8000.,
filter=None,
zlim=[0., 15000.],
zres=100.,
**kwargs):
'''
alcf-stats -- Calculate cloud occurrence statistics.
==========
Synopsis
--------
alcf stats <input> <output> [<options>]
Arguments
---------
- `input`: Input filename or directory.
- `output`: Output filename or directory.
Options
-------
- `blim: <value>`: Backscatter histogram limits (1e-6 m-1.sr-1). Default: `{ 5 200 }`.
- `bres: <value>`: Backscatter histogram resolution (1e-6 m-1.sr-1). Default: `10`.
- `bsd_lim: { <low> <high> }`: Backscatter standard deviation histogram limits (1e-6 m-1.sr-1). Default: `{ 0.001 10 }`.
- `bsd_log: <value>`: Enable/disable logarithmic scale of the backscatter standard deviation histogram (`true` or `false`). Default: `true`.
- `bsd_res: <value>`: Backscatter standard deviation histogram resolution (1e-6 m-1.sr-1). Default: `0.001`.
- `bsd_z: <value>`: Backscatter standard deviation histogram height (m). Default: `8000`.
- `filter: <value> | { <value> ... }`: Filter profiles by condition: `cloudy` for cloudy profiles only, `clear` for clear sky profiles only, `night` for nighttime profiles, `day` for daytime profiles, `none` for all profiles. If an array of values is supplied, all conditions must be true. For `night` and `day`, lidar profiles must contain valid longitude and latitude fields set via the `lon` and `lat` arguments of `alcf lidar` or read implicitly from raw lidar data files if available (mpl, mpl2nc). Default: `none`.
- `tlim: { <start> <end> }`: Time limits (see Time format below). Default: `none`.
- `zlim: { <low> <high> }`: Height limits (m). Default: `{ 0 15000 }`.
- `zres: <value>`: Height resolution (m). Default: `50`.
Time format
-----------
`YYYY-MM-DD[THH:MM[:SS]]`, where `YYYY` is year, `MM` is month, `DD` is day, `HH` is hour, `MM` is minute, `SS` is second. Example: `2000-01-01T00:00:00`.
Examples
--------
Calculate statistics from processed lidar data in `alcf_cl51_lidar` and store the output in `alcf_cl51_stats.nc`.
alcf stats alcf_cl51_lidar alcf_cl51_stats.nc
'''
tlim_jd = parse_time(tlim) if tlim is not None else None
state = {}
options = {
'tlim': tlim_jd,
'blim': np.array(blim, dtype=np.float64) * 1e-6,
'bres': bres * 1e-6,
'bsd_lim': np.array(bsd_lim, dtype=np.float64) * 1e-6,
'bsd_log': bsd_log,
'bsd_res': bsd_res * 1e-6,
'bsd_z': bsd_z,
'filter': filter if type(filter) is list else [filter],
'zlim': zlim,
'zres': zres,
}
if os.path.isdir(input_):
files = sorted(os.listdir(input_))
for file_ in files:
filename = os.path.join(input_, file_)
if not os.path.isfile(filename):
continue
d = ds.read(filename, VARIABLES)
print('<- %s' % filename)
dd = stats.stream([d], state, **options)
else:
d = ds.read(input_, VARIABLES)
print('<- %s' % input_)
dd = stats.stream([d], state, **options)
dd = stats.stream([None], state, **options)
print('-> %s' % output)
ds.write(output, dd[0])
