def get_data_rel(varid, plev, years, datafiles, data, onset, npre, npost):
"""Return daily data aligned relative to onset/withdrawal day.
"""
years = atm.makelist(years)
onset = atm.makelist(onset)
datafiles = atm.makelist(datafiles)
daymin = min(onset) - npre
daymax = max(onset) + npost
# For a single year, add extra year before/after, if necessary
wrap_single = False
years_in = years
if len(years) == 1 and var_type(varid) == 'basic':
filenm = datafiles[0]
year = years[0]
if daymin < 1:
wrap_single = True
file_pre = filenm.replace(str(year), str(year - 1))
if os.path.isfile(file_pre):
years_in = [year - 1] + years_in
datafiles = [file_pre] + datafiles
if daymax > len(atm.season_days('ANN', year)):
wrap_single = True
file_post = filenm.replace(str(year), str(year + 1))
if os.path.isfile(file_post):
years_in = years_in + [year + 1]
datafiles = datafiles + [file_post]
var = get_daily_data(varid, plev, years_in, datafiles, data, daymin=daymin,
daymax=daymax)
# Get rid of extra years
if wrap_single:
var = atm.subset(var, {'year' : (years[0], years[0])})
# Make sure year dimension is included for single year
if len(years) == 1 and 'year' not in var.dims:
var = atm.expand_dims(var, 'year', years[0], axis=0)
# Align relative to onset day
# (not needed for calc variables since they're already aligned)
if var_type(varid) == 'basic':
print('Aligning data relative to onset day')
var = daily_rel2onset(var, onset, npre, npost)
return var
def daily_rel2onset(data, d_onset, npre, npost):
"""Return subset of daily data aligned relative to onset day.
Parameters
----------
data : xray.DataArray
Daily data.
d_onset : ndarray
Array of onset date (day of year) for each year.
npre, npost : int
Number of days before and after onset to extract.
Returns
-------
data_out : xray.DataArray
Subset of N days of daily data for each year, where
N = npre + npost + 1 and the day dimension is
dayrel = day - d_onset.
"""
name, attrs, coords, dimnames = atm.meta(data)
yearnm = atm.get_coord(data, 'year', 'name')
daynm = atm.get_coord(data, 'day', 'name')
years = atm.makelist(atm.get_coord(data, 'year'))
if isinstance(d_onset, xray.DataArray):
d_onset = d_onset.values
else:
d_onset = atm.makelist(d_onset)
relnm = daynm + 'rel'
for y, year in enumerate(years):
dmin, dmax = d_onset[y] - npre, d_onset[y] + npost
subset_dict = {yearnm : (year, None), daynm : (dmin, dmax)}
sub = atm.subset(data, subset_dict)
sub = sub.rename({daynm : relnm})
sub[relnm] = sub[relnm] - d_onset[y]
sub[relnm].attrs['long_name'] = 'Day of year relative to onset day'
if y == 0:
data_out = sub
else:
data_out = xray.concat([data_out, sub], dim=yearnm)
data_out.attrs['d_onset'] = d_onset
return data_out
def contourf_latday(var,
is_precip=False,
clev=None,
cticks=None,
climits=None,
nc_pref=40,
grp=None,
xlims=(-120, 200),
xticks=np.arange(-120, 201, 30),
ylims=(-60, 60),
yticks=np.arange(-60, 61, 20),
dlist=None,
grid=False,
ind_nm='onset',
xlabels=True):
"""Create a filled contour plot of data on latitude-day grid.
"""
var = atm.subset(var, {'lat': ylims})
vals = var.values.T
lat = atm.get_coord(var, 'lat')
days = atm.get_coord(var, 'dayrel')
if var.min() < 0:
symmetric = True
else:
symmetric = False
if is_precip:
cmap = 'PuBuGn'
extend = 'max'
else:
cmap = 'RdBu_r'
extend = 'both'
if clev == None:
cint = atm.cinterval(vals, n_pref=nc_pref, symmetric=symmetric)
clev = atm.clevels(vals, cint, symmetric=symmetric)
elif len(atm.makelist(clev)) == 1:
if is_precip:
clev = np.arange(0, 10 + clev / 2.0, clev)
else:
clev = atm.clevels(vals, clev, symmetric=symmetric)
plt.contourf(days, lat, vals, clev, cmap=cmap, extend=extend)
plt.colorbar(ticks=cticks)
plt.clim(climits)
atm.ax_lims_ticks(xlims, xticks, ylims, yticks)
plt.grid(grid)
if dlist is not None:
for d0 in dlist:
plt.axvline(d0, color='k')
if xlabels:
plt.gca().set_xticklabels(xticks)
plt.xlabel('Days Since ' + ind_nm.capitalize())
else:
plt.gca().set_xticklabels([])
if grp is not None and grp.col == 0:
plt.ylabel('Latitude')
return None
def read_daily_eta(var_id, level, year, month, days=None, concat_dim='TIME',
xsub='[330:2:450]', ysub='[60:2:301]', verbose=True):
"""Return MERRA daily eta-level data for a single variable.
Reads a single eta level of daily MERRA data from OpenDAP urls and
concatenates into a DataArray for the selected days of the month.
Parameters
----------
var_id : str
Variable ID. Can be generic ID from the list below, in which
case get_varname() is called to get the specific ID for MERRA. Or
var_id can be the exact name as it appears in MERRA data files.
Generic IDs:
{'u', 'v', 'omega', 'hgt', 'T', 'q', 'ps', 'evap', 'precip'}
level : int
Eta level to extract (0-71). Level 71 is near-surface and level 0
is the top of atmosphere.
year, month : int
Numeric year and month (1-12).
days : list of ints, optional
Subset of days to read. If None, all days are included.
concat_dim : str, optional
Name of dimension for concatenation.
xsub, ysub : str, optional
Indices of longitude and latitude subsets to extract.
verbose : bool, optional
If True, print updates while processing files.
Returns
-------
data : xray.DataArray or xray.Dataset
Daily data (3-hourly or hourly) for the month or a selected
subset of days.
"""
varnm = get_varname(var_id)
tsub = '[0:1:3]'
zsub = '[%d:1:%d]' % (level, level)
def datafile(year, mon, day, varnm, xsub, ysub, zsub, tsub):
basedir = ('http://goldsmr3.sci.gsfc.nasa.gov:80/opendap/MERRA/'
'MAI6NVANA.5.2.0/')
url = ('%s%d/%02d/MERRA100.prod.assim.inst6_3d_ana_Nv.%d%02d%02d.hdf'
'?%s%s%s%s%s,XDim%s,YDim%s,Height%s,TIME%s') % (basedir, year,
mon, year, mon, day, varnm, tsub, zsub, ysub, xsub, xsub, ysub,
zsub, tsub)
return url
if days is None:
days = range(1, atm.days_this_month(year, month) + 1)
urls = [datafile(year, month, day, varnm, xsub, ysub, zsub, tsub) for day
in atm.makelist(days)]
var = atm.load_concat(urls, varnm, concat_dim, verbose=verbose)
return var
def yearly_values(years, tsdata, index, enso):
data = {}
data['ts'] = tsdata.sel(year=years)
data['onset'] = index['onset'].sel(year=years)
data['retreat'] = index['retreat'].sel(year=years)
data['enso'] = enso.sel(year=years)
# Take average for multiple years
if len(atm.makelist(years)) > 1:
for key in data:
data[key] = data[key].mean(dim='year')
return data['ts'], data['onset'], data['retreat'], data['enso']
def contourf_latday(var, clev=None, title='', nc_pref=40, grp=None,
xlims=(-120, 200), xticks=np.arange(-120, 201, 30),
ylims=(-60, 60), yticks=np.arange(-60, 61, 20),
dlist=None, grid=False):
vals = var.values.T
lat = atm.get_coord(var, 'lat')
days = atm.get_coord(var, 'dayrel')
if var.min() >= 0:
cmap, extend, symmetric = 'PuBuGn', 'max', False
else:
cmap, extend, symmetric = 'RdBu_r', 'both', True
if clev == None:
cint = atm.cinterval(vals, n_pref=nc_pref, symmetric=symmetric)
clev = atm.clevels(vals, cint, symmetric=symmetric)
elif len(atm.makelist(clev)) == 1:
if var.name == 'PREC':
clev = np.arange(0, 10 + clev/2.0, clev)
else:
clev = atm.clevels(vals, clev, symmetric=symmetric)
cticks_dict = {'PRECTOT' : np.arange(0, 13, 2),
'PREC' : np.arange(0, 11, 2),
'T200' : np.arange(-208, 227, 2),
'U200' : np.arange(-60, 61, 10),
'PSI500' : np.arange(-800, 801, 200)}
cticks = cticks_dict.get(var.name)
plt.contourf(days, lat, vals, clev, cmap=cmap, extend=extend)
plt.colorbar(ticks=cticks)
atm.ax_lims_ticks(xlims, xticks, ylims, yticks)
plt.grid(grid)
plt.title(title)
if dlist is not None:
for d0 in dlist:
plt.axvline(d0, color='k')
if grp is not None and grp.row == grp.ncol - 1:
plt.xlabel('Rel Day')
if grp is not None and grp.col == 0:
plt.ylabel('Latitude')
def get_daily_data(varid, plev, years, datafiles, data, daymin=1,
daymax=366, yearnm='year'):
"""Return daily data (basic variable or calculated variable).
Data is read from datafiles if varnm is a basic variable.
If varnm is a calculated variable (e.g. potential temperature),
the base variables for calculation are provided in the dict data.
"""
years = atm.makelist(years)
datafiles = atm.makelist(datafiles)
if isinstance(plev, int) or isinstance(plev, float):
pres = atm.pres_convert(plev, 'hPa', 'Pa')
elif plev == 'LML' and 'PS' in data:
pres = data['PS']
else:
pres = None
def get_var(data, varnm, plev=None):
if plev is None:
plev = ''
elif plev == 'LML' and varnm == 'QV':
varnm = 'Q'
return data[varnm + str(plev)]
if var_type(varid) == 'calc':
print('Computing ' + varid)
if varid == 'THETA':
var = atm.potential_temp(get_var(data, 'T', plev), pres)
elif varid == 'THETA_E':
var = atm.equiv_potential_temp(get_var(data, 'T', plev), pres,
get_var(data, 'QV', plev))
elif varid == 'DSE':
var = atm.dry_static_energy(get_var(data, 'T', plev),
get_var(data, 'H', plev))
elif varid == 'MSE':
var = atm.moist_static_energy(get_var(data, 'T', plev),
get_var(data, 'H', plev),
get_var(data, 'QV', plev))
elif varid == 'VFLXMSE':
Lv = atm.constants.Lv.values
var = data['VFLXCPT'] + data['VFLXPHI'] + data['VFLXQV'] * Lv
var.attrs['units'] = data['VFLXCPT'].attrs['units']
var.attrs['long_name'] = 'Vertically integrated MSE meridional flux'
else:
with xray.open_dataset(datafiles[0]) as ds:
if varid not in ds.data_vars:
varid = varid + str(plev)
var = atm.combine_daily_years(varid, datafiles, years, yearname=yearnm,
subset_dict={'day' : (daymin, daymax)})
var = atm.squeeze(var)
# Make sure year dimension is included for single year
if len(years) == 1 and 'year' not in var.dims:
var = atm.expand_dims(var, yearnm, years[0], axis=0)
# Wrap years for extended day ranges
if daymin < 1 or daymax > 366:
var = wrapyear_all(var, daymin, daymax)
# Convert precip and evap to mm/day
if varid in ['precip', 'PRECTOT', 'EVAP']:
var = atm.precip_convert(var, var.attrs['units'], 'mm/day')
return var
def calc_fluxes(year, month,
var_ids=['u', 'q', 'T', 'theta', 'theta_e', 'hgt'],
concat_dim='TIME', scratchdir=None, keepscratch=False,
verbose=True):
"""Return the monthly mean of MERRA daily fluxes.
Reads MERRA daily data from OpenDAP urls, computes fluxes, and
returns the monthly mean of the daily variable and its zonal and
meridional fluxes.
Parameters
----------
year, month : int
Numeric year and month (1-12).
var_ids : list of str, optional
IDs of variables to include.
concat_dim : str, optional
Name of dimension for concatenation.
scratchdir : str, optional
Directory path to store temporary files while processing data.
If omitted, the current working directory is used.
keepscratch : bool, optional
If True, scratch files are kept in scratchdir. Otherwise they
are deleted.
verbose : bool, optional
If True, print updates while processing files.
Returns
-------
data : xray.Dataset
Mean of daily data and the mean of the daily zonal fluxes
(u * var) and meridional fluxes (v * var), for each variable
in var_ids.
"""
nms = [get_varname(nm) for nm in atm.makelist(var_ids)]
u_nm, v_nm = get_varname('u'), get_varname('v')
nms.extend([u_nm, v_nm])
if 'theta' in nms:
nms.append(get_varname('T'))
if 'theta_e' in nms:
nms.extend([get_varname('T'), get_varname('q')])
nms = set(nms)
days = range(1, atm.days_this_month(year, month) + 1)
def scratchfile(nm, k, year, month, day):
filestr = '%s_level%d_%d%02d%02d.nc' % (nm, k, year, month, day)
if scratchdir is not None:
filestr = scratchdir + '/' + filestr
return filestr
# Read metadata from one file to get pressure-level array
dataset = 'p_daily'
url = url_list(dataset, return_dict=False)[0]
with xray.open_dataset(url) as ds:
pname = atm.get_coord(ds, 'plev', 'name')
plev = atm.get_coord(ds, 'plev')
# Pressure levels in Pa for theta/theta_e calcs
p_units = atm.pres_units(ds[pname].units)
pres = atm.pres_convert(plev, p_units, 'Pa')
# Get daily data (raw and calculate extended variables)
def get_data(nms, pres, year, month, day, concat_dim, subset_dict, verbose):
# Lists of raw and extended variables
ids = list(nms)
ext = []
for var in ['theta', 'theta_e']:
if var in ids:
ext.append(var)
ids.remove(var)
# Read raw data and calculate extended variables
data = read_daily(ids, year, month, day, concat_dim=concat_dim,
subset_dict=subset_dict, verbose=verbose)
if 'theta' in ext:
print_if('Computing potential temperature', verbose)
T = data[get_varname('T')]
data['theta'] = atm.potential_temp(T, pres)
if 'theta_e' in ext:
print_if('Computing equivalent potential temperature', verbose)
T = data[get_varname('T')]
q = data[get_varname('q')]
data['theta_e'] = atm.equiv_potential_temp(T, pres, q)
return data
# Iterate over vertical levels
for k, p in enumerate(plev):
subset_dict = {pname : (p, p)}
print_if('Pressure-level %.1f' % p, verbose)
files = []
for day in days:
# Read data for this level and day
ds = get_data(nms, pres[k], year, month, day, concat_dim,
subset_dict, verbose)
# Compute fluxes
print_if('Computing fluxes', verbose)
u = ds[get_varname('u')]
v = ds[get_varname('v')]
for nm in var_ids:
var = ds[get_varname(nm)]
varname, attrs, _, _ = atm.meta(var)
u_var = u * var
v_var = v * var
u_var.name = get_varname(u_nm) + '*' + var.name
units = var.attrs['units'] + ' * ' + u.attrs['units']
u_var.attrs['units'] = units
v_var.name = get_varname(v_nm) + '*' + var.name
v_var.attrs['units'] = units
ds[u_var.name] = u_var
ds[v_var.name] = v_var
# Save to temporary scratch file
filenm = scratchfile('fluxes', k, year, month, day)
files.append(filenm)
print_if('Saving to scratch file ' + filenm, verbose)
ds.to_netcdf(filenm)
# Concatenate daily scratch files
ds = atm.load_concat(files)
if not keepscratch:
for f in files:
os.remove(f)
# Compute monthly means
print_if('Computing monthly means', verbose)
if k == 0:
data = ds.mean(dim=concat_dim)
else:
data = xray.concat([data, ds.mean(dim=concat_dim)], dim=pname)
for var in data.data_vars:
data[var].attrs = ds[var].attrs
return data
def load_daily_season(pathstr, year, season='ann', var_ids=None,
lat1=-90, lat2=90, lon1=0, lon2=360,
verbose=True, concat_dim=None):
"""Return daily data for a selected year, season and lat-lon subset.
Loads daily data from locally saved files and concatenates it into
a single DataArray or Dataset for that year and season.
Parameters
----------
pathstr : str
Beginning of path for each data file, where each file name is in
the format *yyyymm.nc.
e.g. pathstr = '~/datastore/merra/daily/u200_'
year : int
Year to load.
season : str, optional
Season to load. Valid values are as listed in atm.season_months()
e.g. 'jul', 'jja', 'ann'
Default is entire year ('ann')
var_ids : str or list of str, optional
Variable(s) to extract. If omitted, all variables in the data are
included and the output is a Dataset.
lat1, lat2, lon1, lon2 : floats, optional
Lat-lon subset to extract.
concat_dim : str, optional
Name of time dimension for concatenation. If None, then
atm.get_coord() is called to get the name from the data file.
verbose : bool, optional
If True, print updates while processing files.
Returns
-------
data : xray.DataArray or xray.Dataset
"""
months = atm.season_months(season)
paths = []
for m in months:
datestr = '%d%02d' % (year, m)
paths.append(pathstr + datestr + '.nc')
# Make sure longitude range is consistent with data
with xray.open_dataset(paths[0]) as ds:
lonmax = atm.lon_convention(atm.get_coord(ds, 'lon'))
if concat_dim is None:
concat_dim = atm.get_coord(ds, 'time', 'name')
if lon2 - lon1 == 360:
if lonmax < lon2:
offset = -180
elif lonmax > lon2:
offset = 180
else:
offset = 0
lon1, lon2 = lon1 + offset, lon2 + offset
print(lon1, lon2, lonmax)
# Load daily data
if var_ids is None:
var_nms = None
else:
var_nms = [get_varname(var_id) for var_id in atm.makelist(var_ids)]
subset_dict = {'lat' : (lat1, lat2), 'lon' : (lon1, lon2)}
data = atm.load_concat(paths, var_nms, concat_dim, subset_dict, verbose)
return data
def read_daily(var_ids, year, month, days=None, concat_dim='TIME',
subset_dict=None, verbose=True):
"""Return MERRA daily pressure-level data for selected variable(s).
Reads daily MERRA data from OpenDAP urls and concatenates into a
single DataArray or Dataset for the selected days of the month.
Parameters
----------
var_ids : str or list of str
Variable ID(s). Can be generic ID from the list below, in which
case get_varname() is called to get the specific ID for MERRA. Or
var_id can be the exact name as it appears in MERRA data files.
Generic IDs:
{'u', 'v', 'omega', 'hgt', 'T', 'q', 'ps', 'evap', 'precip'}
year, month : int
Numeric year and month (1-12).
days : list of ints, optional
Subset of days to read. If None, all days are included.
concat_dim : str, optional
Name of dimension for concatenation.
subset_dict : dict of 2-tuples, optional
Dimensions and subsets to extract. Each entry in subset_dict
is in the form {dim_name : (lower_or_list, upper)}, where:
- dim_name : string
Name of dimension to extract from.
The dimension name can be the actual dimension name
(e.g. 'XDim') or a generic name (e.g. 'lon') and get_coord()
is called to find the specific name.
- lower_or_list : scalar or list of int or float
If scalar, then used as the lower bound for the subset range.
If list, then the subset matching the list will be extracted.
- upper : int, float, or None
Upper bound for subset range. If lower_or_list is a list,
then upper is ignored and should be set to None.
verbose : bool, optional
If True, print updates while processing files.
Returns
-------
data : xray.DataArray or xray.Dataset
Daily data (3-hourly or hourly) for the month or a selected
subset of days.
"""
var_ids = atm.makelist(var_ids)
var_nms = [get_varname(var_id) for var_id in var_ids]
dataset = get_dataset(var_ids[0], 'daily')
urls = url_list(dataset)
if days is None:
# All days in the month
dates = ['%d%02d' % (year, month)]
elif isinstance(days, int):
# Single day
dates = ['%d%02d%02d' % (year, month, days)]
else:
# Subset of days
dates = ['%d%02d%02d' % (year, month, d) for d in days]
paths = []
for date in dates:
paths.extend([urls[key] for key in urls.keys() if date in key])
data = atm.load_concat(paths, var_nms, concat_dim, subset_dict,
verbose)
return data
def get_urls(years, months=None, version='merra', varnm='U', opts=None,
monthly=False):
"""Return dict of OpenDAP urls for MERRA and MERRA-2 daily data.
Parameters
----------
years : list or np.ndarray
List of years to extract urls for.
months: list or np.ndarray, optional
List of months to extract urls for. If None, then all months (1-12)
are extracted.
version : {'merra', 'merra2'}, optional
Select MERRA or MERRA-2 data.
varnm : str, optional
Variable ID. If None, then the options in the input parameter opts
are used. If not None, then opts are determined using url_opts(varnm)
and override any value provided to input opts.
opts : dict, optional
Provide the dataset options rather than calling url_opts(varnm).
The key : value pairs of opts are:
'vertical' : 'P', 'X', 'V', or 'E'
Vertical location : on pressure levels (P), 2-D (X), model
layers (V), or model layer edges (E).
'res' : 'N' or 'C'
Horizontal resolution: native (N) or coarse (C).
'time_kind' : 'I' or 'T'
Instantaneous (I) or time-averaged (T) diagnostics.
'kind' : 'ASM', 'SLV', 'FLX', or 'RAD'
Type of dataset: assimilated 3-d (ASM), atmospheric single-level
(SLV), surface turbulent fluxes (FLX), or surface and TOA
radiation fluxes (RAD).
monthly : bool, optional
If True, return urls for monthly data. Otherwise return urls
for daily data.
Returns
-------
urls : dict of date:url for each date in the dataset
"""
if varnm is not None:
opts = url_opts(varnm, version)
# Dataset options
version = version.lower()
time_kind = opts['time_kind'].upper()
res = opts['res'].upper()
vertical = opts['vertical'].upper()
kind = opts['kind'].upper()
# Make dicts of years and months
yearvals = atm.makelist(years)
years = {y : '%d' % y for y in yearvals}
if months is None:
monthvals = range(1, 13)
else:
monthvals = atm.makelist(months)
months = {m : '%02d' % m for m in monthvals}
urlstr = 'http://goldsmr%d.sci.gsfc.nasa.gov/opendap/%s/%s'
dirname = version.upper()
if monthly:
dirname = dirname + '_MONTHLY'
servers = {'merra_X' : urlstr % (2, dirname, 'MA'),
'merra' : urlstr % (3, dirname, 'MA'),
'merra2_X' : urlstr % (4, dirname, 'M2'),
'merra2' : urlstr % (5, dirname, 'M2')}
version_num = {'merra' : '.5.2.0/', 'merra2' : '.5.12.4/'}
fmts = {'merra' : '.hdf', 'merra2' : '.nc4'}
if vertical == 'X':
time_res = '1'
server_key = version + '_X'
else:
time_res = '3'
server_key = version
if monthly:
time_res = 'M'
try:
basedir = servers[server_key]
vnum = version_num[version]
fmt = fmts[version]
except KeyError:
raise ValueError('Invalid version %s. Options are: merra, merra2.' %
version)
basedir = basedir + time_kind + time_res + res + vertical + kind + vnum
print('Scraping filenames from ' + basedir)
# Helper function to make daily urls
def daily_urls(basedir, years, months, fmt):
url_dict = collections.OrderedDict()
for y in years:
for m in months:
print(years[y] + months[m])
dirname = basedir + years[y] + '/' + months[m] + '/'
files = scrape_url(dirname + 'contents.html',
ending=fmt + '.html')
files.sort()
dates = [extract_date(nm, width=8, ending=fmt) for nm in files]
for date, nm in zip(dates, files):
url_dict[date] = dirname + nm
return url_dict
# Helper function to make monthly urls
def monthly_urls(basedir, years, months, fmt):
url_dict = collections.OrderedDict()
for y in years:
dirname = basedir + years[y] + '/'
files = scrape_url(dirname, ending=fmt + '.html')
files.sort()
dates = [extract_date(nm, width=6, ending=fmt) for nm in files]
yr_dict = {date : nm for (date, nm) in zip(dates, files)}
for m in months:
date = years[y] + months[m]
url_dict[date] = dirname + yr_dict[date]
return url_dict
# Extract urls
if monthly:
urls = monthly_urls(basedir, years, months, fmt)
else:
urls = daily_urls(basedir, years, months, fmt)
return urls
