def main(argv):
"""Go Main Go"""
basedir = "/mesonet/data/prism"
outdir = "swatfiles_prism_arealaverage"
if os.path.isdir(outdir):
print("ABORT: as %s exists" % (outdir, ))
return
os.mkdir(outdir)
for dirname in ['precipitation', 'temperature']:
os.mkdir("%s/%s" % (outdir, dirname))
pgconn = get_dbconn('idep')
huc8df = gpd.GeoDataFrame.from_postgis("""
SELECT huc8, ST_Transform(simple_geom, %s) as geo from wbd_huc8
WHERE swat_use ORDER by huc8
""", pgconn, params=(PROJSTR,), index_col='huc8', geom_col='geo')
hucs = huc8df.index.values
years = range(1981, 2018)
nc = netCDF4.Dataset("%s/%s_daily.nc" % (basedir, years[0]))
# compute the affine
ncaffine = Affine(nc.variables['lon'][1] - nc.variables['lon'][0],
0.,
nc.variables['lon'][0],
0.,
nc.variables['lat'][0] - nc.variables['lat'][1],
nc.variables['lat'][-1])
czs = CachingZonalStats(ncaffine)
nc.close()
fps = []
for year in years:
nc = netCDF4.Dataset("%s/%s_daily.nc" % (basedir, year))
basedate, timesz = get_basedate(nc)
for i in tqdm(range(timesz), desc=str(year)):
# date = basedate + datetime.timedelta(days=i)
# keep array logic in top-down order
tasmax = np.flipud(nc.variables['tmax'][i, :, :])
tasmin = np.flipud(nc.variables['tmin'][i, :, :])
pr = np.flipud(nc.variables['ppt'][i, :, :])
mytasmax = czs.gen_stats(tasmax, huc8df['geo'])
mytasmin = czs.gen_stats(tasmin, huc8df['geo'])
mypr = czs.gen_stats(pr, huc8df['geo'])
for j, huc12 in enumerate(hucs):
if i == 0 and year == years[0]:
fps.append([open(('%s/precipitation/P%s.txt'
) % (outdir, huc12), 'w'),
open(('%s/temperature/T%s.txt'
) % (outdir, huc12), 'w')])
fps[j][0].write("%s\n" % (basedate.strftime("%Y%m%d"), ))
fps[j][1].write("%s\n" % (basedate.strftime("%Y%m%d"), ))
fps[j][0].write(("%.1f\n"
) % (mypr[j], ))
fps[j][1].write(("%.2f,%.2f\n"
) % (mytasmax[j], mytasmin[j]))
for fp in fps:
fp[0].close()
fp[1].close()
def get_data(ctx):
"""Do the processing work, please"""
pgconn = get_dbconn("postgis")
states = gpd.GeoDataFrame.from_postgis(
"""
SELECT the_geom, state_abbr from states where state_abbr = %s
""",
pgconn,
params=(ctx["state"], ),
index_col="state_abbr",
geom_col="the_geom",
)
if states.empty:
raise NoDataFound("No data was found.")
with ncopen(iemre.get_daily_ncname(ctx["year"])) as nc:
precip = nc.variables["p01d"]
czs = CachingZonalStats(iemre.AFFINE)
hasdata = np.zeros(
(nc.dimensions["lat"].size, nc.dimensions["lon"].size))
czs.gen_stats(hasdata, states["the_geom"])
for nav in czs.gridnav:
grid = np.ones((nav.ysz, nav.xsz))
grid[nav.mask] = 0.0
jslice = slice(nav.y0, nav.y0 + nav.ysz)
islice = slice(nav.x0, nav.x0 + nav.xsz)
hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice,
islice])
ctx["iowa"] = np.flipud(hasdata)
ctx["iowapts"] = float(np.sum(np.where(hasdata > 0, 1, 0)))
now = datetime.datetime(ctx["year"], 1, 1)
now += datetime.timedelta(days=(ctx["period"] - 1))
ets = datetime.datetime(ctx["year"], 12, 31)
today = datetime.datetime.now()
if ets > today:
ets = today - datetime.timedelta(days=1)
ctx["days"] = []
rows = []
trailthres = ((ctx["trailthres"] * units("inch")).to(
units("mm")).magnitude)
daythres = (ctx["daythres"] * units("inch")).to(units("mm")).magnitude
while now < ets:
rows.append(do_date(ctx, now, precip, daythres, trailthres))
now += datetime.timedelta(days=1)
return pd.DataFrame(rows)
def compute_hasdata(year):
"""Compute the has_data grid"""
nc = ncopen(iemre.get_daily_ncname(year), 'a', timeout=300)
czs = CachingZonalStats(iemre.AFFINE)
pgconn = get_dbconn('postgis')
states = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom, state_abbr from states
where state_abbr not in ('AK', 'HI')
""", pgconn, index_col='state_abbr', geom_col='the_geom')
data = np.flipud(nc.variables['hasdata'][:, :])
czs.gen_stats(data, states['the_geom'])
for nav in czs.gridnav:
grid = np.ones((nav.ysz, nav.xsz))
grid[nav.mask] = 0.
jslice = slice(nav.y0, nav.y0 + nav.ysz)
islice = slice(nav.x0, nav.x0 + nav.xsz)
data[jslice, islice] = np.where(grid > 0, 1, data[jslice, islice])
nc.variables['hasdata'][:, :] = np.flipud(data)
nc.close()
def get_data(ctx):
"""Do the processing work, please"""
pgconn = get_dbconn('postgis')
states = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom, state_abbr from states where state_abbr = %s
""",
pgconn,
params=(ctx['state'], ),
index_col='state_abbr',
geom_col='the_geom')
with ncopen(iemre.get_daily_ncname(ctx['year'])) as nc:
precip = nc.variables['p01d']
czs = CachingZonalStats(iemre.AFFINE)
hasdata = np.zeros(
(nc.dimensions['lat'].size, nc.dimensions['lon'].size))
czs.gen_stats(hasdata, states['the_geom'])
for nav in czs.gridnav:
grid = np.ones((nav.ysz, nav.xsz))
grid[nav.mask] = 0.
jslice = slice(nav.y0, nav.y0 + nav.ysz)
islice = slice(nav.x0, nav.x0 + nav.xsz)
hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice,
islice])
ctx['iowa'] = np.flipud(hasdata)
ctx['iowapts'] = float(np.sum(np.where(hasdata > 0, 1, 0)))
now = datetime.datetime(ctx['year'], 1, 1)
now += datetime.timedelta(days=(ctx['period'] - 1))
ets = datetime.datetime(ctx['year'], 12, 31)
today = datetime.datetime.now()
if ets > today:
ets = today - datetime.timedelta(days=1)
ctx['days'] = []
rows = []
trailthres = (ctx['trailthres'] * units('inch')).to(
units('mm')).magnitude
daythres = (ctx['daythres'] * units('inch')).to(units('mm')).magnitude
while now < ets:
rows.append(do_date(ctx, now, precip, daythres, trailthres))
now += datetime.timedelta(days=1)
return pd.DataFrame(rows)
def compute_hasdata():
"""Compute the has_data grid"""
nc = ncopen(iemre.get_dailyc_ncname(), "a", timeout=300)
czs = CachingZonalStats(iemre.AFFINE)
pgconn = get_dbconn("postgis")
states = gpd.GeoDataFrame.from_postgis(
"SELECT the_geom, state_abbr from state "
"where state_abbr not in ('AK', 'HI')",
pgconn,
index_col="state_abbr",
geom_col="the_geom",
)
data = np.flipud(nc.variables["hasdata"][:, :])
czs.gen_stats(data, states["the_geom"])
for nav in czs.gridnav:
grid = np.ones((nav.ysz, nav.xsz))
grid[nav.mask] = 0.0
yslice = slice(nav.y0, nav.y0 + nav.ysz)
xslice = slice(nav.x0, nav.x0 + nav.xsz)
data[yslice, xslice] = np.where(grid > 0, 1, data[yslice, xslice])
nc.variables["hasdata"][:, :] = np.flipud(data)
nc.close()
def do_day(valid):
""" Process a day please """
idx = iemre.daily_offset(valid)
nc = ncopen(iemre.get_daily_ncname(valid.year), 'r', timeout=300)
high = temperature(nc.variables['high_tmpk_12z'][idx, :, :],
'K').value('F')
low = temperature(nc.variables['low_tmpk_12z'][idx, :, :], 'K').value('F')
precip = distance(nc.variables['p01d_12z'][idx, :, :], 'MM').value("IN")
snow = distance(nc.variables['snow_12z'][idx, :, :], 'MM').value("IN")
snowd = distance(nc.variables['snowd_12z'][idx, :, :], 'MM').value("IN")
nc.close()
# build out the state mappers
pgconn = get_dbconn('postgis')
states = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom, state_abbr from states
where state_abbr not in ('AK', 'HI', 'DC')
""",
pgconn,
index_col='state_abbr',
geom_col='the_geom')
czs = CachingZonalStats(iemre.AFFINE)
sthigh = czs.gen_stats(np.flipud(high), states['the_geom'])
stlow = czs.gen_stats(np.flipud(low), states['the_geom'])
stprecip = czs.gen_stats(np.flipud(precip), states['the_geom'])
stsnow = czs.gen_stats(np.flipud(snow), states['the_geom'])
stsnowd = czs.gen_stats(np.flipud(snowd), states['the_geom'])
statedata = {}
for i, state in enumerate(states.index.values):
statedata[state] = dict(high=sthigh[i],
low=stlow[i],
precip=stprecip[i],
snow=stsnow[i],
snowd=stsnowd[i])
update_database(state + "0000", valid, statedata[state])
# build out climate division mappers
climdiv = gpd.GeoDataFrame.from_postgis("""
SELECT geom, iemid from climdiv
where st_abbrv not in ('AK', 'HI', 'DC')
""",
pgconn,
index_col='iemid',
geom_col='geom')
czs = CachingZonalStats(iemre.AFFINE)
sthigh = czs.gen_stats(np.flipud(high), climdiv['geom'])
stlow = czs.gen_stats(np.flipud(low), climdiv['geom'])
stprecip = czs.gen_stats(np.flipud(precip), climdiv['geom'])
stsnow = czs.gen_stats(np.flipud(snow), climdiv['geom'])
stsnowd = czs.gen_stats(np.flipud(snowd), climdiv['geom'])
for i, iemid in enumerate(climdiv.index.values):
row = dict(high=sthigh[i],
low=stlow[i],
precip=stprecip[i],
snow=stsnow[i],
snowd=stsnowd[i])
# we must have temperature data
if row['high'] is np.ma.masked or row['low'] is np.ma.masked:
print(
("compute_0000 %s has missing temperature data, using state") %
(iemid, ))
row = statedata[iemid[:2]]
update_database(iemid, valid, row)
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
csector = ctx['csector']
sdate = make_tuesday(ctx['sdate'])
edate = make_tuesday(ctx['edate'])
dlevel = ctx['d']
griddelta = 0.1
mp = MapPlot(sector=('state' if len(csector) == 2 else csector),
state=ctx['csector'],
title=('%s at or above "%s" %s - %s') %
(PDICT2[ctx['w']], PDICT[dlevel],
sdate.strftime("%b %-d, %Y"), edate.strftime("%b %-d, %Y")),
subtitle=('based on weekly US Drought Monitor Analysis, '
'%.2f$^\circ$ grid analysis') % (griddelta, ),
continentalcolor='white',
titlefontsize=14)
# compute the affine
(west, east, south, north) = mp.ax.get_extent(ccrs.PlateCarree())
raster = np.zeros((int(
(north - south) / griddelta), int((east - west) / griddelta)))
lons = np.arange(raster.shape[1]) * griddelta + west
lats = np.arange(0, 0 - raster.shape[0], -1) * griddelta + north
lats = lats[::-1]
affine = Affine(griddelta, 0., west, 0., 0 - griddelta, north)
# get the geopandas data
pgconn = get_dbconn('postgis')
df = read_postgis("""
with d as (
select valid, (ST_Dump(st_simplify(geom, 0.01))).geom from usdm where
valid >= %s and valid <= %s and dm >= %s and
ST_Intersects(geom, ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s %s,
%s %s, %s %s, %s %s))'))
)
select valid, st_collect(geom) as the_geom from d GROUP by valid
""",
pgconn,
params=(sdate, edate, dlevel, west, south, west, north,
east, north, east, south, west, south),
geom_col='the_geom')
if df.empty:
raise NoDataFound("No Data Found, sorry!")
# loop over the cached stats
czs = CachingZonalStats(affine)
czs.compute_gridnav(df['the_geom'], raster)
for nav in czs.gridnav:
if nav is None:
continue
grid = np.ones((nav.ysz, nav.xsz))
grid[nav.mask] = 0.
jslice = slice(nav.y0, nav.y0 + nav.ysz)
islice = slice(nav.x0, nav.x0 + nav.xsz)
raster[jslice, islice] += grid
maxval = 10 if np.max(raster) < 11 else np.max(raster)
ramp = np.linspace(1, maxval + 1, 11, dtype='i')
if ctx['w'] == 'percent':
ramp = np.arange(0, 101, 10)
ramp[0] = 1.
ramp[-1] = 100.1
# we add one since we are rectified to tuesdays, so we have an extra
# week in there
raster = raster / ((edate - sdate).days / 7. + 1.) * 100.
# plot
cmap = stretch_cmap(ctx['cmap'], ramp)
cmap.set_under('white')
cmap.set_bad('white')
mp.pcolormesh(lons,
lats,
np.flipud(raster),
ramp,
cmap=cmap,
units='count' if ctx['w'] == 'weeks' else 'Percent')
if len(csector) == 2:
mp.drawcounties()
mp.drawcities()
rows = []
for j in range(raster.shape[0]):
for i in range(raster.shape[1]):
rows.append(dict(lon=lons[i], lat=lats[j], value=raster[j, i]))
return mp.fig, pd.DataFrame(rows)
def plotter(fdict):
""" Go """
ctx = util.get_autoplot_context(fdict, get_description())
date = ctx['date']
sector = ctx['sector']
days = ctx['trailing']
threshold = ctx['threshold']
window_sts = date - datetime.timedelta(days=days)
if window_sts.year != date.year:
raise ValueError('Sorry, do not support multi-year plots yet!')
if len(sector) != 2:
raise ValueError("Sorry, this does not support multi-state plots yet.")
idx0 = iemre.daily_offset(window_sts)
idx1 = iemre.daily_offset(date)
ncfn = iemre.get_daily_mrms_ncname(date.year)
ncvar = 'p01d'
if not os.path.isfile(ncfn):
raise ValueError("No data for that year, sorry.")
nc = util.ncopen(ncfn)
# Get the state weight
df = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom from states where state_abbr = %s
""",
util.get_dbconn('postgis'),
params=(sector, ),
index_col=None,
geom_col='the_geom')
czs = CachingZonalStats(iemre.MRMS_AFFINE)
czs.gen_stats(
np.zeros((nc.variables['lat'].size, nc.variables['lon'].size)),
df['the_geom'])
hasdata = None
jslice = None
islice = None
for nav in czs.gridnav:
hasdata = np.ones((nav.ysz, nav.xsz))
hasdata[nav.mask] = 0.
# careful here as y is flipped in this context
jslice = slice(nc.variables['lat'].size - (nav.y0 + nav.ysz),
nc.variables['lat'].size - nav.y0)
islice = slice(nav.x0, nav.x0 + nav.xsz)
hasdata = np.flipud(hasdata)
today = distance(nc.variables[ncvar][idx1, jslice, islice],
'MM').value('IN')
if (idx1 - idx0) < 32:
p01d = distance(
np.sum(nc.variables[ncvar][idx0:idx1, jslice, islice], 0),
'MM').value('IN')
else:
# Too much data can overwhelm this app, need to chunk it
for i in range(idx0, idx1, 10):
i2 = min([i + 10, idx1])
if idx0 == i:
p01d = distance(
np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0),
'MM').value('IN')
else:
p01d += distance(
np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0),
'MM').value('IN')
nc.close()
# Get climatology
nc = util.ncopen(iemre.get_dailyc_mrms_ncname())
if (idx1 - idx0) < 32:
c_p01d = distance(
np.sum(nc.variables[ncvar][idx0:idx1, jslice, islice], 0),
'MM').value('IN')
else:
# Too much data can overwhelm this app, need to chunk it
for i in range(idx0, idx1, 10):
i2 = min([i + 10, idx1])
if idx0 == i:
c_p01d = distance(
np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0),
'MM').value('IN')
else:
c_p01d += distance(
np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0),
'MM').value('IN')
nc.close()
# we actually don't care about weights at this fine of scale
cells = np.sum(np.where(hasdata > 0, 1, 0))
departure = p01d - c_p01d
# Update departure and today to values unconsidered below when out of state
departure = np.where(hasdata > 0, departure, -9999)
today = np.where(hasdata > 0, today, 0)
ranges = [[-99, -3], [-3, -2], [-2, -1], [-1, 0], [0, 1], [1, 2], [2, 3],
[3, 99]]
x = []
x2 = []
labels = []
for (minv, maxv) in ranges:
labels.append("%.0f to %.0f" % (minv, maxv))
# How many departure cells in this range
hits = np.logical_and(departure < maxv, departure > minv)
hits2 = np.logical_and(hits, today > threshold)
x.append(np.sum(np.where(hits, 1, 0)) / float(cells) * 100.)
x2.append(np.sum(np.where(hits2, 1, 0)) / float(cells) * 100.)
(fig, ax) = plt.subplots(1, 1)
ax.set_title(("%s NOAA MRMS %s %.2f inch Precip Coverage") %
(state_names[sector], date.strftime("%-d %b %Y"), threshold))
ax.bar(np.arange(8) - 0.2,
x,
align='center',
width=0.4,
label='Trailing %s Day Departure' % (days, ))
ax.bar(np.arange(8) + 0.2,
x2,
align='center',
width=0.4,
label='%s Coverage (%.1f%% Tot)' %
(date.strftime("%-d %b %Y"), sum(x2)))
for i, (_x1, _x2) in enumerate(zip(x, x2)):
ax.text(i - 0.2, _x1 + 1, "%.1f" % (_x1, ), ha='center')
ax.text(i + 0.2, _x2 + 1, "%.1f" % (_x2, ), ha='center')
ax.set_xticks(np.arange(8))
ax.set_xticklabels(labels)
ax.set_xlabel("Trailing %s Day Precip Departure [in]" % (days, ))
ax.set_position([0.1, 0.2, 0.8, 0.7])
ax.legend(loc=(0., -0.2), ncol=2)
ax.set_ylabel("Areal Coverage of %s [%%]" % (state_names[sector], ))
ax.grid(True)
ax.set_xlim(-0.5, 7.5)
ax.set_ylim(0, max([max(x2), max(x)]) + 5)
return fig
def plotter(fdict):
""" Go """
ctx = util.get_autoplot_context(fdict, get_description())
date = ctx['date']
sector = ctx['sector']
threshold = ctx['threshold']
threshold_mm = distance(threshold, 'IN').value('MM')
window_sts = date - datetime.timedelta(days=90)
if window_sts.year != date.year:
raise NoDataFound('Sorry, do not support multi-year plots yet!')
# idx0 = iemre.daily_offset(window_sts)
idx1 = iemre.daily_offset(date)
ncfn = iemre.get_daily_mrms_ncname(date.year)
if not os.path.isfile(ncfn):
raise NoDataFound("No data found.")
ncvar = 'p01d'
# Get the state weight
df = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom from states where state_abbr = %s
""",
util.get_dbconn('postgis'),
params=(sector, ),
index_col=None,
geom_col='the_geom')
czs = CachingZonalStats(iemre.MRMS_AFFINE)
with util.ncopen(ncfn) as nc:
czs.gen_stats(
np.zeros((nc.variables['lat'].size, nc.variables['lon'].size)),
df['the_geom'])
jslice = None
islice = None
for nav in czs.gridnav:
# careful here as y is flipped in this context
jslice = slice(nc.variables['lat'].size - (nav.y0 + nav.ysz),
nc.variables['lat'].size - nav.y0)
islice = slice(nav.x0, nav.x0 + nav.xsz)
grid = np.zeros(
(jslice.stop - jslice.start, islice.stop - islice.start))
total = np.zeros(
(jslice.stop - jslice.start, islice.stop - islice.start))
for i, idx in enumerate(range(idx1, idx1 - 90, -1)):
total += nc.variables[ncvar][idx, jslice, islice]
grid = np.where(np.logical_and(grid == 0, total > threshold_mm), i,
grid)
lon = nc.variables['lon'][islice]
lat = nc.variables['lat'][jslice]
mp = MapPlot(sector='state',
state=sector,
titlefontsize=14,
subtitlefontsize=12,
title=("NOAA MRMS Q3: Number of Recent Days "
"till Accumulating %s\" of Precip") % (threshold, ),
subtitle=("valid %s: based on per calendar day "
"estimated preciptation, GaugeCorr and "
"RadarOnly products") %
(date.strftime("%-d %b %Y"), ))
x, y = np.meshgrid(lon, lat)
cmap = plt.get_cmap(ctx['cmap'])
cmap.set_over('k')
cmap.set_under('white')
mp.pcolormesh(x, y, grid, np.arange(0, 81, 10), cmap=cmap, units='days')
mp.drawcounties()
mp.drawcities()
return mp.fig
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
year = ctx['year']
threshold = ctx['threshold']
period = ctx['period']
state = ctx['state']
pgconn = get_dbconn('postgis')
states = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom, state_abbr from states where state_abbr = %s
""",
pgconn,
params=(state, ),
index_col='state_abbr',
geom_col='the_geom')
nc = ncopen(iemre.get_daily_ncname(year))
precip = nc.variables['p01d']
czs = CachingZonalStats(iemre.AFFINE)
hasdata = np.zeros((nc.dimensions['lat'].size, nc.dimensions['lon'].size))
czs.gen_stats(hasdata, states['the_geom'])
for nav in czs.gridnav:
grid = np.ones((nav.ysz, nav.xsz))
grid[nav.mask] = 0.
jslice = slice(nav.y0, nav.y0 + nav.ysz)
islice = slice(nav.x0, nav.x0 + nav.xsz)
hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice,
islice])
hasdata = np.flipud(hasdata)
datapts = np.sum(np.where(hasdata > 0, 1, 0))
now = datetime.date(year, 1, 1)
now += datetime.timedelta(days=(period - 1))
ets = datetime.date(year, 12, 31)
today = datetime.date.today()
if ets > today:
ets = today
days = []
coverage = []
while now <= ets:
idx = iemre.daily_offset(now)
sevenday = np.sum(precip[(idx - period):idx, :, :], 0)
pday = np.where(hasdata > 0, sevenday[:, :], -1)
tots = np.sum(np.where(pday >= (threshold * 25.4), 1, 0))
days.append(now)
coverage.append(tots / float(datapts) * 100.0)
now += datetime.timedelta(days=1)
df = pd.DataFrame(dict(day=pd.Series(days), coverage=pd.Series(coverage)))
(fig, ax) = plt.subplots(1, 1)
ax.bar(days, coverage, fc='g', ec='g')
ax.set_title(
("%s IEM Estimated Areal Coverage Percent of %s\n"
" receiving %.2f inches of rain over trailing %s day period") %
(year, reference.state_names[state], threshold, period))
ax.set_ylabel("Areal Coverage [%]")
ax.xaxis.set_major_formatter(mdates.DateFormatter('%b\n%-d'))
ax.set_yticks(range(0, 101, 25))
ax.grid(True)
return fig, df
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
year = ctx['year']
thres = ctx['thres']
metric = distance(thres, 'IN').value('MM')
state = ctx['state'][:2]
sts = datetime.datetime(year, 10, 1)
ets = datetime.datetime(year + 1, 5, 1)
rows = []
pgconn = get_dbconn('postgis')
states = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom, state_abbr from states where state_abbr = %s
""", pgconn, params=(state, ), index_col='state_abbr',
geom_col='the_geom')
sidx = iemre.daily_offset(sts)
ncfn = iemre.get_daily_ncname(sts.year)
if not os.path.isfile(ncfn):
raise ValueError("Data for year %s not found" % (sts.year, ))
nc = ncopen(ncfn)
czs = CachingZonalStats(iemre.AFFINE)
hasdata = np.zeros((nc.dimensions['lat'].size,
nc.dimensions['lon'].size))
czs.gen_stats(hasdata, states['the_geom'])
for nav in czs.gridnav:
grid = np.ones((nav.ysz, nav.xsz))
grid[nav.mask] = 0.
jslice = slice(nav.y0, nav.y0 + nav.ysz)
islice = slice(nav.x0, nav.x0 + nav.xsz)
hasdata[jslice, islice] = np.where(grid > 0, 1,
hasdata[jslice, islice])
st = np.flipud(hasdata)
stpts = np.sum(np.where(hasdata > 0, 1, 0))
snowd = nc.variables['snowd_12z'][sidx:, :, :]
nc.close()
for i in range(snowd.shape[0]):
rows.append({
'valid': sts + datetime.timedelta(days=i),
'coverage': f(st, snowd[i], metric, stpts),
})
eidx = iemre.daily_offset(ets)
nc = ncopen(iemre.get_daily_ncname(ets.year))
snowd = nc.variables['snowd_12z'][:eidx, :, :]
nc.close()
for i in range(snowd.shape[0]):
rows.append({
'valid': datetime.date(ets.year, 1, 1) + datetime.timedelta(days=i),
'coverage': f(st, snowd[i], metric, stpts),
})
df = pd.DataFrame(rows)
df = df[np.isfinite(df['coverage'])]
(fig, ax) = plt.subplots(1, 1, sharex=True, figsize=(8, 6))
ax.bar(df['valid'].values, df['coverage'].values, fc='tan', ec='tan',
align='center')
ax.set_title(("IEM Estimated Areal Snow Coverage Percent of %s\n"
" percentage of state reporting at least %.2fin snow"
" cover"
) % (reference.state_names[state], thres))
ax.set_ylabel("Areal Coverage [%]")
ax.xaxis.set_major_locator(mdates.DayLocator([1, 15]))
ax.xaxis.set_major_formatter(mdates.DateFormatter("%-d %b\n%Y"))
ax.set_yticks(range(0, 101, 25))
ax.grid(True)
return fig, df
def main():
"""Go Main Go"""
basedir = "/mnt/nrel/akrherz/livneh"
outdir = "swatfiles_livneh"
if os.path.isdir(outdir):
print("ABORT: as %s exists" % (outdir, ))
return
os.mkdir(outdir)
for dirname in ['precipitation', 'temperature']:
os.mkdir("%s/%s" % (outdir, dirname))
pgconn = get_dbconn('idep')
huc12df = gpd.GeoDataFrame.from_postgis("""
SELECT huc12, ST_Transform(simple_geom, %s) as geo from wbd_huc12
WHERE swat_use ORDER by huc12
""", pgconn, params=(PROJSTR,), index_col='huc12', geom_col='geo')
hucs = huc12df.index.values
years = range(1989, 2011)
nc = ncopen("%s/livneh_NAmerExt_15Oct2014.198109.nc" % (basedir, ))
# compute the affine
ncaffine = Affine(nc.variables['lon'][1] - nc.variables['lon'][0],
0.,
nc.variables['lon'][0],
0.,
nc.variables['lat'][0] - nc.variables['lat'][1],
nc.variables['lat'][-1])
czs = CachingZonalStats(ncaffine)
nc.close()
fps = []
for year in years:
for month in range(1, 13):
nc = ncopen(("%s/livneh_NAmerExt_15Oct2014.%s%02i.nc"
) % (basedir, year, month))
basedate, timesz = get_basedate(nc)
for i in tqdm(range(timesz), desc="%s%02i" % (year, month)):
tasmax = np.flipud(nc.variables['Tmax'][i, :, :])
tasmin = np.flipud(nc.variables['Tmin'][i, :, :])
pr = np.flipud(nc.variables['Prec'][i, :, :])
mytasmax = czs.gen_stats(tasmax, huc12df['geo'])
mytasmin = czs.gen_stats(tasmin, huc12df['geo'])
mypr = czs.gen_stats(pr, huc12df['geo'])
for j, huc12 in enumerate(hucs):
if i == 0 and year == years[0] and month == 1:
fps.append([open(('%s/precipitation/P%s.txt'
) % (outdir, huc12), 'w'),
open(('%s/temperature/T%s.txt'
) % (outdir, huc12), 'w')])
fps[j][0].write(
"%s\n" % (basedate.strftime("%Y%m%d"), ))
fps[j][1].write(
"%s\n" % (basedate.strftime("%Y%m%d"), ))
fps[j][0].write(("%.1f\n"
) % (mypr[j]
if mypr[j] is not None else 0, ))
fps[j][1].write(("%.2f,%.2f\n"
) % (mytasmax[j], mytasmin[j]))
for fp in fps:
fp[0].close()
fp[1].close()
def main(argv):
"""Go Main Go"""
model = argv[1]
rcp = argv[2]
basedir = "/mnt/nrel/akrherz/loca/%s/16th/%s/r1i1p1" % (model, rcp)
outdir = "swatfiles_%s_%s" % (model, rcp)
if os.path.isdir(outdir):
print("ABORT: as %s exists" % (outdir, ))
return
os.mkdir(outdir)
for dirname in ['precipitation', 'temperature']:
os.mkdir("%s/%s" % (outdir, dirname))
pgconn = get_dbconn('idep')
huc12df = gpd.GeoDataFrame.from_postgis("""
SELECT huc12, ST_Transform(simple_geom, %s) as geo from wbd_huc12
WHERE swat_use ORDER by huc12
""", pgconn, params=(PROJSTR,), index_col='huc12', geom_col='geo')
hucs = huc12df.index.values
years = range(1989, 2011) if rcp == 'historical' else range(2039, 2061)
nc = netCDF4.Dataset(("%s/pr/pr_day_%s_%s_r1i1p1"
"_%.0f0101-%.0f1231.LOCA_2016-04-02.16th.nc"
) % (basedir, model, rcp, years[0], years[0]))
# compute the affine
ncaffine = Affine(nc.variables['lon'][1] - nc.variables['lon'][0],
0.,
nc.variables['lon'][0] - 360.,
0.,
nc.variables['lat'][0] - nc.variables['lat'][1],
nc.variables['lat'][-1])
czs = CachingZonalStats(ncaffine)
nc.close()
fps = []
for year in years:
# assume 2006-2010 is closely represented by rcp45
if year >= 2006 and year < 2011:
rcp = 'rcp45'
basedir = "/mnt/nrel/akrherz/loca/%s/16th/%s/r1i1p1" % (model, rcp)
pr_nc = netCDF4.Dataset(("%s/pr/pr_day_%s_%s_r1i1p1"
"_%.0f0101-%.0f1231.LOCA_2016-04-02.16th.nc"
) % (basedir, model, rcp, year, year))
tasmax_nc = netCDF4.Dataset(("%s/tasmax/tasmax_day_%s_%s_r1i1p1"
"_%.0f0101-%.0f1231.LOCA_"
"2016-04-02.16th.nc"
) % (basedir, model, rcp, year, year))
tasmin_nc = netCDF4.Dataset(("%s/tasmin/tasmin_day_%s_%s_r1i1p1"
"_%.0f0101-%.0f1231.LOCA_"
"2016-04-02.16th.nc"
) % (basedir, model, rcp, year, year))
basedate, timesz = get_basedate(pr_nc)
for i in tqdm(range(timesz), desc=str(year)):
# date = basedate + datetime.timedelta(days=i)
# keep array logic in top-down order
tasmax = np.flipud(
temperature(tasmax_nc.variables['tasmax'][i, :, :],
'K').value('C'))
tasmin = np.flipud(
temperature(tasmin_nc.variables['tasmin'][i, :, :],
'K').value('C'))
pr = np.flipud(pr_nc.variables['pr'][i, :, :])
mytasmax = czs.gen_stats(tasmax, huc12df['geo'])
mytasmin = czs.gen_stats(tasmin, huc12df['geo'])
mypr = czs.gen_stats(pr, huc12df['geo'])
for j, huc12 in enumerate(hucs):
if i == 0 and year == years[0]:
fps.append([open(('%s/precipitation/P%s.txt'
) % (outdir, huc12), 'w'),
open(('%s/temperature/T%s.txt'
) % (outdir, huc12), 'w')])
fps[j][0].write("%s\n" % (basedate.strftime("%Y%m%d"), ))
fps[j][1].write("%s\n" % (basedate.strftime("%Y%m%d"), ))
fps[j][0].write(("%.1f\n"
) % (mypr[j] * 86400., ))
fps[j][1].write(("%.2f,%.2f\n"
) % (mytasmax[j], mytasmin[j]))
for fp in fps:
fp[0].close()
fp[1].close()
def do_day(valid):
""" Process a day please """
idx = iemre.daily_offset(valid)
with ncopen(iemre.get_daily_ncname(valid.year), "r", timeout=300) as nc:
high = temperature(
nc.variables["high_tmpk_12z"][idx, :, :], "K"
).value("F")
low = temperature(nc.variables["low_tmpk_12z"][idx, :, :], "K").value(
"F"
)
precip = distance(nc.variables["p01d_12z"][idx, :, :], "MM").value(
"IN"
)
snow = distance(nc.variables["snow_12z"][idx, :, :], "MM").value("IN")
snowd = distance(nc.variables["snowd_12z"][idx, :, :], "MM").value(
"IN"
)
# build out the state mappers
pgconn = get_dbconn("postgis")
states = gpd.GeoDataFrame.from_postgis(
"""
SELECT the_geom, state_abbr from states
where state_abbr not in ('AK', 'HI', 'DC')
""",
pgconn,
index_col="state_abbr",
geom_col="the_geom",
)
czs = CachingZonalStats(iemre.AFFINE)
sthigh = czs.gen_stats(np.flipud(high), states["the_geom"])
stlow = czs.gen_stats(np.flipud(low), states["the_geom"])
stprecip = czs.gen_stats(np.flipud(precip), states["the_geom"])
stsnow = czs.gen_stats(np.flipud(snow), states["the_geom"])
stsnowd = czs.gen_stats(np.flipud(snowd), states["the_geom"])
statedata = {}
for i, state in enumerate(states.index.values):
statedata[state] = dict(
high=sthigh[i],
low=stlow[i],
precip=stprecip[i],
snow=stsnow[i],
snowd=stsnowd[i],
)
update_database(state + "0000", valid, statedata[state])
# build out climate division mappers
climdiv = gpd.GeoDataFrame.from_postgis(
"""
SELECT geom, iemid from climdiv
where st_abbrv not in ('AK', 'HI', 'DC')
""",
pgconn,
index_col="iemid",
geom_col="geom",
)
czs = CachingZonalStats(iemre.AFFINE)
sthigh = czs.gen_stats(np.flipud(high), climdiv["geom"])
stlow = czs.gen_stats(np.flipud(low), climdiv["geom"])
stprecip = czs.gen_stats(np.flipud(precip), climdiv["geom"])
stsnow = czs.gen_stats(np.flipud(snow), climdiv["geom"])
stsnowd = czs.gen_stats(np.flipud(snowd), climdiv["geom"])
for i, iemid in enumerate(climdiv.index.values):
row = dict(
high=sthigh[i],
low=stlow[i],
precip=stprecip[i],
snow=stsnow[i],
snowd=stsnowd[i],
)
# we must have temperature data
if row["high"] is np.ma.masked or row["low"] is np.ma.masked:
print(
("compute_0000 %s has missing temperature data, using state")
% (iemid,)
)
row = statedata[iemid[:2]]
update_database(iemid, valid, row)