def load_stage4():
""" It sucks, but we need to load the stage IV data to give us something
to benchmark the MRMS data against, to account for two things:
1) Wind Farms
2) Over-estimates
"""
printt("load_stage4() called...")
# The stage4 files store precip in the rears, so compute 1 AM
one_am = datetime.datetime(VALID.year, VALID.month, VALID.day, 12, 0)
one_am = one_am.replace(tzinfo=pytz.timezone("UTC"))
one_am = one_am.astimezone(pytz.timezone("America/Chicago"))
# stage4 data is stored in the rears, so 1am
one_am = one_am.replace(hour=1, minute=0, second=0)
# clever hack for CST/CDT
tomorrow = one_am + datetime.timedelta(hours=36)
tomorrow = tomorrow.replace(hour=1)
sts_tidx = iemre.hourly_offset(one_am)
ets_tidx = iemre.hourly_offset(tomorrow)
printt(("stage4 sts_tidx:%s[%s] ets_tidx:%s[%s]"
) % (sts_tidx, one_am, ets_tidx, tomorrow))
nc = ncopen("/mesonet/data/stage4/%s_stage4_hourly.nc" % (VALID.year, ))
p01m = nc.variables['p01m']
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
# crossing jan 1
if ets_tidx < sts_tidx:
printt("Exercise special stageIV logic for jan1!")
totals = np.sum(p01m[sts_tidx:, :, :], axis=0)
nc.close()
nc = ncopen(
"/mesonet/data/stage4/%s_stage4_hourly.nc" % (tomorrow.year, )
)
p01m = nc.variables['p01m']
totals += np.sum(p01m[:ets_tidx, :, :], axis=0)
else:
totals = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0)
nc.close()
if np.ma.max(totals) > 0:
pass
else:
print('No StageIV data found, aborting...')
sys.exit()
# set a small non-zero number to keep things non-zero
totals = np.where(totals > 0.001, totals, 0.001)
xaxis = np.arange(MYWEST, MYEAST, 0.01)
yaxis = np.arange(MYSOUTH, MYNORTH, 0.01)
xi, yi = np.meshgrid(xaxis, yaxis)
nn = NearestNDInterpolator((lons.flatten(), lats.flatten()),
totals.flatten())
STAGE4[:] = nn(xi, yi)
write_grid(STAGE4, 'stage4')
printt("load_stage4() finished...")
def init_year(ts):
"""
Create a new NetCDF file for a year of our specification!
"""
fp = iemre.get_daily_mrms_ncname(ts.year)
nc = ncopen(fp, 'a', timeout=300)
nc.createDimension('nv', 2)
lat = nc.variables['lat']
lat.bounds = "lat_bnds"
lat[:] = np.arange(iemre.SOUTH + 0.005, iemre.NORTH, 0.01)
lat_bnds = nc.createVariable('lat_bnds', np.float, ('lat', 'nv'))
lat_bnds[:, 0] = np.arange(iemre.SOUTH, iemre.NORTH, 0.01)
lat_bnds[:, 1] = np.arange(iemre.SOUTH + 0.01, iemre.NORTH + 0.01, 0.01)
lon = nc.variables['lon']
lon.bounds = "lon_bnds"
lon[:] = np.arange(iemre.WEST, iemre.EAST, 0.01)
lon_bnds = nc.createVariable('lon_bnds', np.float, ('lon', 'nv'))
lon_bnds[:, 0] = np.arange(iemre.WEST, iemre.EAST, 0.01)
lon_bnds[:, 1] = np.arange(iemre.WEST + 0.01, iemre.EAST + 0.01, 0.01)
nc.close()
def main():
"""Go Main Go"""
sts = datetime.datetime(2018, 4, 20, 0)
sts = sts.replace(tzinfo=pytz.utc)
ets = datetime.datetime(2018, 5, 11, 0)
ets = ets.replace(tzinfo=pytz.utc)
nc = ncopen(iemre.get_hourly_ncname(sts.year))
lons = nc.variables['lon'][:]
lats = nc.variables['lat'][:]
running = np.zeros((len(nc.dimensions['lat']), len(nc.dimensions['lon'])))
maxval = np.zeros((len(nc.dimensions['lat']), len(nc.dimensions['lon'])))
interval = datetime.timedelta(hours=1)
now = sts
i, j = iemre.find_ij(-93.61, 41.99)
while now < ets:
offset = iemre.hourly_offset(now)
p01m = np.sum(nc.variables['p01m'][offset - 24:offset], axis=0)
# 0.05in is 1.27 mm
this = np.where(p01m > THRESHOLD, 1, 0)
running = np.where(this == 1, 0, running + 1)
maxval = np.where(running > maxval, running, maxval)
print("%s %s %s" % (now, running[j, i], maxval[j, i]))
now += interval
# maxval = numpy.where(domain == 1, maxval, 1.e20)
m = plot.MapPlot(sector='midwest',
title=('Max Period '
'between 24 Hour 0.25+ inch Total Precipitation'),
subtitle=('Period of 20 Apr - 11 May 2018, '
'based on NCEP Stage IV data'))
extra = lons[-1] + (lons[-1] - lons[-2])
lons[-1] = extra
# lons = np.concatenate([lons, [extra, ]])
extra = lats[-1] + (lats[-1] - lats[-2])
lats[-1] = extra
# lats = np.concatenate([lats, [extra, ]])
lons, lats = np.meshgrid(lons, lats)
# m.pcolormesh(x, y, maxval / 24.0, numpy.arange(0,25,1), units='days')
maxval = np.where(maxval > 800, 73., maxval)
cmap = plt.get_cmap('terrain')
m.contourf(lons, lats, maxval / 24.0, np.arange(1, 11.1, 1), cmap=cmap,
units='days', clip_on=False)
m.postprocess(filename='test.png')
def main():
"""Go Main Go"""
nc = ncopen("/mesonet/data/iemre/1979_narr.nc")
data = np.sum(nc.variables['apcp'][:, :, :], axis=0)
m = MapPlot(sector='conus', axisbg='tan',
title=(""),
subtitle='',
titlefontsize=16)
t = distance(data, 'MM').value('IN')
m.pcolormesh(nc.variables['lon'][:], nc.variables['lat'][:], t,
np.arange(0, 60, 2), units='F')
m.postprocess(filename='test.png')
def main():
"""Go Main Go"""
for year in range(1893, 2018):
nc = ncopen(get_daily_ncname(year), 'a', timeout=600)
gridsize = nc.dimensions['lat'].size * nc.dimensions['lon'].size
for vname in ['p01d', 'high_tmpk', 'low_tmpk']:
for i in range(nc.variables[vname].shape[0]):
calday = nc.variables[vname][i, :, :]
# Get a count of missing values
missing = np.sum(calday.mask) / float(gridsize)
if missing > 0.5:
print("%s_12z->%s %s %.2f" % (vname, vname, i, missing))
nc.variables[vname][i, :, :] = (
nc.variables[vname + '_12z'][i, :, :]
)
nc.close()
def main(argv):
"""Run Main"""
hour = int(argv[1])
job = argv[2]
sector = argv[3]
netprod = argv[4]
# Load EET
netpng = Image.open("%s_%s_%s.gif" % (sector, netprod, job))
sz = (netpng.size[1], netpng.size[0])
net = (np.fromstring(netpng.tobytes(), np.uint8)).reshape(sz)
if sector == "US":
# Load up our tmpc surface data
nc = ncopen('data/ructemps.nc')
tmpc = nc.variables['tmpc'][hour, :, :]
tmpc = np.flipud(tmpc)
nc.close()
# mask out the net based on temperature
net = np.where(tmpc > 3.0, net, 15)
else:
# No Filtering... for now
net[:, :] = 15.
# Load N0Q
n0qpng = Image.open("%s_N0Q_%s.gif" % (sector, job))
n0q = (np.fromstring(n0qpng.tobytes(), np.uint8)).reshape(sz)
# Clean n0q
if netprod == 'EET':
n0q = np.where(net < 10, 0, n0q)
else:
n0q = np.where(net < 2, 0, n0q)
# Make the image, please!
png = Image.fromarray(n0q)
# png.putpalette( make_colorramp() )
png.putpalette(n0qpng.getpalette())
meta = PngImagePlugin.PngInfo()
meta.add_text('gentime',
datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
png.save('%s_N0Q_CLEAN_%s.png' % (sector, job), pnginfo=meta)
def main():
"""Go Main Go"""
form = cgi.FieldStorage()
ts1 = datetime.datetime.strptime(form.getfirst("date1"), "%Y-%m-%d")
ts2 = datetime.datetime.strptime(form.getfirst("date2"), "%Y-%m-%d")
if ts1 > ts2:
send_error("date1 larger than date2")
if ts1.year != ts2.year:
send_error("multi-year query not supported yet...")
# Make sure we aren't in the future
tsend = datetime.date.today()
if ts2.date() > tsend:
ts2 = datetime.datetime.now() - datetime.timedelta(days=1)
lat = float(form.getfirst("lat"))
lon = float(form.getfirst("lon"))
if lon < iemre.WEST or lon > iemre.EAST:
send_error("lon value outside of bounds: %s to %s" %
(iemre.WEST, iemre.EAST))
if lat < iemre.SOUTH or lat > iemre.NORTH:
send_error("lat value outside of bounds: %s to %s" %
(iemre.SOUTH, iemre.NORTH))
# fmt = form["format"][0]
i, j = iemre.find_ij(lon, lat)
offset1 = iemre.daily_offset(ts1)
offset2 = iemre.daily_offset(ts2) + 1
# Get our netCDF vars
with ncopen(iemre.get_daily_ncname(ts1.year)) as nc:
hightemp = datatypes.temperature(
nc.variables["high_tmpk"][offset1:offset2, j, i], "K").value("F")
high12temp = datatypes.temperature(
nc.variables["high_tmpk_12z"][offset1:offset2, j, i],
"K").value("F")
lowtemp = datatypes.temperature(
nc.variables["low_tmpk"][offset1:offset2, j, i], "K").value("F")
low12temp = datatypes.temperature(
nc.variables["low_tmpk_12z"][offset1:offset2, j, i],
"K").value("F")
precip = nc.variables["p01d"][offset1:offset2, j, i] / 25.4
precip12 = nc.variables["p01d_12z"][offset1:offset2, j, i] / 25.4
# Get our climatology vars
c2000 = ts1.replace(year=2000)
coffset1 = iemre.daily_offset(c2000)
c2000 = ts2.replace(year=2000)
coffset2 = iemre.daily_offset(c2000) + 1
with ncopen(iemre.get_dailyc_ncname()) as cnc:
chigh = datatypes.temperature(
cnc.variables["high_tmpk"][coffset1:coffset2, j, i],
"K").value("F")
clow = datatypes.temperature(
cnc.variables["low_tmpk"][coffset1:coffset2, j, i], "K").value("F")
cprecip = cnc.variables["p01d"][coffset1:coffset2, j, i] / 25.4
if ts1.year > 1980:
i2, j2 = prismutil.find_ij(lon, lat)
with ncopen("/mesonet/data/prism/%s_daily.nc" % (ts1.year, )) as nc:
prism_precip = nc.variables["ppt"][offset1:offset2, j2, i2] / 25.4
else:
prism_precip = [None] * (offset2 - offset1)
if ts1.year > 2010:
j2 = int((lat - iemre.SOUTH) * 100.0)
i2 = int((lon - iemre.WEST) * 100.0)
with ncopen(iemre.get_daily_mrms_ncname(ts1.year)) as nc:
mrms_precip = nc.variables["p01d"][offset1:offset2, j2, i2] / 25.4
else:
mrms_precip = [None] * (offset2 - offset1)
res = {"data": []}
for i in range(0, offset2 - offset1):
now = ts1 + datetime.timedelta(days=i)
res["data"].append({
"date": now.strftime("%Y-%m-%d"),
"mrms_precip_in": clean(mrms_precip[i]),
"prism_precip_in": clean(prism_precip[i]),
"daily_high_f": clean(hightemp[i]),
"12z_high_f": clean(high12temp[i]),
"climate_daily_high_f": clean(chigh[i]),
"daily_low_f": clean(lowtemp[i]),
"12z_low_f": clean(low12temp[i]),
"climate_daily_low_f": clean(clow[i]),
"daily_precip_in": clean(precip[i]),
"12z_precip_in": clean(precip12[i]),
"climate_daily_precip_in": clean(cprecip[i]),
})
ssw("Content-type: application/json\n\n")
ssw(json.dumps(res))
def grid_hour(ts):
"""
I proctor the gridding of data on an hourly basis
@param ts Timestamp of the analysis, we'll consider a 20 minute window
"""
pprint("grid_hour called...")
nc = ncopen(iemre.get_hourly_ncname(ts.year), "a", timeout=300)
domain = nc.variables["hasdata"][:, :]
nc.close()
ts0 = ts - datetime.timedelta(minutes=10)
ts1 = ts + datetime.timedelta(minutes=10)
utcnow = datetime.datetime.utcnow()
utcnow = utcnow.replace(tzinfo=pytz.utc) - datetime.timedelta(hours=36)
# If we are near realtime, look in IEMAccess instead of ASOS database
mybuf = 2.0
params = (
iemre.WEST - mybuf,
iemre.SOUTH - mybuf,
iemre.WEST - mybuf,
iemre.NORTH + mybuf,
iemre.EAST + mybuf,
iemre.NORTH + mybuf,
iemre.EAST + mybuf,
iemre.SOUTH - mybuf,
iemre.WEST - mybuf,
iemre.SOUTH - mybuf,
ts0,
ts1,
)
if utcnow < ts:
dbconn = get_dbconn("iem", user="******")
sql = """SELECT t.id as station, ST_x(geom) as lon,
ST_y(geom) as lat,
max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf,
max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt,
max(getskyc(skyc1)) as max_skyc1,
max(getskyc(skyc2)) as max_skyc2,
max(getskyc(skyc3)) as max_skyc3,
max(case when phour > 0 and phour < 1000 then phour else 0 end) as phour,
max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf,
max(case when sknt >= 0 then sknt else 0 end) as sknt,
max(case when sknt >= 0 then drct else 0 end) as drct
from current_log s JOIN stations t on (s.iemid = t.iemid)
WHERE ST_Contains(
ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s %s, %s %s, %s %s, %s %s))'),
geom) and valid >= %s and valid < %s GROUP by station, lon, lat
"""
else:
dbconn = get_dbconn("asos", user="******")
sql = """SELECT station, ST_x(geom) as lon, st_y(geom) as lat,
max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf,
max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt,
max(getskyc(skyc1)) as max_skyc1,
max(getskyc(skyc2)) as max_skyc2,
max(getskyc(skyc3)) as max_skyc3,
max(case when p01i > 0 and p01i < 1000 then p01i else 0 end) as phour,
max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf,
max(case when sknt >= 0 then sknt else 0 end) as sknt,
max(case when sknt >= 0 then drct else 0 end) as drct
from alldata a JOIN stations t on (a.station = t.id) WHERE
ST_Contains(
ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s %s, %s %s, %s %s, %s %s))'),
geom) and (t.network ~* 'ASOS' or t.network = 'AWOS') and
valid >= %s and valid < %s GROUP by station, lon, lat"""
df = read_sql(sql, dbconn, params=params, index_col="station")
pprint("got database results")
if df.empty:
print(("%s has no entries, FAIL") % (ts.strftime("%Y-%m-%d %H:%M"), ))
return
ures, vres = grid_wind(df, domain)
pprint("grid_wind is done")
if ures is None:
print("iemre.hourly_analysis failure for uwnd at %s" % (ts, ))
else:
write_grid(ts, "uwnd", ures)
write_grid(ts, "vwnd", vres)
tmpf = generic_gridder(df, "max_tmpf", domain)
pprint("grid tmpf is done")
if tmpf is None:
print("iemre.hourly_analysis failure for tmpk at %s" % (ts, ))
else:
dwpf = generic_gridder(df, "max_dwpf", domain)
pprint("grid dwpf is done")
# require that dwpk <= tmpk
mask = ~np.isnan(dwpf)
mask[mask] &= dwpf[mask] > tmpf[mask]
dwpf = np.where(mask, tmpf, dwpf)
write_grid(ts, "tmpk",
masked_array(tmpf, data_units="degF").to("degK"))
write_grid(ts, "dwpk",
masked_array(dwpf, data_units="degF").to("degK"))
res = grid_skyc(df, domain)
pprint("grid skyc is done")
if res is None:
print("iemre.hourly_analysis failure for skyc at %s" % (ts, ))
else:
write_grid(ts, "skyc", res)
def do(date):
""" Process for a given date
6z file has 6z to 9z data
"""
pgconn = get_dbconn("coop")
ccursor = pgconn.cursor()
ccursor2 = pgconn.cursor()
sts = date.replace(hour=6) # 6z
ets = sts + datetime.timedelta(days=1)
now = sts
interval = datetime.timedelta(hours=3)
mode = NC_MODE
lats, lons = None, None
while now < ets:
# See if we have Grib data first
fn = now.strftime(
"/mesonet/ARCHIVE/data/%Y/%m/%d/model/NARR/rad_%Y%m%d%H00.grib")
if os.path.isfile(fn):
mode = GRIB_MODE
grb = pygrib.open(fn)[1]
if lats is None:
lats, lons = grb.latlons()
total = grb["values"] * 10800.0
else:
total += grb["values"] * 10800.0
now += interval
continue
fn = now.strftime(
"/mesonet/ARCHIVE/data/%Y/%m/%d/model/NARR/rad_%Y%m%d%H00.nc")
if not os.path.isfile(fn):
LOG.info("MISSING NARR: %s", fn)
sys.exit()
with ncopen(fn, timeout=300) as nc:
rad = nc.variables["Downward_shortwave_radiation_flux"][0, :, :]
if now == sts:
xc = nc.variables["x"][:] * 1000.0 # convert to meters
yc = nc.variables["y"][:] * 1000.0 # convert to meters
total = rad * 10800.0 # 3 hr rad to total rad
else:
total += rad * 10800.0
now += interval
ccursor.execute(
"""
SELECT station, ST_x(geom), ST_y(geom), temp24_hour from
alldata a JOIN stations t on
(a.station = t.id) where day = %s
""",
(date.strftime("%Y-%m-%d"), ),
)
for row in ccursor:
if mode == NC_MODE:
(x, y) = pyproj.transform(P4326, LCC, row[1], row[2])
(gridxs, gridys, distances) = get_gp(xc, yc, x, y)
else:
# Note the hacky reversing of grid coords
(gridys, gridxs, distances) = get_gp2(lons, lats, row[1], row[2])
z0 = total[gridys[0], gridxs[0]]
z1 = total[gridys[1], gridxs[1]]
z2 = total[gridys[2], gridxs[2]]
z3 = total[gridys[3], gridxs[3]]
val = (z0 / distances[0] + z1 / distances[1] + z2 / distances[2] +
z3 / distances[3]) / (1.0 / distances[0] + 1.0 / distances[1] +
1.0 / distances[2] + 1.0 / distances[3])
rad_mj = float(val) / 1000000.0
if rad_mj < 0:
LOG.info("WHOA! Negative RAD: %.2f, station: %s", rad_mj, row[0])
continue
if np.isnan(rad_mj):
LOG.info("NARR SRAD is NaN, station: %s", row[0])
rad_mj = None
# if our station is 12z, then this day's data goes into 'tomorrow'
# if our station is not, then this day is today
date2 = date.strftime("%Y-%m-%d")
if row[3] in range(4, 13):
date2 = (date + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
ccursor2.execute(
f"UPDATE alldata_{row[0][:2]} SET narr_srad = %s WHERE "
"day = %s and station = %s",
(rad_mj, date2, row[0]),
)
ccursor2.close()
pgconn.commit()
pgconn.close()
def plotter(fdict):
""" Go """
ctx = util.get_autoplot_context(fdict, get_description())
date = ctx['date']
sector = ctx['sector']
days = ctx['trailing']
threshold = ctx['threshold']
window_sts = date - datetime.timedelta(days=days)
if window_sts.year != date.year:
raise ValueError('Sorry, do not support multi-year plots yet!')
if len(sector) != 2:
raise ValueError("Sorry, this does not support multi-state plots yet.")
idx0 = iemre.daily_offset(window_sts)
idx1 = iemre.daily_offset(date)
ncfn = iemre.get_daily_mrms_ncname(date.year)
ncvar = 'p01d'
if not os.path.isfile(ncfn):
raise ValueError("No data for that year, sorry.")
# Get the state weight
df = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom from states where state_abbr = %s
""",
util.get_dbconn('postgis'),
params=(sector, ),
index_col=None,
geom_col='the_geom')
czs = CachingZonalStats(iemre.MRMS_AFFINE)
with util.ncopen(ncfn) as nc:
czs.gen_stats(
np.zeros((nc.variables['lat'].size, nc.variables['lon'].size)),
df['the_geom'])
hasdata = None
jslice = None
islice = None
for nav in czs.gridnav:
hasdata = np.ones((nav.ysz, nav.xsz))
hasdata[nav.mask] = 0.
# careful here as y is flipped in this context
jslice = slice(nc.variables['lat'].size - (nav.y0 + nav.ysz),
nc.variables['lat'].size - nav.y0)
islice = slice(nav.x0, nav.x0 + nav.xsz)
hasdata = np.flipud(hasdata)
today = distance(nc.variables[ncvar][idx1, jslice, islice],
'MM').value('IN')
if (idx1 - idx0) < 32:
p01d = distance(
np.sum(nc.variables[ncvar][idx0:idx1, jslice, islice], 0),
'MM').value('IN')
else:
# Too much data can overwhelm this app, need to chunk it
for i in range(idx0, idx1, 10):
i2 = min([i + 10, idx1])
if idx0 == i:
p01d = distance(
np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0),
'MM').value('IN')
else:
p01d += distance(
np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0),
'MM').value('IN')
# Get climatology
with util.ncopen(iemre.get_dailyc_mrms_ncname()) as nc:
if (idx1 - idx0) < 32:
c_p01d = distance(
np.sum(nc.variables[ncvar][idx0:idx1, jslice, islice], 0),
'MM').value('IN')
else:
# Too much data can overwhelm this app, need to chunk it
for i in range(idx0, idx1, 10):
i2 = min([i + 10, idx1])
if idx0 == i:
c_p01d = distance(
np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0),
'MM').value('IN')
else:
c_p01d += distance(
np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0),
'MM').value('IN')
# we actually don't care about weights at this fine of scale
cells = np.sum(np.where(hasdata > 0, 1, 0))
departure = p01d - c_p01d
# Update departure and today to values unconsidered below when out of state
departure = np.where(hasdata > 0, departure, -9999)
today = np.where(hasdata > 0, today, 0)
ranges = [[-99, -3], [-3, -2], [-2, -1], [-1, 0], [0, 1], [1, 2], [2, 3],
[3, 99]]
x = []
x2 = []
labels = []
for (minv, maxv) in ranges:
labels.append("%.0f to %.0f" % (minv, maxv))
# How many departure cells in this range
hits = np.logical_and(departure < maxv, departure > minv)
hits2 = np.logical_and(hits, today > threshold)
x.append(np.sum(np.where(hits, 1, 0)) / float(cells) * 100.)
x2.append(np.sum(np.where(hits2, 1, 0)) / float(cells) * 100.)
(fig, ax) = plt.subplots(1, 1)
ax.set_title(("%s NOAA MRMS %s %.2f inch Precip Coverage") %
(state_names[sector], date.strftime("%-d %b %Y"), threshold))
ax.bar(np.arange(8) - 0.2,
x,
align='center',
width=0.4,
label='Trailing %s Day Departure' % (days, ))
ax.bar(np.arange(8) + 0.2,
x2,
align='center',
width=0.4,
label='%s Coverage (%.1f%% Tot)' %
(date.strftime("%-d %b %Y"), sum(x2)))
for i, (_x1, _x2) in enumerate(zip(x, x2)):
ax.text(i - 0.2, _x1 + 1, "%.1f" % (_x1, ), ha='center')
ax.text(i + 0.2, _x2 + 1, "%.1f" % (_x2, ), ha='center')
ax.set_xticks(np.arange(8))
ax.set_xticklabels(labels)
ax.set_xlabel("Trailing %s Day Precip Departure [in]" % (days, ))
ax.set_position([0.1, 0.2, 0.8, 0.7])
ax.legend(loc=(0., -0.2), ncol=2)
ax.set_ylabel("Areal Coverage of %s [%%]" % (state_names[sector], ))
ax.grid(True)
ax.set_xlim(-0.5, 7.5)
ax.set_ylim(0, max([max(x2), max(x)]) + 5)
return fig
def workflow(valid):
"""Our workflow"""
if valid.month == 1 and valid.day == 1:
print("prism_adjust_stage4, sorry Jan 1 processing is a TODO!")
return
# read prism
tidx = daily_offset(valid)
nc = ncopen("/mesonet/data/prism/%s_daily.nc" % (valid.year, ), "r")
ppt = nc.variables["ppt"][tidx, :, :]
# missing as zero
ppt = np.where(ppt.mask, 0, ppt)
lons = nc.variables["lon"][:]
lats = nc.variables["lat"][:]
nc.close()
(lons, lats) = np.meshgrid(lons, lats)
(i, j) = prismutil.find_ij(DEBUGLON, DEBUGLAT)
LOG.debug("prism debug point ppt: %.3f", ppt[j, i])
# Interpolate this onto the stage4 grid
nc = ncopen(
("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ),
"a",
timeout=300,
)
p01m = nc.variables["p01m"]
p01m_status = nc.variables["p01m_status"]
s4lons = nc.variables["lon"][:]
s4lats = nc.variables["lat"][:]
i, j = find_ij(s4lons, s4lats, DEBUGLON, DEBUGLAT)
# Values are in the hourly arrears, so start at -23 and thru current hour
sts_tidx = hourly_offset(valid - datetime.timedelta(hours=23))
ets_tidx = hourly_offset(valid + datetime.timedelta(hours=1))
s4total = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0)
LOG.debug(
"stage4 s4total: %.3f lon: %.2f (%.2f) lat: %.2f (%.2f)",
s4total[i, j],
s4lons[i, j],
DEBUGLON,
s4lats[i, j],
DEBUGLAT,
)
# make sure the s4total does not have zeros
s4total = np.where(s4total < 0.001, 0.001, s4total)
nn = NearestNDInterpolator((lons.flatten(), lats.flatten()), ppt.flat)
prism_on_s4grid = nn(s4lons, s4lats)
LOG.debug(
"shape of prism_on_s4grid: %s s4lons: %s ll: %.2f s4lats: %s ll: %.2f",
np.shape(prism_on_s4grid),
np.shape(s4lons),
s4lons[0, 0],
np.shape(s4lats),
s4lats[0, 0],
)
multiplier = prism_on_s4grid / s4total
LOG.debug(
"prism avg: %.3f stageIV avg: %.3f prismons4grid avg: %.3f mul: %.3f",
np.mean(ppt),
np.mean(s4total),
np.mean(prism_on_s4grid),
np.mean(multiplier),
)
LOG.debug(
"Boone IA0807 prism: %.3f stageIV: %.4f prismons4grid: %.3f mul: %.3f",
ppt[431, 746],
s4total[i, j],
prism_on_s4grid[i, j],
multiplier[i, j],
)
# Do the work now, we should not have to worry about the scale factor
for tidx in range(sts_tidx, ets_tidx):
oldval = p01m[tidx, :, :]
# we threshold the s4total to at least 0.001, so we divide by 24 here
# and denote that if the multiplier is zero, then we net zero
newval = np.where(oldval < 0.001, 0.00004, oldval) * multiplier
nc.variables["p01m"][tidx, :, :] = newval
LOG.debug(
"adjust tidx: %s oldval: %.3f newval: %.3f",
tidx,
oldval[i, j],
newval[i, j],
)
# make sure have data
if np.ma.max(newval) > 0:
p01m_status[tidx] = 2
else:
print(("prism_adjust_stage4 NOOP for time %s[idx:%s]") % (
(datetime.datetime(valid.year, 1, 1, 0) +
datetime.timedelta(hours=tidx)).strftime("%Y-%m-%dT%H"),
tidx,
))
nc.close()
def create_netcdf(valid):
"""Create and return the netcdf file"""
ncfn = "/mesonet/data/iemre/cfs_%s.nc" % (valid.strftime("%Y%m%d%H"), )
nc = ncopen(ncfn, "w")
nc.title = "IEM Regridded CFS Member 1 Forecast %s" % (valid.year, )
nc.platform = "Grided Forecast"
nc.description = "IEM Regridded CFS on 0.125 degree grid"
nc.institution = "Iowa State University, Ames, IA, USA"
nc.source = "Iowa Environmental Mesonet"
nc.project_id = "IEM"
nc.realization = 1
nc.Conventions = "CF-1.0"
nc.contact = "Daryl Herzmann, [email protected], 515-294-5978"
nc.history = ("%s Generated") % (
datetime.datetime.now().strftime("%d %B %Y"), )
nc.comment = "No comment at this time"
# Setup Dimensions
nc.createDimension("lat", iemre.NY)
nc.createDimension("lon", iemre.NX)
days = iemre.daily_offset(valid.replace(month=12, day=31)) + 1
nc.createDimension("time", int(days))
# Setup Coordinate Variables
lat = nc.createVariable("lat", np.float, ("lat", ))
lat.units = "degrees_north"
lat.long_name = "Latitude"
lat.standard_name = "latitude"
lat.axis = "Y"
lat[:] = iemre.YAXIS
lon = nc.createVariable("lon", np.float, ("lon", ))
lon.units = "degrees_east"
lon.long_name = "Longitude"
lon.standard_name = "longitude"
lon.axis = "X"
lon[:] = iemre.XAXIS
tm = nc.createVariable("time", np.float, ("time", ))
tm.units = "Days since %s-01-01 00:00:0.0" % (valid.year, )
tm.long_name = "Time"
tm.standard_name = "time"
tm.axis = "T"
tm.calendar = "gregorian"
tm[:] = np.arange(0, int(days))
high = nc.createVariable("high_tmpk",
np.uint16, ("time", "lat", "lon"),
fill_value=65535)
high.units = "K"
high.scale_factor = 0.01
high.long_name = "2m Air Temperature Daily High"
high.standard_name = "2m Air Temperature"
high.coordinates = "lon lat"
low = nc.createVariable("low_tmpk",
np.uint16, ("time", "lat", "lon"),
fill_value=65535)
low.units = "K"
low.scale_factor = 0.01
low.long_name = "2m Air Temperature Daily Low"
low.standard_name = "2m Air Temperature"
low.coordinates = "lon lat"
p01d = nc.createVariable("p01d",
np.uint16, ("time", "lat", "lon"),
fill_value=65535)
p01d.units = "mm"
p01d.scale_factor = 0.01
p01d.long_name = "Precipitation"
p01d.standard_name = "Precipitation"
p01d.coordinates = "lon lat"
p01d.description = "Precipitation accumulation for the day"
rsds = nc.createVariable("srad",
np.uint16, ("time", "lat", "lon"),
fill_value=65535)
rsds.units = "W m-2"
rsds.scale_factor = 0.01
rsds.long_name = "surface_downwelling_shortwave_flux_in_air"
rsds.standard_name = "surface_downwelling_shortwave_flux_in_air"
rsds.coordinates = "lon lat"
rsds.description = "Global Shortwave Irradiance"
nc.close()
nc = ncopen(ncfn, "a")
return nc
def create_netcdf(ts):
"""Make the file."""
fn = ts.strftime("/mesonet/data/ndfd/%Y%m%d%H_ndfd.nc")
nc = ncopen(fn, "w")
nc.title = "NWS NDFD Forecast From %s" % (ts.strftime("%Y-%m-%dT%H:%MZ"), )
nc.contact = "Daryl Herzmann, [email protected], 515-294-5978"
nc.history = "%s Generated" % (utc().strftime("%Y-%m-%dT%H:%MZ"), )
grbs = pygrib.open(("/mesonet/ARCHIVE/data/2019/05/06/model/"
"ndfd/00/ndfd.t00z.awp2p5f001.grib2"))
grb = grbs[1]
shape = grb.values.shape
lats, lons = grb.latlons()
# Setup dimensions
nc.createDimension("lat", shape[0])
nc.createDimension("lon", shape[1])
nc.createDimension("day", 7)
# Setup Coordinate Variables
lat = nc.createVariable("lat", np.float, ("lat", "lon"))
lat.units = "degrees_north"
lat.long_name = "Latitude"
lat.standard_name = "latitude"
lat.axis = "Y"
lat[:] = lats
lon = nc.createVariable("lon", np.float, ("lat", "lon"))
lon.units = "degrees_east"
lon.long_name = "Longitude"
lon.standard_name = "longitude"
lon.axis = "X"
lon[:] = lons
day = nc.createVariable("day", np.float, ("day", ))
day.units = "days since %s-01-01" % (ts.year, )
doy = int(ts.strftime("%j")) - 1
day[:] = range(doy, doy + 7)
high = nc.createVariable("high_tmpk",
np.uint16, ("day", "lat", "lon"),
fill_value=65535)
high.units = "K"
high.scale_factor = 0.01
high.long_name = "2m Air Temperature Daily High"
high.standard_name = "2m Air Temperature"
high.coordinates = "lon lat"
low = nc.createVariable("low_tmpk",
np.uint16, ("day", "lat", "lon"),
fill_value=65535)
low.units = "K"
low.scale_factor = 0.01
low.long_name = "2m Air Temperature Daily Low"
low.standard_name = "2m Air Temperature"
low.coordinates = "lon lat"
p01d = nc.createVariable("p01d",
np.uint16, ("day", "lat", "lon"),
fill_value=65535)
p01d.units = "mm"
p01d.scale_factor = 0.01
p01d.long_name = "Precipitation"
p01d.standard_name = "Precipitation"
p01d.coordinates = "lon lat"
p01d.description = "Precipitation accumulation for the day"
nc.close()
return ncopen(fn, "a")
def main():
"""Go Main Go"""
# Run for a period of 121 days
ets = datetime.datetime.now() - datetime.timedelta(days=1)
sts = ets - datetime.timedelta(days=121)
# Get the normal accumm
cnc = ncopen(iemre.get_dailyc_ncname(), "r")
lons = cnc.variables["lon"][:]
lats = cnc.variables["lat"][:]
index0 = iemre.daily_offset(sts)
index1 = iemre.daily_offset(ets)
if index1 < index0: # Uh oh, we are spanning a year
clprecip = np.sum(cnc.variables["p01d"][:index1, :, :], 0)
clprecip = clprecip + np.sum(cnc.variables["p01d"][index0:, :, :], 0)
else:
clprecip = np.sum(cnc.variables["p01d"][index0:index1, :, :], 0)
cnc.close()
# Get the observed precip
if sts.year != ets.year: # spanner, darn
onc = ncopen(iemre.get_daily_ncname(sts.year))
obprecip = np.sum(onc.variables["p01d"][index0:, :, :], 0)
onc.close()
onc = ncopen(iemre.get_daily_ncname(ets.year))
obprecip = obprecip + np.sum(onc.variables["p01d"][:index1, :, :], 0)
onc.close()
else:
ncfn = iemre.get_daily_ncname(sts.year)
onc = ncopen(ncfn, "r")
obprecip = np.sum(onc.variables["p01d"][index0:index1, :, :], 0)
onc.close()
lons, lats = np.meshgrid(lons, lats)
# Plot departure from normal
mp = MapPlot(
sector="midwest",
title=("Precipitation Departure %s - %s") %
(sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")),
subtitle="based on IEM Estimates",
)
mp.pcolormesh(lons, lats, (obprecip - clprecip) / 25.4,
np.arange(-10, 10, 1))
mp.postprocess(pqstr="plot c 000000000000 summary/4mon_diff.png bogus png")
mp.close()
# Plot normals
mp = MapPlot(
sector="midwest",
title=("Normal Precipitation:: %s - %s") %
(sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")),
subtitle="based on IEM Estimates",
)
mp.pcolormesh(lons, lats, (clprecip) / 25.4, np.arange(0, 30, 2))
mp.postprocess(
pqstr="plot c 000000000000 summary/4mon_normals.png bogus png")
mp.close()
# Plot Obs
mp = MapPlot(
sector="midwest",
title=("Estimated Precipitation:: %s - %s") %
(sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")),
subtitle="based on IEM Estimates",
)
mp.pcolormesh(lons, lats, (obprecip) / 25.4, np.arange(0, 30, 2))
mp.postprocess(
pqstr="plot c 000000000000 summary/4mon_stage4obs.png bogus png")
mp.close()
def plotter(fdict):
""" Go """
ctx = util.get_autoplot_context(fdict, get_description())
date = ctx["date"]
sector = ctx["sector"]
threshold = ctx["threshold"]
threshold_mm = distance(threshold, "IN").value("MM")
window_sts = date - datetime.timedelta(days=90)
if window_sts.year != date.year:
raise NoDataFound("Sorry, do not support multi-year plots yet!")
# idx0 = iemre.daily_offset(window_sts)
idx1 = iemre.daily_offset(date)
ncfn = iemre.get_daily_mrms_ncname(date.year)
if not os.path.isfile(ncfn):
raise NoDataFound("No data found.")
ncvar = "p01d"
# Get the state weight
df = gpd.GeoDataFrame.from_postgis(
"""
SELECT the_geom from states where state_abbr = %s
""",
util.get_dbconn("postgis"),
params=(sector, ),
index_col=None,
geom_col="the_geom",
)
czs = CachingZonalStats(iemre.MRMS_AFFINE)
with util.ncopen(ncfn) as nc:
czs.gen_stats(
np.zeros((nc.variables["lat"].size, nc.variables["lon"].size)),
df["the_geom"],
)
jslice = None
islice = None
for nav in czs.gridnav:
# careful here as y is flipped in this context
jslice = slice(
nc.variables["lat"].size - (nav.y0 + nav.ysz),
nc.variables["lat"].size - nav.y0,
)
islice = slice(nav.x0, nav.x0 + nav.xsz)
grid = np.zeros(
(jslice.stop - jslice.start, islice.stop - islice.start))
total = np.zeros(
(jslice.stop - jslice.start, islice.stop - islice.start))
for i, idx in enumerate(range(idx1, idx1 - 90, -1)):
total += nc.variables[ncvar][idx, jslice, islice]
grid = np.where(np.logical_and(grid == 0, total > threshold_mm), i,
grid)
lon = nc.variables["lon"][islice]
lat = nc.variables["lat"][jslice]
mp = MapPlot(
sector="state",
state=sector,
titlefontsize=14,
subtitlefontsize=12,
title=("NOAA MRMS Q3: Number of Recent Days "
'till Accumulating %s" of Precip') % (threshold, ),
subtitle=("valid %s: based on per calendar day "
"estimated preciptation, MultiSensorPass2 and "
"RadarOnly products") % (date.strftime("%-d %b %Y"), ),
)
x, y = np.meshgrid(lon, lat)
cmap = get_cmap(ctx["cmap"])
cmap.set_over("k")
cmap.set_under("white")
mp.pcolormesh(x, y, grid, np.arange(0, 81, 10), cmap=cmap, units="days")
mp.drawcounties()
mp.drawcities()
return mp.fig
def main():
"""Go Main Go"""
basedir = "/mnt/nrel/akrherz/livneh"
outdir = "swatfiles_livneh"
if os.path.isdir(outdir):
print("ABORT: as %s exists" % (outdir, ))
return
os.mkdir(outdir)
for dirname in ['precipitation', 'temperature']:
os.mkdir("%s/%s" % (outdir, dirname))
pgconn = get_dbconn('idep')
huc12df = gpd.GeoDataFrame.from_postgis("""
SELECT huc12, ST_Transform(simple_geom, %s) as geo from wbd_huc12
WHERE swat_use ORDER by huc12
""", pgconn, params=(PROJSTR,), index_col='huc12', geom_col='geo')
hucs = huc12df.index.values
years = range(1989, 2011)
nc = ncopen("%s/livneh_NAmerExt_15Oct2014.198109.nc" % (basedir, ))
# compute the affine
ncaffine = Affine(nc.variables['lon'][1] - nc.variables['lon'][0],
0.,
nc.variables['lon'][0],
0.,
nc.variables['lat'][0] - nc.variables['lat'][1],
nc.variables['lat'][-1])
czs = CachingZonalStats(ncaffine)
nc.close()
fps = []
for year in years:
for month in range(1, 13):
nc = ncopen(("%s/livneh_NAmerExt_15Oct2014.%s%02i.nc"
) % (basedir, year, month))
basedate, timesz = get_basedate(nc)
for i in tqdm(range(timesz), desc="%s%02i" % (year, month)):
tasmax = np.flipud(nc.variables['Tmax'][i, :, :])
tasmin = np.flipud(nc.variables['Tmin'][i, :, :])
pr = np.flipud(nc.variables['Prec'][i, :, :])
mytasmax = czs.gen_stats(tasmax, huc12df['geo'])
mytasmin = czs.gen_stats(tasmin, huc12df['geo'])
mypr = czs.gen_stats(pr, huc12df['geo'])
for j, huc12 in enumerate(hucs):
if i == 0 and year == years[0] and month == 1:
fps.append([open(('%s/precipitation/P%s.txt'
) % (outdir, huc12), 'w'),
open(('%s/temperature/T%s.txt'
) % (outdir, huc12), 'w')])
fps[j][0].write(
"%s\n" % (basedate.strftime("%Y%m%d"), ))
fps[j][1].write(
"%s\n" % (basedate.strftime("%Y%m%d"), ))
fps[j][0].write(("%.1f\n"
) % (mypr[j]
if mypr[j] is not None else 0, ))
fps[j][1].write(("%.2f,%.2f\n"
) % (mytasmax[j], mytasmin[j]))
for fp in fps:
fp[0].close()
fp[1].close()
def main():
"""Go main"""
# Wow, why did I do it this way... better not change it as I bet the
# other end is ignoring the headers...
fmt = ("LAT,LONG,STN,DATE,TIME,T,TD,WCI,RH,THI,DIR,SPD,GST,ALT,SLP,VIS,"
"SKY,CEIL,CLD,SKYSUM,PR6,PR24,WX,MINT6,MAXT6,MINT24,MAXT24,AUTO,"
"PR1,PTMP1,PTMP2,PTMP3,PTMP4,SUBS1,SUBS2,SUBS3,SUBS4,STATIONNAME,")
format_tokens = fmt.split(",")
utc = datetime.datetime.utcnow()
fn = "/mesonet/data/madis/mesonet1/%s.nc" % (utc.strftime("%Y%m%d_%H00"), )
if not os.path.isfile(fn):
time.sleep(60)
fn = "/mesonet/data/madis/mesonet1/%s.nc" % (
utc.strftime("%Y%m%d_%H00"), )
if not os.path.isfile(fn):
if utc.minute > 30:
print("madis2csv %s does not exist" % (fn, ))
sys.exit()
nc = ncopen(fn, "r", timeout=300)
if nc is None:
print(("madis2csv Numerous attempts to open MADIS netcdf %s failed!") %
(fn, ))
sys.exit(0)
stations = chartostring(nc.variables["stationId"][:])
stationname = chartostring(nc.variables["stationName"][:])
tmpf = temperature(nc.variables["temperature"][:], "K").value("F")
dwpf = temperature(nc.variables["dewpoint"][:], "K").value("F")
drct = nc.variables["windDir"][:]
smps = nc.variables["windSpeed"][:] * 1.94384449
alti = nc.variables["altimeter"][:] * 29.9196 / 1013.2 # in hPa
vsby = nc.variables["visibility"][:] * 0.000621371192 # in hPa
providers = chartostring(nc.variables["dataProvider"][:])
lat = nc.variables["latitude"][:]
lon = nc.variables["longitude"][:]
# ele = nc.variables["elevation"][:]
p01m = nc.variables["precipAccum"][:] * 25.4
ptmp1 = temperature(nc.variables["roadTemperature1"][:], "K").value("F")
ptmp2 = temperature(nc.variables["roadTemperature2"][:], "K").value("F")
ptmp3 = temperature(nc.variables["roadTemperature3"][:], "K").value("F")
ptmp4 = temperature(nc.variables["roadTemperature4"][:], "K").value("F")
subs1 = temperature(nc.variables["roadSubsurfaceTemp1"][:], "K").value("F")
subs2 = temperature(nc.variables["roadSubsurfaceTemp2"][:], "K").value("F")
subs3 = temperature(nc.variables["roadSubsurfaceTemp3"][:], "K").value("F")
subs4 = temperature(nc.variables["roadSubsurfaceTemp4"][:], "K").value("F")
times = nc.variables["observationTime"][:]
nc.close()
db = {}
for recnum in range(len(stations) - 1, -1, -1):
station = stations[recnum]
if station in db:
continue
ot = times[recnum]
if numpy.ma.is_masked(ot) or ot > 2141347600:
continue
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ot)
ts = ts.replace(tzinfo=pytz.UTC)
db[station] = {
"STN": station,
"PROVIDER": providers[recnum],
"LAT": lat[recnum],
"LONG": lon[recnum],
"STATIONNAME": stationname[recnum].replace(",", " "),
"DATE": ts.day,
"TIME": ts.strftime("%H%M"),
"T": sanity_check(tmpf[recnum], -100, 150, "%.0f", ""),
"DIR": sanity_check(drct[recnum], -1, 361, "%.0f", ""),
"TD": sanity_check(dwpf[recnum], -100, 150, "%.0f", ""),
"SPD": sanity_check(smps[recnum], -1, 150, "%.0f", ""),
"ALT": sanity_check(alti[recnum], 2000, 4000, "%.0f", ""),
"VIS": sanity_check(vsby[recnum], -1, 40, "%.0f", ""),
"PR1": sanity_check(p01m[recnum], -0.01, 10, "%.0f", ""),
"SUBS1": sanity_check(subs1[recnum], -100, 150, "%.0f", ""),
"SUBS2": sanity_check(subs2[recnum], -100, 150, "%.0f", ""),
"SUBS3": sanity_check(subs3[recnum], -100, 150, "%.0f", ""),
"SUBS4": sanity_check(subs4[recnum], -100, 150, "%.0f", ""),
"PTMP1": sanity_check(ptmp1[recnum], -100, 150, "%.0f", ""),
"PTMP2": sanity_check(ptmp2[recnum], -100, 150, "%.0f", ""),
"PTMP3": sanity_check(ptmp3[recnum], -100, 150, "%.0f", ""),
"PTMP4": sanity_check(ptmp4[recnum], -100, 150, "%.0f", ""),
}
out = open("/tmp/madis.csv", "w")
out.write("%s\n" % (fmt, ))
for stid in db:
for key in format_tokens:
if key in db[stid]:
out.write("%s" % (db[stid][key], ))
out.write(",")
out.write("\n")
out.close()
pqstr = "data c %s fn/madis.csv bogus csv" % (utc.strftime("%Y%m%d%H%M"), )
cmd = "pqinsert -i -p '%s' /tmp/madis.csv" % (pqstr, )
subprocess.call(cmd, shell=True)
os.remove("/tmp/madis.csv")
out = open("/tmp/madis.csv", "w")
out.write("%s\n" % (fmt, ))
for stid in db:
if db[stid]["PROVIDER"] in ["IADOT", "NEDOR"]:
for key in format_tokens:
if key in db[stid]:
out.write("%s" % (db[stid][key], ))
out.write(",")
out.write("\n")
out.close()
pqstr = "data c %s fn/madis_iamn.csv bogus csv" % (
utc.strftime("%Y%m%d%H%M"), )
cmd = "pqinsert -i -p '%s' /tmp/madis.csv" % (pqstr, )
subprocess.call(cmd, shell=True)
os.remove("/tmp/madis.csv")
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
year = ctx['year']
thres = ctx['thres']
metric = distance(thres, 'IN').value('MM')
state = ctx['state'][:2]
sts = datetime.datetime(year, 10, 1)
ets = datetime.datetime(year + 1, 5, 1)
rows = []
pgconn = get_dbconn('postgis')
states = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom, state_abbr from states where state_abbr = %s
""",
pgconn,
params=(state, ),
index_col='state_abbr',
geom_col='the_geom')
sidx = iemre.daily_offset(sts)
ncfn = iemre.get_daily_ncname(sts.year)
if not os.path.isfile(ncfn):
raise NoDataFound("Data for year %s not found" % (sts.year, ))
with ncopen(ncfn) as nc:
czs = CachingZonalStats(iemre.AFFINE)
hasdata = np.zeros(
(nc.dimensions['lat'].size, nc.dimensions['lon'].size))
czs.gen_stats(hasdata, states['the_geom'])
for nav in czs.gridnav:
grid = np.ones((nav.ysz, nav.xsz))
grid[nav.mask] = 0.
jslice = slice(nav.y0, nav.y0 + nav.ysz)
islice = slice(nav.x0, nav.x0 + nav.xsz)
hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice,
islice])
st = np.flipud(hasdata)
stpts = np.sum(np.where(hasdata > 0, 1, 0))
snowd = nc.variables['snowd_12z'][sidx:, :, :]
for i in range(snowd.shape[0]):
rows.append({
'valid': sts + datetime.timedelta(days=i),
'coverage': f(st, snowd[i], metric, stpts),
})
eidx = iemre.daily_offset(ets)
with ncopen(iemre.get_daily_ncname(ets.year)) as nc:
snowd = nc.variables['snowd_12z'][:eidx, :, :]
for i in range(snowd.shape[0]):
rows.append({
'valid':
datetime.date(ets.year, 1, 1) + datetime.timedelta(days=i),
'coverage':
f(st, snowd[i], metric, stpts),
})
df = pd.DataFrame(rows)
df = df[np.isfinite(df['coverage'])]
(fig, ax) = plt.subplots(1, 1, sharex=True, figsize=(8, 6))
ax.bar(df['valid'].values,
df['coverage'].values,
fc='tan',
ec='tan',
align='center')
ax.set_title(("IEM Estimated Areal Snow Coverage Percent of %s\n"
" percentage of state reporting at least %.2fin snow"
" cover") % (reference.state_names[state], thres))
ax.set_ylabel("Areal Coverage [%]")
ax.xaxis.set_major_locator(mdates.DayLocator([1, 15]))
ax.xaxis.set_major_formatter(mdates.DateFormatter("%-d %b\n%Y"))
ax.set_yticks(range(0, 101, 25))
ax.grid(True)
return fig, df
def append_cfs(res, lon, lat):
"""Append on needed CFS data."""
gridx, gridy = find_ij(lon, lat)
lastyear = max(res["data"].keys())
thisyear = datetime.date.today().year
lastdate = datetime.date(thisyear, 8, 31)
if lastyear != thisyear:
# We don't have any data yet for this year, so we add some
res["data"][thisyear] = {"dates": [], "high": [], "low": [], "rh": []}
else:
# shrug
if res["data"][lastyear]["dates"]:
lastdate = datetime.datetime.strptime(
res["data"][thisyear]["dates"][-1], "%Y-%m-%d"
).date()
# go find the most recent CFS 0z file
valid = datetime.date.today()
attempt = 0
while True:
testfn = valid.strftime("/mesonet/data/iemre/cfs_%Y%m%d00.nc")
if os.path.isfile(testfn):
break
valid -= datetime.timedelta(hours=24)
attempt += 1
if attempt > 9:
return None
try:
nc = ncopen(testfn, timeout=NCOPEN_TIMEOUT)
except Exception as exp:
LOG.error(exp)
return None
if nc is None:
LOG.debug("Failing %s as nc is None", testfn)
return None
high = (
masked_array(nc.variables["high_tmpk"][:, gridy, gridx], units.degK)
.to(units.degF)
.m
)
low = (
masked_array(nc.variables["low_tmpk"][:, gridy, gridx], units.degK)
.to(units.degF)
.m
)
# RH hack
# found ~20% bias with this value, so arb addition for now
rh = (
relative_humidity_from_dewpoint(
masked_array(high, units.degF), masked_array(low, units.degF)
).m
* 100.0
+ 20.0
)
rh = np.where(rh > 95, 95, rh)
entry = res["data"][thisyear]
# lastdate is either August 31 or a date after, so our first forecast
# date is i+1
tidx = daily_offset(lastdate + datetime.timedelta(days=1))
for i in range(tidx, 365):
lts = datetime.date(thisyear, 1, 1) + datetime.timedelta(days=i)
if lts.month in [9, 10, 11]:
entry["dates"].append(lts.strftime("%Y-%m-%d"))
entry["high"].append(_i(high[i]))
entry["low"].append(_i(low[i]))
entry["rh"].append(_i(rh[i]))
return res
def init_year(ts):
"""
Create a new NetCDF file for a year of our specification!
"""
fp = iemre.get_dailyc_ncname()
nc = ncopen(fp, "w")
nc.title = "IEM Daily Reanalysis Climatology %s" % (ts.year, )
nc.platform = "Grided Climatology"
nc.description = "IEM daily analysis on a 0.125 degree grid"
nc.institution = "Iowa State University, Ames, IA, USA"
nc.source = "Iowa Environmental Mesonet"
nc.project_id = "IEM"
nc.realization = 1
nc.Conventions = "CF-1.0" # *cough*
nc.contact = "Daryl Herzmann, [email protected], 515-294-5978"
nc.history = "%s Generated" % (
datetime.datetime.now().strftime("%d %B %Y"), )
nc.comment = "No Comment at this time"
# Setup Dimensions
nc.createDimension("lat", iemre.NY)
nc.createDimension("lon", iemre.NX)
ts2 = datetime.datetime(ts.year + 1, 1, 1)
days = (ts2 - ts).days
nc.createDimension("time", int(days))
# Setup Coordinate Variables
lat = nc.createVariable("lat", np.float, ("lat", ))
lat.units = "degrees_north"
lat.long_name = "Latitude"
lat.standard_name = "latitude"
lat.axis = "Y"
lat[:] = iemre.YAXIS
lon = nc.createVariable("lon", np.float, ("lon", ))
lon.units = "degrees_east"
lon.long_name = "Longitude"
lon.standard_name = "longitude"
lon.axis = "X"
lon[:] = iemre.XAXIS
tm = nc.createVariable("time", np.float, ("time", ))
tm.units = "Days since %s-01-01 00:00:0.0" % (ts.year, )
tm.long_name = "Time"
tm.standard_name = "time"
tm.axis = "T"
tm.calendar = "gregorian"
tm[:] = np.arange(0, int(days))
# Tracked variables
hasdata = nc.createVariable("hasdata", np.int8, ("lat", "lon"))
hasdata.units = "1"
hasdata.long_name = "Analysis Available for Grid Cell"
hasdata.coordinates = "lon lat"
hasdata[:] = 0
high = nc.createVariable("high_tmpk",
np.float, ("time", "lat", "lon"),
fill_value=1.0e20)
high.units = "K"
high.long_name = "2m Air Temperature Daily High"
high.standard_name = "2m Air Temperature"
high.coordinates = "lon lat"
low = nc.createVariable("low_tmpk",
np.float, ("time", "lat", "lon"),
fill_value=1.0e20)
low.units = "K"
low.long_name = "2m Air Temperature Daily Low"
low.standard_name = "2m Air Temperature"
low.coordinates = "lon lat"
p01d = nc.createVariable("p01d",
np.float, ("time", "lat", "lon"),
fill_value=1.0e20)
p01d.units = "mm"
p01d.long_name = "Precipitation"
p01d.standard_name = "Precipitation"
p01d.coordinates = "lon lat"
p01d.description = "Precipitation accumulation for the day"
nc.close()
def plotter(fdict):
""" Go """
ctx = util.get_autoplot_context(fdict, get_description())
ptype = ctx["ptype"]
sdate = ctx["sdate"]
edate = ctx["edate"]
src = ctx["src"]
opt = ctx["opt"]
usdm = ctx["usdm"]
if sdate.year != edate.year:
raise NoDataFound("Sorry, do not support multi-year plots yet!")
days = (edate - sdate).days
sector = ctx["sector"]
if sdate == edate:
title = sdate.strftime("%-d %B %Y")
else:
title = "%s to %s (inclusive)" % (
sdate.strftime("%-d %b"),
edate.strftime("%-d %b %Y"),
)
x0 = 0
x1 = -1
y0 = 0
y1 = -1
state = None
if len(sector) == 2:
state = sector
sector = "state"
if src == "mrms":
ncfn = iemre.get_daily_mrms_ncname(sdate.year)
clncfn = iemre.get_dailyc_mrms_ncname()
ncvar = "p01d"
source = "MRMS Q3"
subtitle = "NOAA MRMS Project, GaugeCorr and RadarOnly"
elif src == "iemre":
ncfn = iemre.get_daily_ncname(sdate.year)
clncfn = iemre.get_dailyc_ncname()
ncvar = "p01d_12z"
source = "IEM Reanalysis"
subtitle = "IEM Reanalysis is derived from various NOAA datasets"
else:
ncfn = "/mesonet/data/prism/%s_daily.nc" % (sdate.year,)
clncfn = "/mesonet/data/prism/prism_dailyc.nc"
ncvar = "ppt"
source = "OSU PRISM"
subtitle = (
"PRISM Climate Group, Oregon State Univ., "
"http://prism.oregonstate.edu, created 4 Feb 2004."
)
mp = MapPlot(
sector=sector,
state=state,
axisbg="white",
nocaption=True,
title="%s:: %s Precip %s" % (source, title, PDICT3[opt]),
subtitle="Data from %s" % (subtitle,),
titlefontsize=14,
)
idx0 = iemre.daily_offset(sdate)
idx1 = iemre.daily_offset(edate) + 1
if not os.path.isfile(ncfn):
raise NoDataFound("No data for that year, sorry.")
with util.ncopen(ncfn) as nc:
if state is not None:
x0, y0, x1, y1 = util.grid_bounds(
nc.variables["lon"][:],
nc.variables["lat"][:],
state_bounds[state],
)
elif sector in SECTORS:
bnds = SECTORS[sector]
x0, y0, x1, y1 = util.grid_bounds(
nc.variables["lon"][:],
nc.variables["lat"][:],
[bnds[0], bnds[2], bnds[1], bnds[3]],
)
lats = nc.variables["lat"][y0:y1]
lons = nc.variables["lon"][x0:x1]
if sdate == edate:
p01d = mm2inch(nc.variables[ncvar][idx0, y0:y1, x0:x1])
elif (idx1 - idx0) < 32:
p01d = mm2inch(
np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0)
)
else:
# Too much data can overwhelm this app, need to chunk it
for i in range(idx0, idx1, 10):
i2 = min([i + 10, idx1])
if idx0 == i:
p01d = mm2inch(
np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0)
)
else:
p01d += mm2inch(
np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0)
)
if np.ma.is_masked(np.max(p01d)):
raise NoDataFound("Data Unavailable")
plot_units = "inches"
cmap = plt.get_cmap(ctx["cmap"])
cmap.set_bad("white")
if opt == "dep":
# Do departure work now
with util.ncopen(clncfn) as nc:
climo = mm2inch(
np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0)
)
p01d = p01d - climo
[maxv] = np.percentile(np.abs(p01d), [99])
clevs = np.around(np.linspace(0 - maxv, maxv, 11), decimals=2)
elif opt == "per":
with util.ncopen(clncfn) as nc:
climo = mm2inch(
np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0)
)
p01d = p01d / climo * 100.0
cmap.set_under("white")
cmap.set_over("black")
clevs = [1, 10, 25, 50, 75, 100, 125, 150, 200, 300, 500]
plot_units = "percent"
else:
p01d = np.where(p01d < 0.001, np.nan, p01d)
cmap.set_under("white")
clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
if days > 6:
clevs = [0.01, 0.3, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 10, 15, 20]
if days > 29:
clevs = [0.01, 0.5, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35]
if days > 90:
clevs = [0.01, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35, 40]
x2d, y2d = np.meshgrid(lons, lats)
if ptype == "c":
mp.contourf(
x2d, y2d, p01d, clevs, cmap=cmap, units=plot_units, iline=False
)
else:
res = mp.pcolormesh(x2d, y2d, p01d, clevs, cmap=cmap, units=plot_units)
res.set_rasterized(True)
if sector != "midwest":
mp.drawcounties()
mp.drawcities()
if usdm == "yes":
mp.draw_usdm(edate, filled=False, hatched=True)
return mp.fig
def main():
"""GO Main Go"""
pgconn = get_dbconn('iem')
icursor = pgconn.cursor()
utcnow = datetime.datetime.utcnow()
for i in range(10):
now = utcnow - datetime.timedelta(hours=i)
fn = "/mesonet/data/madis/mesonet1/%s.nc" % (
now.strftime("%Y%m%d_%H00"), )
if os.path.isfile(fn):
break
if not os.path.isfile(fn):
print('extract_madis.py found no files? last: %s' % (fn, ))
return
with ncopen(fn) as nc:
providers = chartostring(nc.variables["dataProvider"][:])
stations = chartostring(nc.variables['stationId'][:])
nc_tmpk = nc.variables["temperature"][:]
nc_dwpk = nc.variables["dewpoint"][:]
nc_alti = nc.variables["altimeter"][:]
tmpkqcd = nc.variables["temperatureQCD"][:]
dwpkqcd = nc.variables["dewpointQCD"][:]
altiqcd = nc.variables["altimeterQCD"][:]
times = nc.variables["observationTime"][:]
for p in range(providers.shape[0]):
provider = providers[p]
if provider not in ['IEM', 'IADOT']:
continue
ticks = int(times[p])
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ticks)
ts = ts.replace(tzinfo=pytz.UTC)
(tmpf, tmpf_qc_av, tmpf_qc_sc) = (None, None, None)
(dwpf, dwpf_qc_av, dwpf_qc_sc) = (None, None, None)
(alti, alti_qc_av, alti_qc_sc) = (None, None, None)
if not np.ma.is_masked(nc_tmpk[p]):
tmpf = check(temperature(nc_tmpk[p], 'K').value('F'))
tmpf_qc_av = figure(nc_tmpk[p], tmpkqcd[p, 0])
tmpf_qc_sc = figure(nc_tmpk[p], tmpkqcd[p, 6])
if not np.ma.is_masked(nc_dwpk[p]):
dwpf = check(temperature(nc_dwpk[p], 'K').value('F'))
dwpf_qc_av = figure(nc_dwpk[p], dwpkqcd[p, 0])
dwpf_qc_sc = figure(nc_dwpk[p], dwpkqcd[p, 6])
if not np.ma.is_masked(nc_alti[p]):
alti = check((nc_alti[p] / 100.0) * 0.0295298)
alti_qc_av = figure_alti(altiqcd[p, 0] * 0.0295298)
alti_qc_sc = figure_alti(altiqcd[p, 6] * 0.0295298)
sql = """
UPDATE current_qc SET tmpf = %s, tmpf_qc_av = %s,
tmpf_qc_sc = %s, dwpf = %s, dwpf_qc_av = %s,
dwpf_qc_sc = %s, alti = %s, alti_qc_av = %s,
alti_qc_sc = %s, valid = %s WHERE station = %s
"""
args = (tmpf, tmpf_qc_av, tmpf_qc_sc, dwpf, dwpf_qc_av, dwpf_qc_sc,
alti, alti_qc_av, alti_qc_sc, ts, stations[p])
icursor.execute(sql, args)
icursor.close()
pgconn.commit()
pgconn.close()
def plotter(fdict):
""" Go """
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
ctx = get_autoplot_context(fdict, get_description())
year = ctx['year']
threshold = ctx['threshold']
period = ctx['period']
state = ctx['state']
pgconn = get_dbconn('postgis')
states = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom, state_abbr from states where state_abbr = %s
""", pgconn, params=(state, ), index_col='state_abbr',
geom_col='the_geom')
nc = ncopen(iemre.get_daily_ncname(year))
precip = nc.variables['p01d']
czs = CachingZonalStats(iemre.AFFINE)
hasdata = np.zeros((nc.dimensions['lat'].size,
nc.dimensions['lon'].size))
czs.gen_stats(hasdata, states['the_geom'])
for nav in czs.gridnav:
grid = np.ones((nav.ysz, nav.xsz))
grid[nav.mask] = 0.
jslice = slice(nav.y0, nav.y0 + nav.ysz)
islice = slice(nav.x0, nav.x0 + nav.xsz)
hasdata[jslice, islice] = np.where(grid > 0, 1,
hasdata[jslice, islice])
hasdata = np.flipud(hasdata)
datapts = np.sum(np.where(hasdata > 0, 1, 0))
now = datetime.date(year, 1, 1)
now += datetime.timedelta(days=(period-1))
ets = datetime.date(year, 12, 31)
today = datetime.date.today()
if ets > today:
ets = today
days = []
coverage = []
while now <= ets:
idx = iemre.daily_offset(now)
sevenday = np.sum(precip[(idx-period):idx, :, :], 0)
pday = np.where(hasdata > 0, sevenday[:, :], -1)
tots = np.sum(np.where(pday >= (threshold * 25.4), 1, 0))
days.append(now)
coverage.append(tots / float(datapts) * 100.0)
now += datetime.timedelta(days=1)
df = pd.DataFrame(dict(day=pd.Series(days),
coverage=pd.Series(coverage)))
(fig, ax) = plt.subplots(1, 1)
ax.bar(days, coverage, fc='g', ec='g')
ax.set_title(("IEM Estimated Areal Coverage Percent of %s\n"
" receiving %.2f inches of rain over trailing %s day period"
) % (reference.state_names[state], threshold, period))
ax.set_ylabel("Areal Coverage [%]")
ax.xaxis.set_major_formatter(mdates.DateFormatter('%b\n%Y'))
ax.set_yticks(range(0, 101, 25))
ax.grid(True)
return fig, df
def main():
"""Do Something Fun!"""
form = cgi.FieldStorage()
ts = datetime.datetime.strptime(form.getfirst("date", "2019-03-01"),
"%Y-%m-%d")
lat = float(form.getfirst("lat", 41.99))
lon = float(form.getfirst("lon", -95.1))
fmt = form.getfirst("format", "json")
if fmt != 'json':
ssw("Content-type: text/plain\n\n")
ssw("ERROR: Service only emits json at this time")
return
i, j = iemre.find_ij(lon, lat)
offset = iemre.daily_offset(ts)
res = {
'data': [],
}
fn = iemre.get_daily_ncname(ts.year)
ssw('Content-type: application/json\n\n')
if not os.path.isfile(fn):
ssw(json.dumps(res))
sys.exit()
if i is None or j is None:
ssw(json.dumps({'error': 'Coordinates outside of domain'}))
return
if ts.year > 1980:
ncfn = "/mesonet/data/prism/%s_daily.nc" % (ts.year, )
if not os.path.isfile(ncfn):
prism_precip = None
else:
i2, j2 = prismutil.find_ij(lon, lat)
with ncopen(ncfn) as nc:
prism_precip = nc.variables['ppt'][offset, j2, i2] / 25.4
else:
prism_precip = None
if ts.year > 2010:
ncfn = iemre.get_daily_mrms_ncname(ts.year)
if not os.path.isfile(ncfn):
mrms_precip = None
else:
j2 = int((lat - iemre.SOUTH) * 100.0)
i2 = int((lon - iemre.WEST) * 100.0)
with ncopen(ncfn) as nc:
mrms_precip = nc.variables['p01d'][offset, j2, i2] / 25.4
else:
mrms_precip = None
c2000 = ts.replace(year=2000)
coffset = iemre.daily_offset(c2000)
with ncopen(fn) as nc:
with ncopen(iemre.get_dailyc_ncname()) as cnc:
res['data'].append({
'prism_precip_in':
myrounder(prism_precip, 2),
'mrms_precip_in':
myrounder(mrms_precip, 2),
'daily_high_f':
myrounder(
datatypes.temperature(
nc.variables['high_tmpk'][offset, j, i],
'K').value('F'), 1),
'12z_high_f':
myrounder(
datatypes.temperature(
nc.variables['high_tmpk_12z'][offset, j, i],
'K').value('F'), 1),
'climate_daily_high_f':
myrounder(
datatypes.temperature(
cnc.variables['high_tmpk'][coffset, j, i],
'K').value("F"), 1),
'daily_low_f':
myrounder(
datatypes.temperature(
nc.variables['low_tmpk'][offset, j, i],
'K').value("F"), 1),
'12z_low_f':
myrounder(
datatypes.temperature(
nc.variables['low_tmpk_12z'][offset, j, i],
'K').value('F'), 1),
'avg_dewpoint_f':
myrounder(
datatypes.temperature(
nc.variables['avg_dwpk'][offset, j, i],
'K').value('F'), 1),
'climate_daily_low_f':
myrounder(
datatypes.temperature(
cnc.variables['low_tmpk'][coffset, j, i],
'K').value("F"), 1),
'daily_precip_in':
myrounder(nc.variables['p01d'][offset, j, i] / 25.4, 2),
'12z_precip_in':
myrounder(nc.variables['p01d_12z'][offset, j, i] / 25.4, 2),
'climate_daily_precip_in':
myrounder(cnc.variables['p01d'][coffset, j, i] / 25.4, 2),
'srad_mj':
myrounder(
nc.variables['rsds'][offset, j, i] * 86400. / 1000000., 2),
'avg_windspeed_mps':
myrounder(nc.variables['wind_speed'][offset, j, i], 2),
})
ssw(json.dumps(res))
def plotter(fdict):
""" Go """
ctx = util.get_autoplot_context(fdict, get_description())
date = ctx['date']
sector = ctx['sector']
threshold = ctx['threshold']
threshold_mm = distance(threshold, 'IN').value('MM')
window_sts = date - datetime.timedelta(days=90)
if window_sts.year != date.year:
raise ValueError('Sorry, do not support multi-year plots yet!')
# idx0 = iemre.daily_offset(window_sts)
idx1 = iemre.daily_offset(date)
ncfn = iemre.get_daily_mrms_ncname(date.year)
ncvar = 'p01d'
nc = util.ncopen(ncfn)
if nc is None:
raise ValueError("No data for that year, sorry.")
# Get the state weight
df = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom from states where state_abbr = %s
""",
util.get_dbconn('postgis'),
params=(sector, ),
index_col=None,
geom_col='the_geom')
czs = CachingZonalStats(iemre.MRMS_AFFINE)
czs.gen_stats(
np.zeros((nc.variables['lat'].size, nc.variables['lon'].size)),
df['the_geom'])
jslice = None
islice = None
for nav in czs.gridnav:
# careful here as y is flipped in this context
jslice = slice(nc.variables['lat'].size - (nav.y0 + nav.ysz),
nc.variables['lat'].size - nav.y0)
islice = slice(nav.x0, nav.x0 + nav.xsz)
grid = np.zeros((jslice.stop - jslice.start, islice.stop - islice.start))
total = np.zeros((jslice.stop - jslice.start, islice.stop - islice.start))
for i, idx in enumerate(range(idx1, idx1 - 90, -1)):
total += nc.variables[ncvar][idx, jslice, islice]
grid = np.where(np.logical_and(grid == 0, total > threshold_mm), i,
grid)
lon = nc.variables['lon'][islice]
lat = nc.variables['lat'][jslice]
nc.close()
mp = MapPlot(sector='state',
state=sector,
titlefontsize=14,
subtitlefontsize=12,
title=("NOAA MRMS Q3: Number of Recent Days "
"till Accumulating %s\" of Precip") % (threshold, ),
subtitle=("valid %s: based on per calendar day "
"estimated preciptation, GaugeCorr and "
"RadarOnly products") %
(date.strftime("%-d %b %Y"), ))
x, y = np.meshgrid(lon, lat)
cmap = plt.get_cmap(ctx['cmap'])
cmap.set_over('k')
cmap.set_under('white')
mp.pcolormesh(x, y, grid, np.arange(0, 81, 10), cmap=cmap, units='days')
mp.drawcounties()
mp.drawcities()
return mp.fig
def application(environ, start_response):
"""Go main go"""
os.chdir("/tmp")
form = parse_formvars(environ)
ts0 = datetime.datetime.strptime(form.get("date0"), "%Y-%m-%d")
ts1 = datetime.datetime.strptime(form.get("date1"), "%Y-%m-%d")
base = int(form.get("base", 50))
ceil = int(form.get("ceil", 86))
# Make sure we aren't in the future
tsend = datetime.date.today()
if ts1.date() >= tsend:
ts1 = tsend - datetime.timedelta(days=1)
ts1 = datetime.datetime(ts1.year, ts1.month, ts1.day)
fmt = form.get("format")
offset0 = iemre.daily_offset(ts0)
offset1 = iemre.daily_offset(ts1)
with ncopen(iemre.get_daily_ncname(ts0.year)) as nc:
# 2-D precipitation, inches
precip = np.sum(nc.variables["p01d"][offset0:offset1, :, :] / 25.4,
axis=0)
# GDD
H = datatypes.temperature(nc.variables["high_tmpk"][offset0:offset1],
"K").value("F")
H = np.where(H < base, base, H)
H = np.where(H > ceil, ceil, H)
L = datatypes.temperature(nc.variables["low_tmpk"][offset0:offset1],
"K").value("F")
L = np.where(L < base, base, L)
gdd = np.sum((H + L) / 2.0 - base, axis=0)
if fmt == "json":
# For example: 19013
ugc = "IAC" + form.get("county")[2:]
# Go figure out where this is!
postgis = get_dbconn("postgis")
pcursor = postgis.cursor()
pcursor.execute(
"""
SELECT ST_x(ST_Centroid(geom)), ST_y(ST_Centroid(geom)) from ugcs WHERE
ugc = %s and end_ts is null
""",
(ugc, ),
)
row = pcursor.fetchone()
lat = row[1]
lon = row[0]
(i, j) = iemre.find_ij(lon, lat)
myGDD = gdd[j, i]
myPrecip = precip[j, i]
res = {"data": []}
res["data"].append({
"gdd": "%.0f" % (myGDD, ),
"precip": "%.1f" % (myPrecip, ),
"latitude": "%.4f" % (lat, ),
"longitude": "%.4f" % (lon, ),
})
headers = [("Content-type", "application/json")]
start_response("200 OK", headers)
return [json.dumps(res).encode("ascii")]
# Time to create the shapefiles
basefn = "iemre_%s_%s" % (ts0.strftime("%Y%m%d"), ts1.strftime("%Y%m"))
w = shapefile.Writer(basefn)
w.field("GDD", "F", 10, 2)
w.field("PREC_IN", "F", 10, 2)
for x in iemre.XAXIS:
for y in iemre.YAXIS:
w.poly([[
(x, y),
(x, y + iemre.DY),
(x + iemre.DX, y + iemre.DY),
(x + iemre.DX, y),
(x, y),
]])
for i in range(len(iemre.XAXIS)):
for j in range(len(iemre.YAXIS)):
w.record(gdd[j, i], precip[j, i])
w.close()
# Create zip file, send it back to the clients
shutil.copyfile("/opt/iem/data/gis/meta/4326.prj", "%s.prj" % (basefn, ))
z = zipfile.ZipFile("%s.zip" % (basefn, ), "w", zipfile.ZIP_DEFLATED)
for suffix in ["shp", "shx", "dbf", "prj"]:
z.write("%s.%s" % (basefn, suffix))
z.close()
headers = [
("Content-type", "application/octet-stream"),
("Content-Disposition", "attachment; filename=%s.zip" % (basefn, )),
]
start_response("200 OK", headers)
content = open(basefn + ".zip", "rb").read()
for suffix in ["zip", "shp", "shx", "dbf", "prj"]:
os.unlink("%s.%s" % (basefn, suffix))
return [content]
def main():
"""Go Main Go"""
pgconn = get_dbconn("coop")
ccursor = pgconn.cursor()
fn = sys.argv[1]
station = sys.argv[2]
with ncopen(fn) as nc:
byear = nc.variables["byear"][:]
maxt = nc.variables["maxt"][:]
mint = nc.variables["mint"][:]
pcpn = nc.variables["pcpn"][:]
snow = nc.variables["snow"][:]
snwg = nc.variables["snwg"][:]
current = read_sql(
"""
SELECT day, high, low, precip, snow, snowd from
alldata_ia WHERE station = %s ORDER by day ASC
""",
pgconn,
params=(station, ),
index_col="day",
)
added = 0
for yr in range(byear, 2016):
for mo in range(12):
for dy in range(31):
try:
date = datetime.date(yr, mo + 1, dy + 1)
except ValueError:
continue
high = maxt[yr - byear, mo, dy]
if (np.ma.is_masked(high) or np.isnan(high) or high < -100
or high > 150):
high = None
else:
high = int(high)
low = mint[yr - byear, mo, dy]
if (np.ma.is_masked(low) or np.isnan(low) or low < -100
or low > 150):
low = None
else:
low = int(low)
precip = convert(pcpn[yr - byear, mo, dy], 2)
snowfall = convert(snow[yr - byear, mo, dy], 1)
snowd = convert(snwg[yr - byear, mo, dy], 1)
if all(
[a is None for a in [high, low, precip, snowfall, snowd]]):
continue
if date not in current.index.values:
sday = "%02i%02i" % (date.month, date.day)
added += 1
ccursor.execute(
"""
INSERT into alldata_ia(station, day, high,
low, precip, snow, sday, year, month, snowd) VALUES
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
""",
(
station,
date,
high,
low,
precip,
snowfall,
sday,
int(date.year),
int(date.month),
snowd,
),
)
print("added %s" % (added, ))
ccursor.close()
pgconn.commit()
def init_year(ts):
"""
Create a new NetCDF file for a year of our specification!
"""
# Load up the example grib file to base our file on
grbs = pygrib.open(TEMPLATE_FN)
grb = grbs[1]
# grid shape is y, x
lats, lons = grb.latlons()
fp = "%s/%s_stage4_hourly.nc" % (BASEDIR, ts.year)
nc = ncopen(fp, "w")
nc.title = "IEM Packaged NOAA Stage IV for %s" % (ts.year, )
nc.platform = "Grided Estimates"
nc.description = "NOAA Stage IV on HRAP Grid"
nc.institution = "Iowa State University, Ames, IA, USA"
nc.source = "Iowa Environmental Mesonet"
nc.project_id = "IEM"
nc.realization = 1
nc.Conventions = "CF-1.0"
nc.contact = "Daryl Herzmann, [email protected], 515-294-5978"
nc.history = ("%s Generated") % (
datetime.datetime.now().strftime("%d %B %Y"), )
nc.comment = "No Comment at this time"
# Setup Dimensions
nc.createDimension("x", lats.shape[1])
nc.createDimension("y", lats.shape[0])
nc.createDimension("bnds", 2)
ts2 = datetime.datetime(ts.year + 1, 1, 1)
days = (ts2 - ts).days
print("Year %s has %s days" % (ts.year, days))
nc.createDimension("time", int(days) * 24)
# Setup Coordinate Variables
lat = nc.createVariable("lat", np.float, ("y", "x"))
lat.units = "degrees_north"
lat.long_name = "Latitude"
lat.standard_name = "latitude"
lat.axis = "Y"
lat[:] = lats
lon = nc.createVariable("lon", np.float, ("y", "x"))
lon.units = "degrees_east"
lon.long_name = "Longitude"
lon.standard_name = "longitude"
lon.axis = "X"
lon[:] = lons
tm = nc.createVariable("time", np.float, ("time", ))
tm.units = "Hours since %s-01-01 00:00:0.0" % (ts.year, )
tm.long_name = "Time"
tm.standard_name = "time"
tm.axis = "T"
tm.calendar = "gregorian"
tm[:] = np.arange(0, int(days) * 24)
tm.bounds = "time_bnds"
tmb = nc.createVariable("time_bnds", "d", ("time", "bnds"))
tmb[:, 0] = np.arange(0, int(days) * 24) - 1
tmb[:, 1] = np.arange(0, int(days) * 24)
p01m = nc.createVariable("p01m",
np.ushort, ("time", "y", "x"),
fill_value=65535)
p01m.scale_factor = 0.01
p01m.add_offset = 0.0
p01m.units = "mm"
p01m.long_name = "Precipitation"
p01m.standard_name = "Precipitation"
p01m.coordinates = "lon lat"
p01m.description = "Precipitation accumulation for the previous hour"
# Track variable status to prevent double writes of data, prior to bias
# correction making some changes
status = nc.createVariable("p01m_status",
np.int8, ("time", ),
fill_value=-1)
status.units = "1"
status.long_name = "p01m Variable Status"
status.description = "-1 unset, 1 grib data copied, 2 QC Run"
nc.close()
def process(df, ncfn, south, west):
"""Do some extraction."""
fn1 = "/mesonet/share/pickup/yieldfx/baseline/%s" % (ncfn, )
fn2 = "/mesonet/share/pickup/yieldfx/2019/%s" % (ncfn, )
if not os.path.isfile(fn1) or not os.path.isfile(fn2):
print("Missing %s or %s" % (fn1, fn2))
return
with ncopen(fn1) as nc:
with ncopen(fn2) as nc2019:
for _, row in df.iterrows():
if row["State"] not in [
"nd",
"sd",
"ne",
"ks",
"mo",
"ia",
"mn",
"wi",
"il",
"in",
"ky",
"oh",
"mi",
]:
continue
i = int((row["long"] - west) / 0.1250)
j = int((row["lat"] - south) / 0.1250)
nc_prcp = nc.variables["prcp"][:, j, i]
nc_tmax = nc.variables["tmax"][:, j, i]
nc_tmin = nc.variables["tmin"][:, j, i]
nc_srad = nc.variables["srad"][:, j, i]
nc2019_prcp = nc2019.variables["prcp"][:, j, i]
nc2019_tmax = nc2019.variables["tmax"][:, j, i]
nc2019_tmin = nc2019.variables["tmin"][:, j, i]
nc2019_srad = nc2019.variables["srad"][:, j, i]
fp = open(
("/mesonet/share/pickup/yieldfx/county/"
"%s_%s_%.4f_%.4f.csv") % (
row["State"].replace(" ", "_"),
row["County"].replace(" ", "_"),
0 - row["long"],
row["lat"],
),
"w",
)
fp.write(("year,yday,prcp (mm/day),srad (W/m^2),tmax (deg c),"
"tmin (deg c)\n"))
base = datetime.date(1980, 1, 1)
for tstep, days in enumerate(nc.variables["time"]):
ts = base + datetime.timedelta(days=int(days))
fp.write("%s,%s,%.3f,%.3f,%.3f,%.3f\n" % (
ts.year,
int(ts.strftime("%j")),
nc_prcp[tstep],
nc_srad[tstep] * 1e6 / 86400.0,
nc_tmax[tstep],
nc_tmin[tstep],
))
base = datetime.date(2019, 1, 1)
for tstep, days in enumerate(nc2019.variables["time"]):
ts = base + datetime.timedelta(days=int(days))
fp.write("%s,%s,%.3f,%.3f,%.3f,%.3f\n" % (
ts.year,
int(ts.strftime("%j")),
nc2019_prcp[tstep],
nc2019_srad[tstep] * 1e6 / 86400.0,
nc2019_tmax[tstep],
nc2019_tmin[tstep],
))
fp.close()
def init_year(ts):
"""
Create a new NetCDF file for a year of our specification!
"""
fn = iemre.get_daily_ncname(ts.year)
nc = ncopen(fn, 'w')
nc.title = "IEM Daily Reanalysis %s" % (ts.year, )
nc.platform = "Grided Observations"
nc.description = "IEM daily analysis on a 0.125 degree grid"
nc.institution = "Iowa State University, Ames, IA, USA"
nc.source = "Iowa Environmental Mesonet"
nc.project_id = "IEM"
nc.realization = 1
nc.Conventions = 'CF-1.0'
nc.contact = "Daryl Herzmann, [email protected], 515-294-5978"
nc.history = ("%s Generated") % (
datetime.datetime.now().strftime("%d %B %Y"), )
nc.comment = "No Comment at this time"
# Setup Dimensions
nc.createDimension('lat', iemre.NY)
nc.createDimension('lon', iemre.NX)
days = ((ts.replace(year=ts.year + 1)) - ts).days
nc.createDimension('time', int(days))
# Setup Coordinate Variables
lat = nc.createVariable('lat', np.float, ('lat', ))
lat.units = "degrees_north"
lat.long_name = "Latitude"
lat.standard_name = "latitude"
lat.axis = "Y"
lat[:] = iemre.YAXIS
lon = nc.createVariable('lon', np.float, ('lon', ))
lon.units = "degrees_east"
lon.long_name = "Longitude"
lon.standard_name = "longitude"
lon.axis = "X"
lon[:] = iemre.XAXIS
tm = nc.createVariable('time', np.float, ('time', ))
tm.units = "Days since %s-01-01 00:00:0.0" % (ts.year, )
tm.long_name = "Time"
tm.standard_name = "time"
tm.axis = "T"
tm.calendar = "gregorian"
tm[:] = np.arange(0, int(days))
# Tracked variables
hasdata = nc.createVariable('hasdata', np.int8, ('lat', 'lon'))
hasdata.units = '1'
hasdata.long_name = 'Analysis Available for Grid Cell'
hasdata.coordinates = "lon lat"
hasdata[:] = 0
high = nc.createVariable('high_tmpk',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
high.units = "K"
high.scale_factor = 0.01
high.long_name = "2m Air Temperature Daily High"
high.standard_name = "2m Air Temperature"
high.coordinates = "lon lat"
low = nc.createVariable('low_tmpk',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
low.units = "K"
low.scale_factor = 0.01
low.long_name = "2m Air Temperature Daily Low"
low.standard_name = "2m Air Temperature"
low.coordinates = "lon lat"
high12 = nc.createVariable('high_tmpk_12z',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
high12.units = "K"
high12.scale_factor = 0.01
high12.long_name = "2m Air Temperature 24 Hour Max at 12 UTC"
high12.standard_name = "2m Air Temperature"
high12.coordinates = "lon lat"
low12 = nc.createVariable('low_tmpk_12z',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
low12.units = "K"
low12.scale_factor = 0.01
low12.long_name = "2m Air Temperature 12 Hour Min at 12 UTC"
low12.standard_name = "2m Air Temperature"
low12.coordinates = "lon lat"
p01d = nc.createVariable('p01d',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
p01d.units = 'mm'
p01d.scale_factor = 0.01
p01d.long_name = 'Precipitation'
p01d.standard_name = 'Precipitation'
p01d.coordinates = "lon lat"
p01d.description = "Precipitation accumulation for the day"
p01d12 = nc.createVariable('p01d_12z',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
p01d12.units = 'mm'
p01d12.scale_factor = 0.01
p01d12.long_name = 'Precipitation'
p01d12.standard_name = 'Precipitation'
p01d12.coordinates = "lon lat"
p01d12.description = "24 Hour Precipitation Ending 12 UTC"
# 0 -> 65535 so 0 to 6553.5
rsds = nc.createVariable('rsds',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
rsds.units = "W m-2"
rsds.scale_factor = 0.1
rsds.long_name = 'surface_downwelling_shortwave_flux_in_air'
rsds.standard_name = 'surface_downwelling_shortwave_flux_in_air'
rsds.coordinates = "lon lat"
rsds.description = "Global Shortwave Irradiance"
snow = nc.createVariable('snow_12z',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
snow.units = 'mm'
snow.scale_factor = 0.01
snow.long_name = 'Snowfall'
snow.standard_name = 'Snowfall'
snow.coordinates = "lon lat"
snow.description = "Snowfall accumulation for the day"
# 0 to 6553.5
snowd = nc.createVariable('snowd_12z',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
snowd.units = 'mm'
snowd.scale_factor = 0.1
snowd.long_name = 'Snow Depth'
snowd.standard_name = 'Snow Depth'
snowd.coordinates = "lon lat"
snowd.description = "Snow depth at time of observation"
v1 = nc.createVariable('avg_dwpk',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
v1.units = 'K'
v1.scale_factor = 0.01
v1.long_name = '2m Average Dew Point Temperature'
v1.standard_name = 'Dewpoint'
v1.coordinates = "lon lat"
v1.description = "Dew Point average computed by averaging mixing ratios"
v2 = nc.createVariable('wind_speed',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
v2.units = 'm s-1'
v2.scale_factor = 0.001
v2.long_name = 'Wind Speed'
v2.standard_name = 'Wind Speed'
v2.coordinates = "lon lat"
v2.description = "Daily averaged wind speed magnitude"
nc.close()
def do_day(valid):
""" Process a day please """
idx = iemre.daily_offset(valid)
with ncopen(iemre.get_daily_ncname(valid.year), 'r', timeout=300) as nc:
high = temperature(nc.variables['high_tmpk_12z'][idx, :, :],
'K').value('F')
low = temperature(nc.variables['low_tmpk_12z'][idx, :, :],
'K').value('F')
precip = distance(nc.variables['p01d_12z'][idx, :, :],
'MM').value("IN")
snow = distance(nc.variables['snow_12z'][idx, :, :], 'MM').value("IN")
snowd = distance(nc.variables['snowd_12z'][idx, :, :],
'MM').value("IN")
# build out the state mappers
pgconn = get_dbconn('postgis')
states = gpd.GeoDataFrame.from_postgis("""
SELECT the_geom, state_abbr from states
where state_abbr not in ('AK', 'HI', 'DC')
""",
pgconn,
index_col='state_abbr',
geom_col='the_geom')
czs = CachingZonalStats(iemre.AFFINE)
sthigh = czs.gen_stats(np.flipud(high), states['the_geom'])
stlow = czs.gen_stats(np.flipud(low), states['the_geom'])
stprecip = czs.gen_stats(np.flipud(precip), states['the_geom'])
stsnow = czs.gen_stats(np.flipud(snow), states['the_geom'])
stsnowd = czs.gen_stats(np.flipud(snowd), states['the_geom'])
statedata = {}
for i, state in enumerate(states.index.values):
statedata[state] = dict(high=sthigh[i],
low=stlow[i],
precip=stprecip[i],
snow=stsnow[i],
snowd=stsnowd[i])
update_database(state + "0000", valid, statedata[state])
# build out climate division mappers
climdiv = gpd.GeoDataFrame.from_postgis("""
SELECT geom, iemid from climdiv
where st_abbrv not in ('AK', 'HI', 'DC')
""",
pgconn,
index_col='iemid',
geom_col='geom')
czs = CachingZonalStats(iemre.AFFINE)
sthigh = czs.gen_stats(np.flipud(high), climdiv['geom'])
stlow = czs.gen_stats(np.flipud(low), climdiv['geom'])
stprecip = czs.gen_stats(np.flipud(precip), climdiv['geom'])
stsnow = czs.gen_stats(np.flipud(snow), climdiv['geom'])
stsnowd = czs.gen_stats(np.flipud(snowd), climdiv['geom'])
for i, iemid in enumerate(climdiv.index.values):
row = dict(high=sthigh[i],
low=stlow[i],
precip=stprecip[i],
snow=stsnow[i],
snowd=stsnowd[i])
# we must have temperature data
if row['high'] is np.ma.masked or row['low'] is np.ma.masked:
print(
("compute_0000 %s has missing temperature data, using state") %
(iemid, ))
row = statedata[iemid[:2]]
update_database(iemid, valid, row)
def init_year(ts):
"""
Create a new NetCDF file for a year of our specification!
"""
fn = iemre.get_hourly_ncname(ts.year)
nc = ncopen(fn, 'w')
nc.title = "IEM Hourly Reanalysis %s" % (ts.year, )
nc.platform = "Grided Observations"
nc.description = "IEM hourly analysis on a 0.125 degree grid"
nc.institution = "Iowa State University, Ames, IA, USA"
nc.source = "Iowa Environmental Mesonet"
nc.project_id = "IEM"
nc.realization = 1
nc.Conventions = 'CF-1.0'
nc.contact = "Daryl Herzmann, [email protected], 515-294-5978"
nc.history = ("%s Generated") % (
datetime.datetime.now().strftime("%d %B %Y"), )
nc.comment = "No Comment at this time"
# Setup Dimensions
nc.createDimension('lat', iemre.NY)
nc.createDimension('lon', iemre.NX)
ts2 = datetime.datetime(ts.year + 1, 1, 1)
days = (ts2 - ts).days
print('Year %s has %s days' % (ts.year, days))
nc.createDimension('time', int(days) * 24)
# Setup Coordinate Variables
lat = nc.createVariable('lat', np.float, ('lat', ))
lat.units = "degrees_north"
lat.long_name = "Latitude"
lat.standard_name = "latitude"
lat.axis = "Y"
lat[:] = iemre.YAXIS
lon = nc.createVariable('lon', np.float, ('lon', ))
lon.units = "degrees_east"
lon.long_name = "Longitude"
lon.standard_name = "longitude"
lon.axis = "X"
lon[:] = iemre.XAXIS
tm = nc.createVariable('time', np.float, ('time', ))
tm.units = "Hours since %s-01-01 00:00:0.0" % (ts.year, )
tm.long_name = "Time"
tm.standard_name = "time"
tm.axis = "T"
tm.calendar = "gregorian"
tm[:] = np.arange(0, int(days) * 24)
# Tracked variables
hasdata = nc.createVariable('hasdata', np.int8, ('lat', 'lon'))
hasdata.units = '1'
hasdata.long_name = 'Analysis Available for Grid Cell'
hasdata.coordinates = "lon lat"
hasdata[:] = 0
# can storage -128->127 actual values are 0 to 100
skyc = nc.createVariable('skyc',
np.int8, ('time', 'lat', 'lon'),
fill_value=-128)
skyc.long_name = "ASOS Sky Coverage"
skyc.stanard_name = "ASOS Sky Coverage"
skyc.units = "%"
skyc.valid_range = [0, 100]
skyc.coordinates = "lon lat"
# 0->65535
tmpk = nc.createVariable('tmpk',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
tmpk.units = "K"
tmpk.scale_factor = 0.01
tmpk.long_name = "2m Air Temperature"
tmpk.standard_name = "2m Air Temperature"
tmpk.coordinates = "lon lat"
# 0->65535 0 to 655.35
dwpk = nc.createVariable('dwpk',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65335)
dwpk.units = "K"
dwpk.scale_factor = 0.01
dwpk.long_name = "2m Air Dew Point Temperature"
dwpk.standard_name = "2m Air Dew Point Temperature"
dwpk.coordinates = "lon lat"
# 0->65535 0 to 65.535
uwnd = nc.createVariable('uwnd',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65335)
uwnd.units = "meters per second"
uwnd.scale_factor = 0.001
uwnd.long_name = "U component of the wind"
uwnd.standard_name = "U component of the wind"
uwnd.coordinates = "lon lat"
# 0->65535 0 to 65.535
vwnd = nc.createVariable('vwnd',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
vwnd.units = "meters per second"
vwnd.scale_factor = 0.001
vwnd.long_name = "V component of the wind"
vwnd.standard_name = "V component of the wind"
vwnd.coordinates = "lon lat"
# 0->65535 0 to 655.35
p01m = nc.createVariable('p01m',
np.uint16, ('time', 'lat', 'lon'),
fill_value=65535)
p01m.units = 'mm'
p01m.scale_factor = 0.01
p01m.long_name = 'Precipitation'
p01m.standard_name = 'Precipitation'
p01m.coordinates = "lon lat"
p01m.description = "Precipitation accumulation for the hour valid time"
nc.close()
def init_year(ts):
"""
Create a new NetCDF file for a year of our specification!
"""
fn = iemre.get_dailyc_mrms_ncname()
nc = ncopen(fn, "w")
nc.title = "IEM Daily Reanalysis Climatology %s" % (ts.year, )
nc.platform = "Grided Climatology"
nc.description = "IEM daily analysis on a 0.01 degree grid"
nc.institution = "Iowa State University, Ames, IA, USA"
nc.source = "Iowa Environmental Mesonet"
nc.project_id = "IEM"
nc.realization = 1
nc.Conventions = "CF-1.0" # *cough*
nc.contact = "Daryl Herzmann, [email protected], 515-294-5978"
nc.history = "%s Generated" % (
datetime.datetime.now().strftime("%d %B %Y"), )
nc.comment = "No Comment at this time"
# Setup Dimensions
nc.createDimension("lat", (iemre.NORTH - iemre.SOUTH) * 100.0)
nc.createDimension("lon", (iemre.EAST - iemre.WEST) * 100.0)
ts2 = datetime.datetime(ts.year + 1, 1, 1)
days = (ts2 - ts).days
nc.createDimension("time", int(days))
nc.createDimension("nv", 2)
# Setup Coordinate Variables
lat = nc.createVariable("lat", np.float, ("lat", ))
lat.units = "degrees_north"
lat.long_name = "Latitude"
lat.standard_name = "latitude"
lat.bounds = "lat_bnds"
lat.axis = "Y"
# Grid centers
lat[:] = np.arange(iemre.SOUTH + 0.005, iemre.NORTH, 0.01)
lat_bnds = nc.createVariable("lat_bnds", np.float, ("lat", "nv"))
lat_bnds[:, 0] = np.arange(iemre.SOUTH, iemre.NORTH, 0.01)
lat_bnds[:, 1] = np.arange(iemre.SOUTH + 0.01, iemre.NORTH + 0.01, 0.01)
lon = nc.createVariable("lon", np.float, ("lon", ))
lon.units = "degrees_east"
lon.long_name = "Longitude"
lon.standard_name = "longitude"
lon.bounds = "lon_bnds"
lon.axis = "X"
lon[:] = np.arange(iemre.WEST, iemre.EAST, 0.01)
lon_bnds = nc.createVariable("lon_bnds", np.float, ("lon", "nv"))
lon_bnds[:, 0] = np.arange(iemre.WEST, iemre.EAST, 0.01)
lon_bnds[:, 1] = np.arange(iemre.WEST + 0.01, iemre.EAST + 0.01, 0.01)
tm = nc.createVariable("time", np.float, ("time", ))
tm.units = "Days since %s-01-01 00:00:0.0" % (ts.year, )
tm.long_name = "Time"
tm.standard_name = "time"
tm.axis = "T"
tm.calendar = "gregorian"
tm[:] = np.arange(0, int(days))
p01d = nc.createVariable("p01d",
np.uint16, ("time", "lat", "lon"),
fill_value=65535)
p01d.units = "mm"
p01d.scale_factor = 0.01
p01d.long_name = "Precipitation"
p01d.standard_name = "Precipitation"
p01d.coordinates = "lon lat"
p01d.description = "Precipitation accumulation for the day"
nc.close()
def plotter(fdict):
""" Go """
ctx = util.get_autoplot_context(fdict, get_description())
ptype = ctx['ptype']
sdate = ctx['sdate']
edate = ctx['edate']
src = ctx['src']
opt = ctx['opt']
usdm = ctx['usdm']
if sdate.year != edate.year:
raise ValueError('Sorry, do not support multi-year plots yet!')
days = (edate - sdate).days
sector = ctx['sector']
if sdate == edate:
title = sdate.strftime("%-d %B %Y")
else:
title = "%s to %s (inclusive)" % (sdate.strftime("%-d %b"),
edate.strftime("%-d %b %Y"))
x0 = 0
x1 = -1
y0 = 0
y1 = -1
state = None
if len(sector) == 2:
state = sector
sector = 'state'
if src == 'mrms':
ncfn = iemre.get_daily_mrms_ncname(sdate.year)
clncfn = iemre.get_dailyc_mrms_ncname()
ncvar = 'p01d'
source = 'MRMS Q3'
subtitle = 'NOAA MRMS Project, GaugeCorr and RadarOnly'
elif src == 'iemre':
ncfn = iemre.get_daily_ncname(sdate.year)
clncfn = iemre.get_dailyc_ncname()
ncvar = 'p01d_12z'
source = 'IEM Reanalysis'
subtitle = 'IEM Reanalysis is derived from various NOAA datasets'
else:
ncfn = "/mesonet/data/prism/%s_daily.nc" % (sdate.year, )
clncfn = "/mesonet/data/prism/prism_dailyc.nc"
ncvar = 'ppt'
source = 'OSU PRISM'
subtitle = ('PRISM Climate Group, Oregon State Univ., '
'http://prism.oregonstate.edu, created 4 Feb 2004.')
mp = MapPlot(sector=sector,
state=state,
axisbg='white',
nocaption=True,
title='%s:: %s Precip %s' % (source, title, PDICT3[opt]),
subtitle='Data from %s' % (subtitle, ),
titlefontsize=14)
idx0 = iemre.daily_offset(sdate)
idx1 = iemre.daily_offset(edate) + 1
if not os.path.isfile(ncfn):
raise ValueError("No data for that year, sorry.")
with util.ncopen(ncfn) as nc:
if state is not None:
x0, y0, x1, y1 = util.grid_bounds(nc.variables['lon'][:],
nc.variables['lat'][:],
state_bounds[state])
elif sector in SECTORS:
bnds = SECTORS[sector]
x0, y0, x1, y1 = util.grid_bounds(
nc.variables['lon'][:], nc.variables['lat'][:],
[bnds[0], bnds[2], bnds[1], bnds[3]])
lats = nc.variables['lat'][y0:y1]
lons = nc.variables['lon'][x0:x1]
if (idx1 - idx0) < 32:
p01d = distance(
np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0),
'MM').value('IN')
else:
# Too much data can overwhelm this app, need to chunk it
for i in range(idx0, idx1, 10):
i2 = min([i + 10, idx1])
if idx0 == i:
p01d = distance(
np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0),
'MM').value('IN')
else:
p01d += distance(
np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0),
'MM').value('IN')
if np.ma.is_masked(np.max(p01d)):
raise ValueError("Data Unavailable")
units = 'inches'
cmap = plt.get_cmap(ctx['cmap'])
cmap.set_bad('white')
if opt == 'dep':
# Do departure work now
with util.ncopen(clncfn) as nc:
climo = distance(
np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0),
'MM').value('IN')
p01d = p01d - climo
[maxv] = np.percentile(np.abs(p01d), [
99,
])
clevs = np.around(np.linspace(0 - maxv, maxv, 11), decimals=2)
elif opt == 'per':
with util.ncopen(clncfn) as nc:
climo = distance(
np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0),
'MM').value('IN')
p01d = p01d / climo * 100.
cmap.set_under('white')
cmap.set_over('black')
clevs = [1, 10, 25, 50, 75, 100, 125, 150, 200, 300, 500]
units = 'percent'
else:
p01d = np.where(p01d < 0.001, np.nan, p01d)
cmap.set_under('white')
clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
if days > 6:
clevs = [0.01, 0.3, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 10, 15, 20]
if days > 29:
clevs = [0.01, 0.5, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35]
if days > 90:
clevs = [0.01, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35, 40]
x2d, y2d = np.meshgrid(lons, lats)
if ptype == 'c':
mp.contourf(x2d, y2d, p01d, clevs, cmap=cmap, units=units, iline=False)
else:
res = mp.pcolormesh(x2d, y2d, p01d, clevs, cmap=cmap, units=units)
res.set_rasterized(True)
if sector != 'midwest':
mp.drawcounties()
mp.drawcities()
if usdm == 'yes':
mp.draw_usdm(edate, filled=False, hatched=True)
return mp.fig
def main(argv):
"""Go Main Go"""
network = argv[1]
wxcfn = argv[2]
utc = datetime.datetime.utcnow()
utc = utc.replace(tzinfo=pytz.UTC)
out = open(wxcfn, 'w')
out.write("""Weather Central 001d0300 Surface Data TimeStamp=%s
12
5 Station
25 Station Name
8 Lat
10 Lon
2 Hour
2 Minute
5 Air Temperature F
5 Dew Point F
5 Wind Direction deg
5 Wind Speed mph
5 Heat Index F
5 Wind Chill F
""" % (utc.strftime("%Y.%m.%d.%H%M"), ))
fn = None
for i in range(4):
now = utc - datetime.timedelta(hours=i)
testfn = now.strftime("/mesonet/data/madis/mesonet1/%Y%m%d_%H00.nc")
if os.path.isfile(testfn):
fn = testfn
break
if fn is None:
sys.exit()
indices = {}
BOGUS = datetime.datetime(2000, 1, 1)
BOGUS = BOGUS.replace(tzinfo=pytz.UTC)
nc = ncopen(fn)
providers = chartostring(nc.variables["dataProvider"][:])
stations = chartostring(nc.variables["stationId"][:])
names = chartostring(nc.variables["stationName"][:])
for i, provider in enumerate(providers):
if provider != network:
continue
sid = stations[i]
# We have an ob!
ticks = int(nc.variables["observationTime"][i])
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ticks)
ts = ts.replace(tzinfo=pytz.UTC)
if ts > indices.get(sid, {'ts': BOGUS})['ts']:
indices[sid] = {'ts': ts, 'idx': i}
for sid in indices:
idx = indices[sid]['idx']
name = names[idx]
latitude = nc.variables['latitude'][idx]
longitude = nc.variables['longitude'][idx]
tmpf = s(nc.variables['temperature'][idx])
dwpf = s(nc.variables['dewpoint'][idx])
qcd = nc.variables['temperatureQCD'][idx][0]
if qcd < -10 or qcd > 10:
tmpf = "M"
dwpf = "M"
heat = "M"
if tmpf != "M" and dwpf != "M":
t = temperature(nc.variables['temperature'][idx], 'K')
d = temperature(nc.variables['dewpoint'][idx], 'K')
# relh = meteorology.relh(t, d).value("%")
heat = "%5.1f" % (meteorology.heatindex(t, d).value("F"), )
drct = s2(nc.variables['windDir'][idx])
smps = s2(nc.variables['windSpeed'][idx])
sped = "M"
if smps != "M":
sped = "%5.1f" % (nc.variables['windSpeed'][idx] * 2.23694, )
wcht = "M"
if tmpf != "M" and sped != "M":
t = temperature(nc.variables['temperature'][idx], 'K')
sped = speed(nc.variables['windSpeed'][idx], 'MPS')
wcht = "%5.1f" % (meteorology.windchill(t, sped).value("F"), )
ts = indices[sid]['ts']
out.write(("%5.5s %25.25s %8.4f %10.4f "
"%02i %02i %5s %5s %5s %5s %5s %5s\n") %
(sid, name, latitude, longitude, ts.hour, ts.minute, tmpf,
dwpf, drct, sped, heat, wcht))
nc.close()
out.close()
pqstr = "data c 000000000000 wxc/wxc_%s.txt bogus txt" % (
network.lower(), )
subprocess.call("/home/ldm/bin/pqinsert -p '%s' %s" % (pqstr, wxcfn),
shell=True)
os.remove(wxcfn)
def init_year(ts):
"""
Create a new NetCDF file for a year of our specification!
"""
nc = ncopen("/mesonet/data/ndfd/ndfd_dailyc.nc", "w")
nc.title = "NDFD"
nc.contact = "Daryl Herzmann, [email protected], 515-294-5978"
nc.history = "%s Generated" % (
datetime.datetime.now().strftime("%d %B %Y"), )
grbs = pygrib.open("/mesonet/ARCHIVE/data/2019/05/06/model/ndfd/"
"00/ndfd.t00z.awp2p5f001.grib2")
grb = grbs[1]
shape = grb.values.shape
lats, lons = grb.latlons()
# Setup Dimensions
nc.createDimension("lat", shape[0])
nc.createDimension("lon", shape[1])
ts2 = datetime.datetime(ts.year + 1, 1, 1)
days = (ts2 - ts).days
nc.createDimension("time", int(days))
# Setup Coordinate Variables
lat = nc.createVariable("lat", np.float, ("lat", "lon"))
lat.units = "degrees_north"
lat.long_name = "Latitude"
lat.standard_name = "latitude"
lat.bounds = "lat_bnds"
lat.axis = "Y"
lat[:] = lats
lon = nc.createVariable("lon", np.float, ("lat", "lon"))
lon.units = "degrees_east"
lon.long_name = "Longitude"
lon.standard_name = "longitude"
lon.bounds = "lon_bnds"
lon.axis = "X"
lon[:] = lons
tm = nc.createVariable("time", np.float, ("time", ))
tm.units = "Days since %s-01-01 00:00:0.0" % (ts.year, )
tm.long_name = "Time"
tm.standard_name = "time"
tm.axis = "T"
tm.calendar = "gregorian"
tm[:] = np.arange(0, int(days))
p01d = nc.createVariable("p01d",
np.uint16, ("time", "lat", "lon"),
fill_value=65535)
p01d.units = "mm"
p01d.scale_factor = 0.01
p01d.long_name = "Precipitation"
p01d.standard_name = "Precipitation"
p01d.coordinates = "lon lat"
p01d.description = "Precipitation accumulation for the day"
high = nc.createVariable("high_tmpk",
np.uint16, ("time", "lat", "lon"),
fill_value=65535)
high.units = "K"
high.scale_factor = 0.01
high.long_name = "2m Air Temperature Daily High"
high.standard_name = "2m Air Temperature"
high.coordinates = "lon lat"
low = nc.createVariable("low_tmpk",
np.uint16, ("time", "lat", "lon"),
fill_value=65535)
low.units = "K"
low.scale_factor = 0.01
low.long_name = "2m Air Temperature Daily Low"
low.standard_name = "2m Air Temperature"
low.coordinates = "lon lat"
gdd = nc.createVariable("gdd50",
np.uint16, ("time", "lat", "lon"),
fill_value=65535)
gdd.units = "F"
gdd.scale_factor = 0.01
gdd.long_name = "Growing Degree Days"
gdd.standard_name = "Growing Degree Days"
gdd.coordinates = "lon lat"
nc.close()
def main():
"""Go Main Go"""
pgconn = get_dbconn("iem")
icursor = pgconn.cursor()
now = datetime.datetime.utcnow() - datetime.timedelta(hours=3)
fn = "/mesonet/data/madis/metar/%s.nc" % (now.strftime("%Y%m%d_%H00"),)
table = "current_qc"
if not os.path.isfile(fn):
sys.exit()
nc = ncopen(fn)
ids = chartostring(nc.variables["stationName"][:])
nc_tmpk = nc.variables["temperature"]
nc_dwpk = nc.variables["dewpoint"]
nc_alti = nc.variables["altimeter"]
tmpkqcd = nc.variables["temperatureQCD"]
dwpkqcd = nc.variables["dewpointQCD"]
altiqcd = nc.variables["altimeterQCD"]
for j in range(ids.shape[0]):
sid = ids[j]
if len(sid) < 4:
continue
if sid[0] == "K":
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(
seconds=int(nc.variables["timeObs"][j])
)
ts = ts.replace(tzinfo=pytz.utc)
(tmpf, tmpf_qc_av, tmpf_qc_sc) = ("Null", "Null", "Null")
(dwpf, dwpf_qc_av, dwpf_qc_sc) = ("Null", "Null", "Null")
(alti, alti_qc_av, alti_qc_sc) = ("Null", "Null", "Null")
if (
not np.ma.is_masked(nc_tmpk[j])
and not np.ma.is_masked(tmpkqcd[j, 0])
and not np.ma.is_masked(tmpkqcd[j, 6])
):
tmpf = check(temperature(nc_tmpk[j], "K").value("F"))
tmpf_qc_av = figure(nc_tmpk[j], tmpkqcd[j, 0])
tmpf_qc_sc = figure(nc_tmpk[j], tmpkqcd[j, 6])
if (
not np.ma.is_masked(nc_dwpk[j])
and not np.ma.is_masked(dwpkqcd[j, 0])
and not np.ma.is_masked(dwpkqcd[j, 6])
):
dwpf = check(temperature(nc_dwpk[j], "K").value("F"))
dwpf_qc_av = figure(nc_dwpk[j], dwpkqcd[j, 0])
dwpf_qc_sc = figure(nc_dwpk[j], dwpkqcd[j, 6])
if not np.ma.is_masked(nc_alti[j]):
alti = check(nc_alti[j] / 100.0 * 0.0295298)
alti_qc_av = figure_alti(altiqcd[j, 0] * 0.0295298)
alti_qc_sc = figure_alti(altiqcd[j, 6] * 0.0295298)
sql = """
UPDATE %s SET tmpf = %s, tmpf_qc_av = %s,
tmpf_qc_sc = %s, dwpf = %s, dwpf_qc_av = %s,
dwpf_qc_sc = %s, alti = %s, alti_qc_av = %s,
alti_qc_sc = %s, valid = '%s' WHERE
station = '%s'
""" % (
table,
tmpf,
tmpf_qc_av,
tmpf_qc_sc,
dwpf,
dwpf_qc_av,
dwpf_qc_sc,
alti,
alti_qc_av,
alti_qc_sc,
ts.strftime("%Y-%m-%d %H:%M+00"),
sid[1:],
)
sql = sql.replace("--", "Null").replace("nan", "Null")
try:
icursor.execute(sql)
except Exception as exp:
print(exp)
print(sql)
nc.close()
icursor.close()
pgconn.commit()
pgconn.close()
