def draw_map():
"""make maps, not war."""
m = MapPlot(sector='conus',
title='4 March 2019 :: DEP Precip Points')
update_grid(m.ax)
m.postprocess(filename='/tmp/map_clipoints.png')
m.close()
def make_plots(nc):
''' Generate some plots '''
sts = compute_sts(nc)
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
rts = (sts.astimezone(pytz.timezone("America/Chicago"))).strftime(
"%d %b %Y %H %p")
for i, tm in enumerate(nc.variables['time'][:]):
dt = sts + datetime.timedelta(minutes=float(tm))
if dt.minute != 0:
continue
fhour = int( tm / 60.0 )
fts = (dt.astimezone(pytz.timezone("America/Chicago"))).strftime(
"%d %b %Y %H %p")
for pvar in PVARS:
m = MapPlot(title='ISUMM5/Bridget Modelled %s' % (
PVARS[pvar]['title'],),
subtitle='Model Run: %s Forecast Valid: %s' % (rts, fts))
vals = nc.variables[pvar][i,:,:]
if pvar == 'bdeckt':
vals = temperature(vals, 'K').value('F')
m.pcolormesh(lons, lats, vals, PVARS[pvar]['levels'], units='mm')
pqstr = "plot c %s model/frost/bridget/%02i/%s_%02i_f%03i.png bogus png" % (
sts.strftime("%Y%m%d%H%M"), sts.hour,
pvar, sts.hour, fhour)
m.postprocess(pqstr=pqstr)
m.close()
def compute(valid):
''' Get me files '''
prob = None
for hr in range(-15,0):
ts = valid + datetime.timedelta(hours=hr)
fn = ts.strftime("hrrr.ref.%Y%m%d%H00.grib2")
if not os.path.isfile(fn):
continue
grbs = pygrib.open(fn)
gs = grbs.select(level=1000,forecastTime=(-1 * hr * 60))
ref = generic_filter(gs[0]['values'], np.max, size=10)
if prob is None:
lats, lons = gs[0].latlons()
prob = np.zeros( np.shape(ref) )
prob = np.where(ref > 29, prob+1, prob)
prob = np.ma.array(prob / 15. * 100.)
prob.mask = np.ma.where(prob < 1, True, False)
m = MapPlot(sector='iowa',
title='HRRR Composite Forecast 4 PM 20 May 2014 30+ dbZ Reflectivity',
subtitle='frequency of previous 15 model runs all valid at %s, ~15km smoothed' % (valid.astimezone(pytz.timezone("America/Chicago")).strftime("%-d %b %Y %I:%M %p %Z"),))
m.pcolormesh(lons, lats, prob, np.arange(0,101,10), units='%',
clip_on=False)
m.map.drawcounties()
m.postprocess(filename='test.ps')
m.close()
def main():
"""Go Main"""
grbs = pygrib.open('ds.snow.bin')
# skip 1-off first field
total = None
lats = lons = None
for grb in grbs[1:]:
if lats is None:
lats, lons = grb.latlons()
total = grb['values']
continue
total += grb['values']
# TODO tz-hack here
analtime = grb.analDate - datetime.timedelta(hours=5)
mp = MapPlot(
sector='custom', west=-100, east=-92, north=45, south=41,
axisbg='tan',
title=("NWS Forecasted Accumulated Snowfall "
"thru 7 PM 12 April 2019"),
subtitle='NDFD Forecast Issued %s' % (
analtime.strftime("%-I %p %-d %B %Y"), )
)
cmap = nwssnow()
cmap.set_bad('tan')
mp.pcolormesh(
lons, lats, total * 39.3701,
[0.01, 1, 2, 3, 4, 6, 8, 12, 18, 24, 30, 36],
cmap=cmap,
units='inch')
mp.drawcounties()
mp.drawcities()
mp.postprocess(filename='test.png')
mp.close()
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
We should total files from 1 AM to midnight local time
"""
sts = ts.replace(hour=1)
ets = sts + datetime.timedelta(hours=24)
interval = datetime.timedelta(hours=1)
now = sts
total = None
lts = None
while now < ets:
gmt = now.astimezone(pytz.timezone("UTC"))
fn = gmt.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/"
+"stage4/ST4.%Y%m%d%H.01h.grib"))
if os.path.isfile(fn):
lts = now
grbs = pygrib.open(fn)
if total is None:
g = grbs[1]
total = g["values"]
lats, lons = g.latlons()
else:
total += grbs[1]["values"]
grbs.close()
now += interval
if lts is None and ts.hour > 1:
print 'stage4_today_total.py found no data!'
if lts is None:
return
lts = lts - datetime.timedelta(minutes=1)
subtitle = "Total between 12:00 AM and %s" % (lts.strftime("%I:%M %p %Z"),)
routes = 'ac'
if not realtime:
routes = 'a'
for sector in ['iowa', 'midwest', 'conus']:
pqstr = "plot %s %s00 %s_stage4_1d.png %s_stage4_1d.png png" % (routes,
ts.strftime("%Y%m%d%H"), sector, sector )
m = MapPlot(sector=sector,
title="%s NCEP Stage IV Today's Precipitation" % (
ts.strftime("%-d %b %Y"),),
subtitle=subtitle)
clevs = np.arange(0, 0.25, 0.05)
clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs[0] = 0.01
m.pcolormesh(lons, lats, total / 24.5, clevs, units='inch')
#map.drawstates(zorder=2)
if sector == 'iowa':
m.drawcounties()
m.postprocess(pqstr=pqstr)
m.close()
def do(ts, hours):
"""
Create a plot of precipitation stage4 estimates for some day
"""
ts = ts.replace(minute=0)
sts = ts - datetime.timedelta(hours=hours)
ets = ts
interval = datetime.timedelta(hours=1)
now = sts
total = None
lts = None
while now < ets:
fn = "/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib" % (
now.strftime("%Y/%m/%d"),
now.strftime("%Y%m%d%H") )
if os.path.isfile(fn):
lts = now
grbs = pygrib.open(fn)
if total is None:
g = grbs[1]
total = g["values"]
lats, lons = g.latlons()
else:
total += grbs[1]["values"]
grbs.close()
now += interval
if lts is None and ts.hour > 1:
print 'Missing StageIV data!'
if lts is None:
return
cmap = cm.get_cmap("jet")
cmap.set_under('white')
cmap.set_over('black')
clevs = [0.01,0.1,0.25,0.5,1,2,3,5,8,9.9]
localtime = (ts - datetime.timedelta(minutes=1)).astimezone(
pytz.timezone("America/Chicago"))
for sector in ['iowa', 'midwest', 'conus']:
m = MapPlot(sector=sector,
title='NCEP Stage IV %s Hour Precipitation' % (hours,),
subtitle='Total up to %s' % (
localtime.strftime("%d %B %Y %I %p %Z"),))
m.pcolormesh(lons, lats, total / 24.5, clevs, units='inch')
pqstr = "plot %s %s00 %s_stage4_%sh.png %s_stage4_%sh_%s.png png" % (
'ac', ts.strftime("%Y%m%d%H"), sector, hours,
sector, hours, ts.strftime("%H"))
if sector == 'iowa':
m.drawcounties()
m.postprocess(pqstr=pqstr)
m.close()
def doit(now):
"""
Generate some plots for the COOP data!
"""
# We'll assume all COOP data is 12z, sigh for now
sql = """SELECT id, pday, network
from summary_%s s JOIN stations t ON (t.iemid = s.iemid)
WHERE day = '%s' and
t.network ~* 'COOP' and pday >= 0""" % (now.year,
now.strftime("%Y-%m-%d") )
lats = []
lons = []
vals = []
icursor.execute( sql )
iamax = 0.
for row in icursor:
sid = row[0]
if not st.sts.has_key(sid):
continue
#labels.append( id[2:] )
lats.append( st.sts[sid]['lat'] )
lons.append( st.sts[sid]['lon'] )
vals.append( row[1] )
if row[2] == 'IA_COOP' and row[1] > iamax:
iamax = row[1]
#if iamax == 0:
# Dummy in some bad data to prevent the contouring from going mad
# lats.append( 42. )
# lons.append( -96.0 )
# vals.append( 1. )
# Plot Iowa
m = MapPlot(sector='iowa',
title='24 Hour NWS COOP Precipitation [inch]',
subtitle='Ending %s at roughly 12Z' % (now.strftime("%d %B %Y"),))
m.contourf(lons, lats, vals, clevs, units='inch')
pqstr = "plot ac %s0000 iowa_coop_12z_precip.png iowa_coop_12z_precip.png png" % (now.strftime("%Y%m%d"),)
m.postprocess(pqstr=pqstr)
m.close()
m = MapPlot(sector='midwest',
title='24 Hour NWS COOP Precipitation [inch]',
subtitle='Ending %s at roughly 12Z' % (now.strftime("%d %B %Y"),))
m.contourf(lons, lats, vals, clevs, units='inch')
pqstr = "plot ac %s0000 midwest_coop_12z_precip.png midwest_coop_12z_precip.png png" % (now.strftime("%Y%m%d"),)
m.postprocess(pqstr=pqstr)
m.close()
def draw(self):
''' For debugging, draw the polygons!'''
from descartes.patch import PolygonPatch
from pyiem.plot import MapPlot
for sig in self.sigmets:
m = MapPlot(sector='conus')
x, y = m.map(sig.geom.exterior.xy[0], sig.geom.exterior.xy[1])
patch = PolygonPatch(Polygon(zip(x, y)), fc='r', label='Outlook')
m.ax.add_patch(patch)
fn = '/tmp/%s.png' % (sig.label,)
print ':: creating plot %s' % (fn,)
m.postprocess(filename=fn)
m.close()
def plot():
"""Make a pretty plot"""
df = pd.read_csv('wfo.csv')
df.set_index('wfo', inplace=True)
m = MapPlot(sector='conus',
title="Percentage of Flash Flood Watches receiving 1+ FFW",
subtitle='PRELIMINARY PLOT! Please do not share :)')
cmap = plt.get_cmap('jet')
df2 = df[df['freq'].notnull()]
m.fill_cwas(df2['freq'].to_dict(), cmap=cmap, units='%',
lblformat='%.0f', ilabel=True)
m.postprocess(filename='test.png')
m.close()
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
lts = datetime.datetime.utcnow().replace(
tzinfo=pytz.timezone("UTC"))
lts = lts.astimezone(pytz.timezone("America/Chicago"))
# make assumptions about the last valid MRMS data
if realtime:
# Up until :59 after of the last hour
lts = (lts - datetime.timedelta(hours=1)).replace(minute=59)
else:
lts = lts.replace(year=ts.year, month=ts.month, day=ts.day,
hour=23, minute=59)
idx = iemre.daily_offset(ts)
ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (ts.year,)
nc = netCDF4.Dataset(ncfn)
precip = nc.variables['p01d'][idx, :, :]
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
subtitle = "Total between 12:00 AM and %s" % (
lts.strftime("%I:%M %p %Z"),)
routes = 'ac'
if not realtime:
routes = 'a'
# clevs = np.arange(0, 0.25, 0.05)
# clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
# clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
10]
sector = 'iowa'
pqstr = ("plot %s %s00 %s_q2_1d.png %s_q2_1d.png png"
) % (routes, ts.strftime("%Y%m%d%H"), sector, sector)
m = MapPlot(title=("%s NCEP MRMS Q3 Today's Precipitation"
) % (ts.strftime("%-d %b %Y"),),
subtitle=subtitle, sector=sector)
(x, y) = np.meshgrid(lons, lats)
m.pcolormesh(x, y, distance(precip, 'MM').value('IN'), clevs,
cmap=nwsprecip(), units='inch')
m.drawcounties()
m.postprocess(pqstr=pqstr, view=False)
m.close()
def plot_precip_month(valid):
""" Go Main Go
Args:
valid (datetime): The timestamp we are interested in!
"""
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
cursor = pgconn.cursor()
d1 = valid.replace(day=1)
d2 = d1 + datetime.timedelta(days=35)
d2 = d2.replace(day=1)
cursor.execute("""SELECT sum(pday), id, st_x(geom), st_y(geom)
from summary s JOIN stations t on
(t.iemid = s.iemid) WHERE s.day >= %s and s.day < %s
and t.network in ('IA_COOP', 'NE_COOP', 'MO_COOP', 'IL_COOP', 'WI_COOP',
'MN_COOP')
and pday is not null and pday >= 0 and
extract(hour from coop_valid) between 5 and 10
GROUP by id, st_x, st_y""", (d1.date(), d2.date()))
labels = []
vals = []
lats = []
lons = []
for row in cursor:
labels.append(row[1])
vals.append(pretty(row[0]))
lats.append(row[3])
lons.append(row[2])
m = MapPlot(title='%s NWS COOP Month Precipitation Totals [inch]' % (
valid.strftime("%-d %b %Y"),),
subtitle='Reports valid between 6 and 9 AM',
axisbg='white', figsize=(10.24, 7.68))
m.plot_values(lons, lats, vals, fmt='%s', labels=labels,
labelcolor='tan')
m.drawcounties()
pqstr = "plot ac %s0000 coopMonthPlot.gif coopMonthPlot.gif gif" % (
valid.strftime("%Y%m%d"),)
m.postprocess(pqstr=pqstr)
m.close()
pgconn.close()
def main():
"""Go Main!"""
nt = NetworkTable("IACLIMATE")
pgconn = get_dbconn('coop')
df = read_sql("""
with monthly as (
select station, year, month, avg((high+low)/2.) from alldata_ia
WHERE day < '2018-06-01' and high is not null
GROUP by station, year, month),
agg as (
select station, year, month, avg,
lag(avg) OVER (PARTITION by station ORDER by year ASC, month ASC)
from monthly),
agg2 as (
select station, year, month, avg, lag, avg - lag as val,
rank() OVER (PARTITION by station ORDER by avg - lag DESC)
from agg WHERE lag is not null)
select * from agg2 where rank = 1 ORDER by station
""", pgconn, index_col='station')
df['lat'] = 0.
df['lon'] = 0.
for station, _ in df.iterrows():
if station in nt.sts and station != 'IA0000' and station[2] != 'C':
df.at[station, 'lat'] = nt.sts[station]['lat']
df.at[station, 'lon'] = nt.sts[station]['lon']
mp = MapPlot(title="Largest Positive Change in Month to Month Average Temperature",
subtitle=('values in red set record for April to May 2018'), sector='state',
state='IA',
drawstates=True, continentalcolor='white')
df2 = df[df['year'] == 2018]
mp.plot_values(df2['lon'].values,
df2['lat'].values,
df2['val'].values, fmt='%.1f', textsize=12, labelbuffer=5,
color='r')
df2 = df[df['year'] != 2018]
mp.plot_values(df2['lon'].values,
df2['lat'].values,
df2['val'].values, fmt='%.1f', textsize=12, labelbuffer=5,
color='b')
mp.drawcounties()
mp.postprocess(filename='test.png')
mp.close()
def run( fn ):
''' Run for a given filename! '''
nc = netCDF4.Dataset(fn, 'r')
# time
idx0, idx1 = get_time( nc )
# somewhat a hack for now
dmul = 1.0
if (idx1 - idx0) == 19:
dmul = 1.0
elif (idx1 - idx0) / 19.0 < 13:
dmul = 365.0 / 12.0
# Either pr or Prcp
pvar = 'pr' if nc.variables.has_key('pr') else 'Prcp'
pmul = 1.0 if nc.variables[pvar].units in ['mm/d', 'mm'] else 86400.0
precip = np.sum(nc.variables[pvar][idx0:idx1+1,:,:],0) * dmul * pmul / 19.0 / 24.5
# lat or latitude
latvar = 'lat' if nc.variables.has_key('lat') else 'latitude'
lats = nc.variables[latvar][:]
lats = np.concatenate([lats, [lats[-1]+(lats[-1]-lats[-2])]])
# lon or longitude
lonvar = 'lon' if nc.variables.has_key('lon') else 'longitude'
lons = nc.variables[lonvar][:] - 360.
lons = np.concatenate([lons, [lons[-1]+(lons[-1]-lons[-2])]])
print fn, np.shape(lons), np.shape(lats), np.max(precip)
try:
title = '2046-2065 Precip %s' % (nc.title,)
except:
title = '2046-2065 Yearly Average Precipitation '
subtitle = 'filename: %s' % (fn,)
m = MapPlot(title=title, subtitle=subtitle,
sector='conus', nologo=True, caption='')
x,y = np.meshgrid(lons, lats)
cmap = cm.get_cmap('gist_rainbow')
cmap.set_over('black')
m.pcolormesh(x, y, precip, np.array([0,1,5,10,15,20,25,30,35,40,50,60,70,80,100]),
cmap=cmap, units='inch/yr')
m.postprocess(filename='/tmp/plots/%s.png' % (fn.replace(".nc", ""),))
nc.close()
m.close()
def plot_snowdepth(valid):
""" Go Main Go
Args:
valid (datetime): The timestamp we are interested in!
"""
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
cursor = pgconn.cursor()
cursor.execute("""SELECT snowd, id, st_x(geom), st_y(geom)
from summary s JOIN stations t on
(t.iemid = s.iemid) WHERE s.day = %s
and t.network in ('IA_COOP', 'NE_COOP', 'MO_COOP', 'IL_COOP', 'WI_COOP',
'MN_COOP')
and snowd is not null and snowd >= 0 and
extract(hour from coop_valid) between 5 and 10""", (valid.date(),))
labels = []
vals = []
lats = []
lons = []
for row in cursor:
labels.append(row[1])
vals.append(pretty(row[0], precision=0))
lats.append(row[3])
lons.append(row[2])
m = MapPlot(title='%s NWS COOP Snowfall Depth Reports [inch]' % (
valid.strftime("%-d %b %Y"),),
subtitle='Reports valid between 6 and 9 AM',
axisbg='white', figsize=(10.24, 7.68))
m.plot_values(lons, lats, vals, fmt='%s', labels=labels,
labelcolor='tan')
m.drawcounties()
pqstr = "plot ac %s0000 coopSnowDepth.gif coopSnowDepth.gif gif" % (
valid.strftime("%Y%m%d"),)
m.postprocess(pqstr=pqstr)
m.close()
pgconn.close()
def run( fn ):
''' Run for a given filename! '''
nc = netCDF4.Dataset(fn, 'r')
# time
idx0, idx1 = get_time( nc )
# somewhat a hack for now
dmul = 1.0
if (idx1 - idx0) == 19:
dmul = 1.0
elif (idx1 - idx0) / 19.0 < 13:
dmul = 365.0 / 12.0
pvar = 'tmin'
tmpf = c2f(np.average(nc.variables[pvar][idx0:idx1+1,:,:],0) * dmul )
# lat or latitude
latvar = 'lat' if nc.variables.has_key('lat') else 'latitude'
lats = nc.variables[latvar][:]
lats = np.concatenate([lats, [lats[-1]+(lats[-1]-lats[-2])]])
# lon or longitude
lonvar = 'lon' if nc.variables.has_key('lon') else 'longitude'
lons = nc.variables[lonvar][:] - 360.
lons = np.concatenate([lons, [lons[-1]+(lons[-1]-lons[-2])]])
print fn, np.shape(lons), np.shape(lats), np.max(tmpf)
try:
title = '2046-2065 Average Daily High Temp %s' % (nc.title,)
except:
title = '2046-2065 Average Daily Low Temperature '
subtitle = 'filename: %s' % (fn,)
m = MapPlot(title=title, subtitle=subtitle,
sector='conus', nologo=True, caption='')
x,y = np.meshgrid(lons, lats)
cmap = cm.get_cmap('gist_rainbow')
cmap.set_over('black')
m.pcolormesh(x, y, tmpf, np.arange(50,120,10),
cmap=cmap, units='F')
m.postprocess(filename='/tmp/plots/%s.png' % (fn.replace(".nc", ""),))
nc.close()
m.close()
def doit(ts):
"""
Generate hourly plot of stage4 data
"""
routes = "a"
if ((gmtnow - ts).days * 86400. + (gmtnow - ts).seconds) < 7200:
routes = "ac"
fn = "/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib" % (
ts.strftime("%Y/%m/%d"), ts.strftime("%Y%m%d%H") )
if not os.path.isfile(fn):
print 'current/stage4_hourly.py Missing stage4 %s' % (fn,)
return
grbs = pygrib.open(fn)
grib = grbs[1]
lats, lons = grib.latlons()
vals = grib.values / 25.4
cmap = cm.get_cmap("jet")
cmap.set_under('white')
cmap.set_over('black')
clevs = [0.01,0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1,1.5,2,3]
localtime = ts.astimezone(pytz.timezone("America/Chicago"))
for sector in ['iowa', 'midwest', 'conus']:
m = MapPlot(sector=sector,
title='Stage IV One Hour Precipitation',
subtitle='Hour Ending %s' % (
localtime.strftime("%d %B %Y %I %p %Z"),))
m.pcolormesh(lons, lats, vals, clevs, units='inch')
pqstr = "plot %s %s00 %s_stage4_1h.png %s_stage4_1h_%s.png png" % (
routes, ts.strftime("%Y%m%d%H"), sector,
sector, ts.strftime("%H"))
if sector == 'iowa':
m.drawcounties()
m.postprocess(view=False, pqstr=pqstr)
m.close()
def compute(valid):
""" Get me files """
prob = None
for hr in range(-15, 0):
ts = valid + datetime.timedelta(hours=hr)
fn = ts.strftime("/tmp/ncep_hrrr_%Y%m%d%H.grib2")
print hr, fn
if not os.path.isfile(fn):
continue
grbs = pygrib.open(fn)
try:
gs = grbs.select(level=1000, forecastTime=(-1 * hr * 60))
except:
print fn, "ERROR"
continue
ref = gs[0]["values"]
# ref = generic_filter(gs[0]['values'], np.max, size=10)
if prob is None:
lats, lons = gs[0].latlons()
prob = np.zeros(np.shape(ref))
prob = np.where(ref > 29, prob + 1, prob)
prob = np.ma.array(prob / 15.0 * 100.0)
prob.mask = np.ma.where(prob < 1, True, False)
m = MapPlot(
sector="iowa",
title="HRRR Composite Forecast 6 PM 22 Sep 2015 30+ dbZ Reflectivity",
subtitle="frequency of previous 15 NCEP model runs all valid at %s"
% (valid.astimezone(pytz.timezone("America/Chicago")).strftime("%-d %b %Y %I:%M %p %Z"),),
)
m.pcolormesh(lons, lats, prob, np.arange(0, 101, 10), units="% of runs", clip_on=False)
m.map.drawcounties()
m.postprocess(filename="test.png")
m.close()
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
nc = netCDF4.Dataset("/mesonet/data/iemre/%s_ifc_daily.nc" % (ts.year,))
idx = daily_offset(ts)
xaxis = nc.variables['lon'][:]
yaxis = nc.variables['lat'][:]
total = nc.variables['p01d'][idx, :, :]
nc.close()
lastts = datetime.datetime(ts.year, ts.month, ts.day, 23, 59)
if realtime:
now = datetime.datetime.now() - datetime.timedelta(minutes=60)
lastts = now.replace(minute=59)
subtitle = "Total between 12:00 AM and %s" % (
lastts.strftime("%I:%M %p"),)
routes = 'ac'
if not realtime:
routes = 'a'
clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
10]
pqstr = ("plot %s %s00 iowa_ifc_1d.png iowa_ifc_1d.png png"
) % (routes, ts.strftime("%Y%m%d%H"))
m = MapPlot(title=("%s Iowa Flood Center Today's Precipitation"
) % (ts.strftime("%-d %b %Y"),),
subtitle=subtitle, sector='custom',
west=xaxis[0], east=xaxis[-1],
south=yaxis[0], north=yaxis[-1])
(x, y) = np.meshgrid(xaxis, yaxis)
m.pcolormesh(x, y, distance(total, 'MM').value("IN"), clevs,
cmap=nwsprecip(), units='inch')
m.drawcounties()
m.postprocess(pqstr=pqstr, view=False)
m.close()
def main():
"""Go Main Go"""
pgconn = get_dbconn('postgis')
cursor = pgconn.cursor()
sts = datetime.date(2018, 1, 24)
ets = datetime.date(2017, 8, 1)
now = sts
xaxis = np.arange(reference.IA_WEST, reference.IA_EAST, GX)
yaxis = np.arange(reference.IA_SOUTH, reference.IA_NORTH, GX)
"""
days = np.ones((len(yaxis), len(xaxis)), 'i') * -1
count = 0
while now >= ets:
cursor.execute(""
select distinct st_x(geom) as lon, st_y(geom) as lat
from nldn_all n, states s
WHERE n.valid >= %s and n.valid < %s
and s.state_abbr = 'IA' and n.geom && s.the_geom
"", (now, now + datetime.timedelta(days=1)))
print("date: %s rows: %s" % (now, cursor.rowcount))
for row in cursor:
yidx = int((row[1] - reference.IA_SOUTH) / GX)
xidx = int((row[0] - reference.IA_WEST) / GX)
if days[yidx, xidx] < 0:
days[yidx, xidx] = count
now -= datetime.timedelta(days=1)
count += 1
np.save('days', days)
"""
days = np.load('days.npy')
mp = MapPlot()
ramp = [1, 24, 55, 85, 116, 146, 177]
mp.pcolormesh(xaxis, yaxis, days, ramp,
cmap=plt.get_cmap('afmhot_r'))
mp.postprocess(filename='test.png')
mp.close()
def plot_hilo(valid):
""" Go Main Go
Args:
valid (datetime): The timestamp we are interested in!
"""
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
cursor = pgconn.cursor()
cursor.execute("""SELECT max_tmpf, min_tmpf, id, st_x(geom), st_y(geom)
from summary s JOIN stations t on
(t.iemid = s.iemid) WHERE s.day = %s
and t.network in ('IA_COOP', 'NE_COOP', 'MO_COOP', 'IL_COOP', 'WI_COOP',
'MN_COOP')
and max_tmpf is not null and max_tmpf >= -30 and
min_tmpf is not null and min_tmpf < 99 and
extract(hour from coop_valid) between 5 and 10""", (valid.date(),))
data = []
for row in cursor:
data.append(dict(lat=row[4], lon=row[3], tmpf=row[0],
dwpf=row[1], id=row[2]))
m = MapPlot(title=('%s NWS COOP 24 Hour High/Low Temperature [$^\circ$F]'
) % (valid.strftime("%-d %b %Y"),),
subtitle='Reports valid between 6 and 9 AM',
axisbg='white', figsize=(10.24, 7.68))
m.plot_station(data)
m.drawcounties()
pqstr = "plot ac %s0000 coopHighLow.gif coopHighLow.gif gif" % (
valid.strftime("%Y%m%d"),)
m.postprocess(pqstr=pqstr)
m.close()
pgconn.close()
def do(ts, hours):
"""
Create a plot of precipitation stage4 estimates for some day
"""
ts = ts.replace(minute=0)
sts = ts - datetime.timedelta(hours=hours)
ets = ts
interval = datetime.timedelta(hours=1)
now = sts
total = None
lts = None
while now < ets:
fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib") % (
now.strftime("%Y/%m/%d"),
now.strftime("%Y%m%d%H"),
)
if os.path.isfile(fn):
lts = now
grbs = pygrib.open(fn)
if total is None:
g = grbs[1]
total = g["values"].filled(0)
lats, lons = g.latlons()
else:
total += grbs[1]["values"].filled(0)
grbs.close()
now += interval
if lts is None and ts.hour > 1:
LOG.info("Missing StageIV data!")
if lts is None:
return
cmap = cm.get_cmap("jet")
cmap.set_under("white")
cmap.set_over("black")
clevs = [0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 8, 9.9]
localtime = (ts - datetime.timedelta(minutes=1)).astimezone(
pytz.timezone("America/Chicago"))
for sector in ["iowa", "midwest", "conus"]:
mp = MapPlot(
sector=sector,
title="NCEP Stage IV %s Hour Precipitation" % (hours, ),
subtitle="Total up to %s" %
(localtime.strftime("%d %B %Y %I %p %Z"), ),
)
mp.pcolormesh(lons,
lats,
distance(total, "MM").value("IN"),
clevs,
units="inch")
pqstr = "plot %s %s00 %s_stage4_%sh.png %s_stage4_%sh_%s.png png" % (
"ac",
ts.strftime("%Y%m%d%H"),
sector,
hours,
sector,
hours,
ts.strftime("%H"),
)
if sector == "iowa":
mp.drawcounties()
mp.postprocess(pqstr=pqstr)
mp.close()
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
# Start at midnight
now = ts.replace(hour=0, minute=0)
ets = now + datetime.timedelta(hours=24)
interval = datetime.timedelta(minutes=5)
currenttime = datetime.datetime.utcnow()
currenttime = currenttime.replace(tzinfo=pytz.timezone("UTC"))
total = None
lastts = None
while now <= ets:
gmt = now.astimezone(pytz.timezone("UTC"))
if gmt > currenttime:
break
fn = gmt.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/"
"GIS/ifc/p05m_%Y%m%d%H%M.png"))
if not os.path.isfile(fn):
now += interval
continue
png = gdal.Open(fn, 0)
data = np.flipud(png.ReadAsArray()) # units are mm per 5 minutes
data = np.where(data > 254, 0, data) / 10.0
if total is None:
total = data
else:
total += data
lastts = now
now += interval
if lastts is None:
print(
('No IFC Data found for date: %s') % (now.strftime("%d %B %Y"), ))
return
lastts = lastts - datetime.timedelta(minutes=1)
subtitle = "Total between 12:00 AM and %s" % (
lastts.strftime("%I:%M %p %Z"), )
routes = 'ac'
if not realtime:
routes = 'a'
clevs = np.arange(0, 0.25, 0.05)
clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs[0] = 0.01
sector = 'iowa'
pqstr = ("plot %s %s00 %s_ifc_1d.png %s_ifc_1d.png png") % (
routes, ts.strftime("%Y%m%d%H"), sector, sector)
m = MapPlot(title=("%s Iowa Flood Center Today's Precipitation") %
(ts.strftime("%-d %b %Y"), ),
subtitle=subtitle,
sector=sector)
xaxis = -97.154167 + np.arange(1741) * 0.004167
yaxis = 40.133331 + np.arange(1057) * 0.004167
(x, y) = np.meshgrid(xaxis, yaxis)
m.pcolormesh(x, y, distance(total, 'MM').value("IN"), clevs, units='inch')
m.drawcounties()
m.postprocess(pqstr=pqstr, view=False)
m.close()
def doit(now):
"""
Generate some plots for the COOP data!
"""
st = NetworkTable([
"IA_COOP",
"MO_COOP",
"KS_COOP",
"NE_COOP",
"SD_COOP",
"ND_ASOS",
"MN_COOP",
"WI_COOP",
"IL_COOP",
"IN_COOP",
"OH_COOP",
"MI_COOP",
])
pgconn = get_dbconn("iem", user="******")
icursor = pgconn.cursor()
clevs = [0, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1, 2, 3, 4, 5, 10]
# We'll assume all COOP data is 12z, sigh for now
sql = """SELECT id, pday, network
from summary_%s s JOIN stations t ON (t.iemid = s.iemid)
WHERE day = '%s' and
t.network ~* 'COOP' and pday >= 0""" % (
now.year,
now.strftime("%Y-%m-%d"),
)
lats = []
lons = []
vals = []
icursor.execute(sql)
iamax = 0.0
for row in icursor:
sid = row[0]
if sid not in st.sts:
continue
# labels.append( id[2:] )
lats.append(st.sts[sid]["lat"])
lons.append(st.sts[sid]["lon"])
vals.append(row[1])
if row[2] == "IA_COOP" and row[1] > iamax:
iamax = row[1]
# Plot Iowa
mp = MapPlot(
sector="iowa",
title="24 Hour NWS COOP Precipitation [inch]",
subtitle=("Ending %s at roughly 12Z") % (now.strftime("%d %B %Y"), ),
)
mp.contourf(lons, lats, vals, clevs, units="inch")
pqstr = ("plot ac %s0000 iowa_coop_12z_precip.png "
"iowa_coop_12z_precip.png png") % (now.strftime("%Y%m%d"), )
mp.drawcounties()
mp.postprocess(pqstr=pqstr)
mp.close()
mp = MapPlot(
sector="midwest",
title="24 Hour NWS COOP Precipitation [inch]",
subtitle=("Ending %s at roughly 12Z") % (now.strftime("%d %B %Y"), ),
)
mp.contourf(lons, lats, vals, clevs, units="inch")
pqstr = ("plot ac %s0000 midwest_coop_12z_precip.png "
"midwest_coop_12z_precip.png png") % (now.strftime("%Y%m%d"), )
mp.postprocess(pqstr=pqstr)
mp.close()
def main(argv):
"""Go Main Go"""
nt = Table("ISUSM")
qdict = loadqc()
idbconn = get_dbconn("isuag", user="******")
pdbconn = get_dbconn("postgis", user="******")
day_ago = int(argv[1])
ts = datetime.date.today() - datetime.timedelta(days=day_ago)
hlons, hlats, hvals = do_nam(ts)
nam = temperature(hvals, "K").value("F")
window = np.ones((3, 3))
nam = convolve2d(nam, window / window.sum(), mode="same", boundary="symm")
# mp = MapPlot(sector='midwest')
# mp.pcolormesh(hlons, hlats, nam,
# range(20, 90, 5))
# mp.postprocess(filename='test.png')
# sys.exit()
# Query out the data
df = read_sql(
"""
WITH ranges as (
select station, count(*), min(tsoil_c_avg_qc),
max(tsoil_c_avg_qc) from sm_hourly WHERE
valid >= %s and valid < %s GROUP by station
)
SELECT d.station, d.tsoil_c_avg_qc,
r.max as hourly_max_c, r.min as hourly_min_c, r.count
from sm_daily d JOIN ranges r on (d.station = r.station)
where valid = %s and tsoil_c_avg_qc > -40 and r.count > 19
""",
idbconn,
params=(ts, ts + datetime.timedelta(days=1), ts),
index_col="station",
)
for col, newcol in zip(
["tsoil_c_avg_qc", "hourly_min_c", "hourly_max_c"],
["ob", "min", "max"],
):
df[newcol] = temperature(df[col].values, "C").value("F")
df.drop(col, axis=1, inplace=True)
for stid, row in df.iterrows():
df.at[stid, "ticket"] = qdict.get(stid, {}).get("soil4", False)
x, y = get_idx(hlons, hlats, nt.sts[stid]["lon"], nt.sts[stid]["lat"])
df.at[stid, "nam"] = nam[x, y]
df.at[stid, "lat"] = nt.sts[stid]["lat"]
df.at[stid, "lon"] = nt.sts[stid]["lon"]
# ticket is an object type from above
df = df[~df["ticket"].astype("bool")]
df["diff"] = df["ob"] - df["nam"]
bias = df["diff"].mean()
nam = nam + bias
print("fancy_4inch NAM bias correction of: %.2fF applied" % (bias, ))
# apply nam bias to sampled data
df["nam"] += bias
df["diff"] = df["ob"] - df["nam"]
# we are going to require data be within 1 SD of sampled or 5 deg
std = 5.0 if df["nam"].std() < 5.0 else df["nam"].std()
for station in df[df["diff"].abs() > std].index.values:
print(("fancy_4inch %s QC'd %s out std: %.2f, ob:%.1f nam:%.1f") % (
ts.strftime("%Y%m%d"),
station,
std,
df.at[station, "ob"],
df.at[station, "nam"],
))
df.drop(station, inplace=True)
# Query out centroids of counties...
cdf = read_sql(
"""SELECT ST_x(ST_centroid(the_geom)) as lon,
ST_y(ST_centroid(the_geom)) as lat
from uscounties WHERE state_name = 'Iowa'
""",
pdbconn,
index_col=None,
)
for i, row in cdf.iterrows():
x, y = get_idx(hlons, hlats, row["lon"], row["lat"])
cdf.at[i, "nam"] = nam[x, y]
mp = MapPlot(
sector="iowa",
title=("Average 4 inch Depth Soil Temperatures for %s") %
(ts.strftime("%b %d, %Y"), ),
subtitle=("County est. based on bias adj. "
"NWS NAM Model (black numbers), "
"ISUSM network observations (red numbers)"),
)
mp.pcolormesh(
hlons,
hlats,
nam,
np.arange(10, 101, 5),
cmap=cm.get_cmap("jet"),
units=r"$^\circ$F",
)
mp.plot_values(df["lon"],
df["lat"],
df["ob"],
fmt="%.0f",
color="r",
labelbuffer=5)
mp.plot_values(
cdf["lon"],
cdf["lat"],
cdf["nam"],
fmt="%.0f",
textsize=11,
labelbuffer=5,
)
mp.drawcounties()
routes = "a" if day_ago >= 4 else "ac"
pqstr = ("plot %s %s0000 soilt_day%s.png isuag_county_4inch_soil.png png"
) % (routes, ts.strftime("%Y%m%d"), day_ago)
mp.postprocess(pqstr=pqstr)
mp.close()
def run(valid, routes):
''' Generate the plot for the given UTC time '''
fn = valid.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/hrrr/%H/"
"hrrr.t%Hz.refd.grib2"))
if not os.path.isfile(fn):
print("hrrr/plot_ref missing %s" % (fn, ))
return
grbs = pygrib.open(fn)
lats = None
lons = None
i = 0
for minute in range(0, HOURS[valid.hour] * 60 + 1, 15):
if minute > (18 * 60) and minute % 60 != 0:
continue
now = valid + datetime.timedelta(minutes=minute)
now = now.astimezone(pytz.timezone("America/Chicago"))
grbs.seek(0)
try:
gs = grbs.select(level=1000,
forecastTime=(minute if minute <=
(18 * 60) else int(minute / 60)))
except ValueError:
continue
if lats is None:
lats, lons = gs[0].latlons()
x1, x2, y1, y2 = compute_bounds(lons, lats)
lats = lats[x1:x2, y1:y2]
lons = lons[x1:x2, y1:y2]
# HACK..............
if len(gs) > 1 and minute > (18 * 60):
reflect = gs[-1]['values'][x1:x2, y1:y2]
else:
reflect = gs[0]['values'][x1:x2, y1:y2]
mp = MapPlot(sector='midwest',
axisbg='tan',
title=('%s UTC NCEP HRRR 1 km AGL Reflectivity') %
(valid.strftime("%-d %b %Y %H"), ),
subtitle=('valid: %s') %
(now.strftime("%-d %b %Y %I:%M %p %Z"), ))
mp.pcolormesh(lons,
lats,
reflect,
np.arange(0, 75, 5),
units='dBZ',
clip_on=False)
pngfn = '/tmp/hrrr_ref_%s_%03i.png' % (valid.strftime("%Y%m%d%H"), i)
mp.postprocess(filename=pngfn)
mp.close()
subprocess.call(("convert %s "
"%s.gif") % (pngfn, pngfn[:-4]),
shell=True)
i += 1
# Generate anim GIF
subprocess.call(("gifsicle --loopcount=0 --delay=50 "
"/tmp/hrrr_ref_%s_???.gif > /tmp/hrrr_ref_%s.gif") %
(valid.strftime("%Y%m%d%H"), valid.strftime("%Y%m%d%H")),
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
pqstr = ("plot %s %s model/hrrr/hrrr_1km_ref.gif "
"model/hrrr/hrrr_1km_ref_%02i.gif gif") % (
routes, valid.strftime("%Y%m%d%H%M"), valid.hour)
subprocess.call(("/home/ldm/bin/pqinsert -p '%s' /tmp/hrrr_ref_%s.gif") %
(pqstr, valid.strftime("%Y%m%d%H")),
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
subprocess.call("rm -f /tmp/hrrr_ref_%s*" % (valid.strftime("%Y%m%d%H"), ),
shell=True)
def run(basets, endts, view):
"""Generate this plot for the given basets"""
pgconn = get_dbconn('postgis', user='******')
df = read_sql("""SELECT state,
max(magnitude) as val, ST_x(geom) as lon, ST_y(geom) as lat
from lsrs WHERE type in ('S') and magnitude >= 0 and
valid > %s and valid < %s GROUP by state, lon, lat
""",
pgconn,
params=(basets, endts),
index_col=None)
df['used'] = False
df['textplot'] = True
df.sort_values(by='val', ascending=False, inplace=True)
# Now, we need to add in zeros, lets say we are looking at a .25 degree box
mybuffer = 0.75
newrows = []
for lat in np.arange(reference.MW_SOUTH, reference.MW_NORTH, mybuffer):
for lon in np.arange(reference.MW_WEST, reference.MW_EAST, mybuffer):
df2 = df[(df['lat'] >= lat) & (df['lat'] < (lat + mybuffer)) &
(df['lon'] >= lon) & (df['lon'] < (lon + mybuffer))]
if df2.empty:
newrows.append(
dict(lon=(lon + mybuffer / 2.),
lat=(lat + mybuffer / 2.),
val=0,
used=True,
textplot=False))
continue
maxval = df.at[df2.index[0], 'val']
df.loc[df2[df2['val'] > (maxval * 0.5)].index, 'used'] = True
df.loc[df2[df2['val'] < (maxval * 0.5)].index, 'textplot'] = False
dfnew = pd.DataFrame(newrows)
df = df.append(dfnew)
cdf = df[df['used']]
tdf = df[df['textplot']]
rng = [0.01, 1, 2, 3, 4, 6, 8, 12, 18, 24, 30, 36]
cmap = nwssnow()
mp = MapPlot(sector='iowa',
axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
if cdf['val'].max() > 0:
mp.contourf(cdf['lon'].values,
cdf['lat'].values,
cdf['val'].values,
rng,
cmap=cmap)
mp.drawcounties()
if len(tdf.index) > 0:
mp.plot_values(tdf['lon'].values,
tdf['lat'].values,
tdf['val'].values,
fmt='%.1f',
labelbuffer=5)
mp.drawcities()
pqstr = "plot c 000000000000 lsr_snowfall.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
# slightly different title to help uniqueness
mp = MapPlot(sector='iowa',
axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports valid over past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
if cdf['val'].max() > 0:
mp.contourf(cdf['lon'].values,
cdf['lat'].values,
cdf['val'].values,
rng,
cmap=cmap)
mp.drawcounties()
mp.drawcities()
pqstr = "plot c 000000000000 lsr_snowfall_nv.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
mp = MapPlot(sector='midwest',
axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
if cdf['val'].max() > 0:
mp.contourf(cdf['lon'].values,
cdf['lat'].values,
cdf['val'].values,
rng,
cmap=cmap)
mp.drawcities()
pqstr = "plot c 000000000000 mw_lsr_snowfall.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
def runYear(year):
sql = """SELECT station, avg(high) as avg_high, avg(low) as avg_low,
avg( (high+low)/2 ) as avg_tmp, max(day)
from alldata_ia WHERE year = %s and station != 'IA0000' and
high is not Null and low is not Null and substr(station,3,1) != 'C'
GROUP by station""" % (year,)
ccursor.execute(sql)
# Plot Average Highs
lats = []
lons = []
vals = []
labels = []
for row in ccursor:
sid = row['station'].upper()
if sid not in nt.sts:
continue
labels.append(sid[2:])
lats.append(nt.sts[sid]['lat'])
lons.append(nt.sts[sid]['lon'])
vals.append(row['avg_high'])
maxday = row['max']
# ---------- Plot the points
m = MapPlot(title="Average Daily High Temperature [F] (%s)" % (year,),
subtitle='1 January - %s' % (maxday.strftime("%d %B"),),
axisbg='white')
m.plot_values(lons, lats, vals, labels=labels, labeltextsize=8,
labelcolor='tan')
pqstr = "plot m %s bogus %s/summary/avg_high.png png" % (
now.strftime("%Y%m%d%H%M"), year,)
m.postprocess(pqstr=pqstr)
m.close()
# Plot Average Lows
lats = []
lons = []
vals = []
labels = []
ccursor.execute(sql)
for row in ccursor:
sid = row['station'].upper()
if sid not in nt.sts:
continue
labels.append(sid[2:])
lats.append(nt.sts[sid]['lat'])
lons.append(nt.sts[sid]['lon'])
vals.append(row['avg_low'])
# ---------- Plot the points
m = MapPlot(title="Average Daily Low Temperature [F] (%s)" % (year,),
subtitle='1 January - %s' % (maxday.strftime("%d %B"),),
axisbg='white')
m.plot_values(lons, lats, vals, labels=labels, labeltextsize=8,
labelcolor='tan')
pqstr = "plot m %s bogus %s/summary/avg_low.png png" % (
now.strftime("%Y%m%d%H%M"), year,)
m.postprocess(pqstr=pqstr)
m.close()
# Plot Average Highs
lats = []
lons = []
vals = []
labels = []
ccursor.execute(sql)
for row in ccursor:
sid = row['station'].upper()
if sid not in nt.sts:
continue
labels.append(sid[2:])
lats.append(nt.sts[sid]['lat'])
lons.append(nt.sts[sid]['lon'])
vals.append(row['avg_tmp'])
# ---------- Plot the points
m = MapPlot(title="Average Daily Temperature [F] (%s)" % (year,),
subtitle='1 January - %s' % (maxday.strftime("%d %B"),),
axisbg='white')
m.plot_values(lons, lats, vals, labels=labels, labeltextsize=8,
labelcolor='tan')
pqstr = "plot m %s bogus %s/summary/avg_temp.png png" % (
now.strftime("%Y%m%d%H%M"), year,)
m.postprocess(pqstr=pqstr)
m.close()
def do(valid):
"""Do Something"""
clevs = np.arange(0, 0.25, 0.05)
clevs = np.append(clevs, np.arange(0.25, 3.0, 0.25))
clevs = np.append(clevs, np.arange(3.0, 10.0, 1))
clevs[0] = 0.01
precip = load_precip(valid)
stage4 = load_precip(valid, "stage4")
inqcprecip = load_precip(valid, "inqcprecip")
outqcprecip = load_precip(valid, "outqcprecip")
multiplier = load_precip(valid, "multiplier")
yidx = int((43.27 - SOUTH) / 0.01)
xidx = int((-94.39 - WEST) / 0.01)
# print(("yidx:%s xidx:%s precip:%.2f stage4: %.2f inqc: %.2f outqc: %.2f "
# "mul: %.2f"
# ) % (yidx, xidx, precip[yidx, xidx], stage4[yidx, xidx],
# inqcprecip[yidx, xidx], outqcprecip[yidx, xidx],
# multiplier[yidx, xidx]))
projection = ccrs.Mercator()
pgconn = get_dbconn("idep")
df = read_postgis(
"""
SELECT ST_Transform(simple_geom, %s) as geom, huc_12,
ST_x(ST_Transform(ST_Centroid(geom), 4326)) as centroid_x,
ST_y(ST_Transform(ST_Centroid(geom), 4326)) as centroid_y,
hu_12_name
from huc12 i WHERE i.scenario = 0 and huc_12 = '071100010901'
""",
pgconn,
params=(projection.proj4_init,),
geom_col="geom",
index_col="huc_12",
)
mp = MapPlot(
sector="custom",
axisbg="white",
west=-92.0,
south=40.24,
east=-91.69,
north=40.4,
subtitle="Flowpath numbers are labelled.",
title=("%s DEP Quality Controlled Precip Totals for 071100010901")
% (valid.strftime("%-d %b %Y"),),
)
(lons, lats) = np.meshgrid(XS, YS)
mp.pcolormesh(lons, lats, precip / 25.4, clevs, units="inch")
mp.drawcounties()
for _huc12, row in df.iterrows():
p = Polygon(row["geom"].exterior, ec="k", fc="None", zorder=100, lw=2)
mp.ax.add_patch(p)
df = read_postgis(
"""
SELECT ST_Transform(geom, %s) as geom, fpath
from flowpaths WHERE scenario = 0 and huc_12 = '071100010901'
""",
pgconn,
params=(projection.proj4_init,),
geom_col="geom",
index_col="fpath",
)
for fpath, row in df.iterrows():
mp.ax.plot(row["geom"].xy[0], row["geom"].xy[1], c="k")
df = read_postgis(
"""
SELECT ST_Transform(st_pointn(geom, 1), %s) as geom, fpath
from flowpaths WHERE scenario = 0 and huc_12 = '071100010901'
""",
pgconn,
params=(projection.proj4_init,),
geom_col="geom",
index_col="fpath",
)
for fpath, row in df.iterrows():
mp.ax.text(row["geom"].x, row["geom"].y, str(fpath), fontsize=12)
mp.postprocess(filename="qc.png")
mp.close()
mp = MapPlot(
sector=SECTOR,
axisbg="white",
title=("%s Stage IV Precip Totals") % (valid.strftime("%-d %b %Y"),),
)
mp.pcolormesh(lons, lats, stage4 / 25.4, clevs, units="inch")
mp.drawcounties()
mp.postprocess(filename="stageIV.png")
mp.close()
mp = MapPlot(
sector=SECTOR,
axisbg="white",
title=("%s High-res total prior to QC")
% (valid.strftime("%-d %b %Y"),),
)
mp.pcolormesh(lons, lats, inqcprecip / 25.4, clevs, units="inch")
mp.drawcounties()
mp.postprocess(filename="inqcprecip.png")
mp.close()
mp = MapPlot(
sector=SECTOR,
axisbg="white",
title=("%s High-res total after QC") % (valid.strftime("%-d %b %Y"),),
)
mp.pcolormesh(lons, lats, outqcprecip / 25.4, clevs, units="inch")
mp.drawcounties()
mp.postprocess(filename="outqcprecip.png")
mp.close()
mp = MapPlot(
sector=SECTOR,
axisbg="white",
title=("%s QC Change in Precip Out - In")
% (valid.strftime("%-d %b %Y"),),
)
diff = (outqcprecip - inqcprecip) / 25.4
mp.pcolormesh(
lons,
lats,
diff,
np.arange(-1.4, 1.5, 0.1),
cmap=plt.get_cmap("BrBG"),
units="inch",
)
mp.drawcounties()
mp.postprocess(filename="qcprecipdiff.png")
mp.close()
mp = MapPlot(
sector=SECTOR,
axisbg="white",
title=("%s multiplier") % (valid.strftime("%-d %b %Y"),),
)
mp.pcolormesh(
lons,
lats,
multiplier,
np.arange(0.0, 2.5, 0.2),
cmap=plt.get_cmap("jet"),
units="ratio",
)
mp.drawcounties()
mp.postprocess(filename="multiplier.png")
mp.close()
mp = MapPlot(
sector=SECTOR,
axisbg="white",
title=("%s manually computed QC Precip mul * stage4")
% (valid.strftime("%-d %b %Y"),),
)
mp.pcolormesh(lons, lats, multiplier * stage4 / 25.4, clevs, units="inch")
mp.drawcounties()
mp.postprocess(filename="qcmancalc.png")
mp.close()
# Go MRMS
ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (valid.year,)
if not os.path.isfile(ncfn):
return
nc = netCDF4.Dataset(ncfn, "r")
xidx = int((WEST - nc.variables["lon"][0]) * 100.0)
yidx = int((SOUTH - nc.variables["lat"][0]) * 100.0)
idx = daily_offset(valid)
mrms = nc.variables["p01d"][idx, yidx : (yidx + 800), xidx : (xidx + 921)]
nc.close()
def run(utc, routes):
""" Generate the plot for the given UTC time """
subprocess.call("python dl_hrrrref.py %s" % (utc.strftime("%Y %m %d %H"),), shell=True)
fn = "/tmp/ncep_hrrr_%s.grib2" % (utc.strftime("%Y%m%d%H"),)
grbs = pygrib.open(fn)
subprocess.call("rm /tmp/hrrr_ref_???.gif", shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
lats = None
lons = None
i = 0
for minute in range(0, 901, 15):
now = utc + datetime.timedelta(minutes=minute)
now = now.astimezone(pytz.timezone("America/Chicago"))
grbs.seek(0)
try:
gs = grbs.select(level=1000, forecastTime=minute)
except:
continue
g = gs[0]
if lats is None:
lats, lons = g.latlons()
x1, x2, y1, y2 = compute_bounds(lons, lats)
lats = lats[x1:x2, y1:y2]
lons = lons[x1:x2, y1:y2]
ref = g["values"][x1:x2, y1:y2]
m = MapPlot(
sector="midwest",
axisbg="tan",
title=("%s UTC NCEP HRRR 1 km AGL Reflectivity") % (utc.strftime("%-d %b %Y %H"),),
subtitle=("valid: %s") % (now.strftime("%-d %b %Y %I:%M %p %Z"),),
)
m.pcolormesh(lons, lats, ref, np.arange(0, 75, 5), units="dBZ", clip_on=False)
m.postprocess(filename="/tmp/hrrr_ref_%03i.png" % (i,))
m.close()
subprocess.call(("convert /tmp/hrrr_ref_%03i.png " "/tmp/hrrr_ref_%03i.gif") % (i, i), shell=True)
i += 1
# Generate anim GIF
subprocess.call(
("gifsicle --loopcount=0 --delay=50 " "/tmp/hrrr_ref_???.gif > /tmp/hrrr_ref.gif"),
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
pqstr = ("plot %s %s model/hrrr/hrrr_1km_ref.gif " "model/hrrr/hrrr_1km_ref_%02i.gif gif") % (
routes,
utc.strftime("%Y%m%d%H%M"),
utc.hour,
)
subprocess.call(
"/home/ldm/bin/pqinsert -p '%s' /tmp/hrrr_ref.gif" % (pqstr,),
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
# os.remove("/tmp/hrrr_ref.gif")
os.remove(fn)
def run(basets, endts, view):
"""Generate this plot for the given basets"""
pgconn = get_dbconn('postgis', user='******')
df = read_sql("""SELECT state,
max(magnitude) as val, ST_x(geom) as lon, ST_y(geom) as lat
from lsrs WHERE type in ('S') and magnitude >= 0 and
valid > %s and valid < %s GROUP by state, lon, lat
""", pgconn, params=(basets, endts), index_col=None)
df.sort_values(by='val', ascending=False, inplace=True)
df['useme'] = False
# First, we need to filter down the in-bound values to get rid of small
cellsize = 0.25
newrows = []
for lat in np.arange(reference.MW_SOUTH, reference.MW_NORTH,
cellsize * 2):
for lon in np.arange(reference.MW_WEST, reference.MW_EAST,
cellsize * 2):
df2 = df[(df['lat'] >= lat) & (df['lat'] < (lat + cellsize * 2)) &
(df['lon'] >= lon) & (df['lon'] < (lon + cellsize * 2))]
if df2.empty:
df3 = df[(df['lat'] >= lat) &
(df['lat'] < (lat + cellsize * 4)) &
(df['lon'] >= lon) &
(df['lon'] < (lon + cellsize * 4))]
if df3.empty:
newrows.append({'lon': lon + cellsize * 1.5,
'lat': lat + cellsize * 1.5,
'val': 0,
'useme': True,
'state': 'NA'})
continue
maxval = df.at[df2.index[0], 'val']
df.loc[df2[df2['val'] > (maxval * 0.8)].index, 'useme'] = True
dfall = pd.concat([df, pd.DataFrame(newrows)], ignore_index=True)
df2 = dfall[dfall['useme']]
xi = np.arange(reference.MW_WEST, reference.MW_EAST, cellsize)
yi = np.arange(reference.MW_SOUTH, reference.MW_NORTH, cellsize)
xi, yi = np.meshgrid(xi, yi)
gridder = Rbf(df2['lon'].values, df2['lat'].values,
pd.to_numeric(df2['val'].values, errors='ignore'),
function='thin_plate')
vals = gridder(xi, yi)
vals[np.isnan(vals)] = 0
window = np.ones((2, 2))
vals = convolve2d(vals, window / window.sum(), mode='same',
boundary='symm')
vals[vals < 0.1] = 0
rng = [0.01, 1, 2, 3, 4, 6, 8, 12, 18, 24, 30, 36]
cmap = nwssnow()
mp = MapPlot(sector='iowa', axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
if df['val'].max() > 0:
mp.contourf(xi, yi, vals, rng, cmap=cmap)
mp.drawcounties()
if not df.empty:
mp.plot_values(df['lon'].values, df['lat'].values, df['val'].values,
fmt='%.1f', labelbuffer=2)
mp.drawcities()
pqstr = "plot c 000000000000 lsr_snowfall.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
# slightly different title to help uniqueness
mp = MapPlot(sector='iowa', axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports valid over past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
if df['val'].max() > 0:
mp.contourf(xi, yi, vals, rng, cmap=cmap)
mp.drawcounties()
mp.drawcities()
pqstr = "plot c 000000000000 lsr_snowfall_nv.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
mp = MapPlot(sector='midwest', axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
if df['val'].max() > 0:
mp.contourf(xi, yi, vals, rng, cmap=cmap)
mp.drawcities()
pqstr = "plot c 000000000000 mw_lsr_snowfall.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
def main():
"""Go Main Go"""
st = NetworkTable('IACLIMATE')
pgconn = get_dbconn('coop', user='******')
ccursor = pgconn.cursor()
ts = datetime.datetime.now() - datetime.timedelta(days=1)
nrain = []
lats = []
lons = []
# Get normals!
ccursor.execute("""SELECT station, sum(precip) as acc from climate51
WHERE valid <= '2000-%s' and station NOT IN ('IA7842','IA4381')
and substr(station,0,3) = 'IA'
GROUP by station ORDER by acc ASC""" % (ts.strftime("%m-%d"), ))
for row in ccursor:
station = row[0]
if station not in st.sts:
continue
nrain.append(row[1])
lats.append(st.sts[station]['lat'])
lons.append(st.sts[station]['lon'])
mp = MapPlot(axisbg='white',
title=("Iowa %s Normal Precipitation Accumulation") %
(ts.strftime("%Y"), ),
subtitle="1 Jan - %s" % (ts.strftime("%d %b %Y"), ))
rng = np.arange(int(min(nrain)) - 1, int(max(nrain)) + 1)
if max(nrain) < 10:
rng = np.arange(0, 10)
mp.contourf(lons, lats, nrain, rng, units='inch')
pqstr = "plot c 000000000000 summary/year/normals.png bogus png"
mp.postprocess(view=False, pqstr=pqstr)
mp.close()
# ----------------------------------
# - Compute departures
nrain = []
lats = []
lons = []
ccursor.execute("""select climate.station, norm, obs from
(select c.station, sum(c.precip) as norm from climate51 c
where c.valid < '2000-%s' and substr(station,0,3) = 'IA'
GROUP by c.station) as climate,
(select a.station, sum(a.precip) as obs from alldata a
WHERE a.year = %s and substr(a.station,0,3) = 'IA'
GROUP by station) as obs
WHERE obs.station = climate.station""" % (ts.strftime("%m-%d"), ts.year))
for row in ccursor:
station = row[0]
if station not in st.sts:
continue
nrain.append(row[2] - row[1])
lats.append(st.sts[station]['lat'])
lons.append(st.sts[station]['lon'])
mp = MapPlot(axisbg='white',
title=("Iowa %s Precipitation Depature") %
(ts.strftime("%Y"), ),
subtitle="1 Jan - %s" % (ts.strftime("%d %b %Y"), ))
rng = np.arange(int(min(nrain)) - 1, int(max(nrain)) + 1)
if max(nrain) < 10:
rng = np.arange(0, 10)
mp.contourf(lons, lats, nrain, rng, units='inch')
pqstr = "plot c 000000000000 summary/year/diff.png bogus png"
mp.postprocess(view=False, pqstr=pqstr)
mp.close()
def main():
"""Go Main Go"""
# Run for a period of 121 days
ets = datetime.datetime.now() - datetime.timedelta(days=1)
sts = ets - datetime.timedelta(days=121)
# Get the normal accumm
cnc = netCDF4.Dataset("/mesonet/data/iemre/mw_dailyc.nc", 'r')
lons = cnc.variables['lon'][:]
lats = cnc.variables['lat'][:]
index0 = iemre.daily_offset(sts)
index1 = iemre.daily_offset(ets)
if index1 < index0: # Uh oh, we are spanning a year
clprecip = np.sum(cnc.variables['p01d'][:index1, :, :], 0)
clprecip = clprecip + np.sum(cnc.variables['p01d'][index0:, :, :], 0)
else:
clprecip = np.sum(cnc.variables['p01d'][index0:index1, :, :], 0)
cnc.close()
# Get the observed precip
if sts.year != ets.year: # spanner, darn
onc = netCDF4.Dataset(
"/mesonet/data/iemre/%s_mw_daily.nc" % (sts.year,))
obprecip = np.sum(onc.variables['p01d'][index0:, :, :], 0)
onc.close()
onc = netCDF4.Dataset(
"/mesonet/data/iemre/%s_mw_daily.nc" % (ets.year,))
obprecip = obprecip + np.sum(onc.variables['p01d'][:index1, :, :], 0)
onc.close()
else:
ncfn = "/mesonet/data/iemre/%s_mw_daily.nc" % (sts.year,)
onc = netCDF4.Dataset(ncfn, 'r')
obprecip = np.sum(onc.variables['p01d'][index0:index1, :, :], 0)
onc.close()
lons, lats = np.meshgrid(lons, lats)
# Plot departure from normal
mp = MapPlot(sector='midwest',
title=('Precipitation Departure %s - %s'
) % (sts.strftime("%b %d %Y"),
ets.strftime("%b %d %Y")),
subtitle='based on IEM Estimates'
)
mp.pcolormesh(lons, lats, (obprecip - clprecip) / 25.4,
np.arange(-10, 10, 1))
mp.postprocess(
pqstr="plot c 000000000000 summary/4mon_diff.png bogus png")
mp.close()
# Plot normals
mp = MapPlot(sector='midwest',
title=('Normal Precipitation:: %s - %s'
) % (sts.strftime("%b %d %Y"),
ets.strftime("%b %d %Y")),
subtitle='based on IEM Estimates'
)
mp.pcolormesh(lons, lats, (clprecip) / 25.4, np.arange(0, 30, 2))
mp.postprocess(
pqstr="plot c 000000000000 summary/4mon_normals.png bogus png")
mp.close()
# Plot Obs
mp = MapPlot(sector='midwest',
title=('Estimated Precipitation:: %s - %s'
) % (sts.strftime("%b %d %Y"),
ets.strftime("%b %d %Y")),
subtitle='based on IEM Estimates'
)
mp.pcolormesh(lons, lats, (obprecip) / 25.4, np.arange(0, 30, 2))
mp.postprocess(
pqstr="plot c 000000000000 summary/4mon_stage4obs.png bogus png")
mp.close()
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
We should total files from 1 AM to midnight local time
"""
sts = ts.replace(hour=1)
ets = sts + datetime.timedelta(hours=24)
interval = datetime.timedelta(hours=1)
now = sts
total = None
lts = None
while now < ets:
gmt = now.astimezone(pytz.utc)
fn = gmt.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/"
"stage4/ST4.%Y%m%d%H.01h.grib"))
if os.path.isfile(fn):
lts = now
grbs = pygrib.open(fn)
if total is None:
total = grbs[1]["values"]
lats, lons = grbs[1].latlons()
else:
total += grbs[1]["values"]
grbs.close()
now += interval
if lts is None:
if ts.hour > 1:
LOG.info("found no data for date: %s", ts)
return
lts = lts - datetime.timedelta(minutes=1)
subtitle = "Total between 12:00 AM and %s" % (
lts.strftime("%I:%M %p %Z"), )
routes = "ac"
if not realtime:
routes = "a"
total = masked_array(total, units("mm")).to(units("inch")).m
for sector in ["iowa", "midwest", "conus"]:
pqstr = ("plot %s %s00 %s_stage4_1d.png %s_stage4_1d.png png") % (
routes,
ts.strftime("%Y%m%d%H"),
sector,
sector,
)
mp = MapPlot(
sector=sector,
title=("%s NCEP Stage IV Today's Precipitation") %
(ts.strftime("%-d %b %Y"), ),
subtitle=subtitle,
)
clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
mp.pcolormesh(lons, lats, total, clevs, cmap=nwsprecip(), units="inch")
# map.drawstates(zorder=2)
if sector == "iowa":
mp.drawcounties()
mp.postprocess(pqstr=pqstr)
mp.close()
def run(basets, endts, view):
"""Generate this plot for the given basets"""
df = read_sql("""SELECT state,
max(magnitude) as val, ST_x(geom) as lon, ST_y(geom) as lat
from lsrs WHERE type in ('S') and magnitude >= 0 and
valid > %s and valid < %s GROUP by state, lon, lat
""", POSTGIS, params=(basets, endts), index_col=None)
df['used'] = False
df['textplot'] = True
df.sort_values(by='val', ascending=False, inplace=True)
# Now, we need to add in zeros, lets say we are looking at a .25 degree box
mybuffer = 0.75
newrows = []
for lat in np.arange(reference.MW_SOUTH, reference.MW_NORTH, mybuffer):
for lon in np.arange(reference.MW_WEST, reference.MW_EAST, mybuffer):
df2 = df[(df['lat'] >= lat) & (df['lat'] < (lat+mybuffer)) &
(df['lon'] >= lon) & (df['lon'] < (lon+mybuffer))]
if len(df2.index) == 0:
newrows.append(dict(lon=(lon+mybuffer/2.),
lat=(lat+mybuffer/2.),
val=0, used=True, textplot=False))
continue
maxval = df.at[df2.index[0], 'val']
df.loc[df2[df2['val'] > (maxval * 0.5)].index, 'used'] = True
df.loc[df2[df2['val'] < (maxval * 0.5)].index, 'textplot'] = False
dfnew = pd.DataFrame(newrows)
df = df.append(dfnew)
cdf = df[df['used']]
tdf = df[df['textplot']]
rng = [0.01, 1, 2, 3, 4, 6, 8, 12, 18, 24, 30, 36]
cmap = nwssnow()
m = MapPlot(sector='iowa', axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
m.contourf(cdf['lon'].values, cdf['lat'].values, cdf['val'].values, rng,
cmap=cmap)
m.drawcounties()
m.plot_values(tdf['lon'].values, tdf['lat'].values, tdf['val'].values,
fmt='%.1f')
pqstr = "plot c 000000000000 lsr_snowfall.png bogus png"
m.postprocess(view=view, pqstr=pqstr)
m.close()
# slightly different title to help uniqueness
m = MapPlot(sector='iowa', axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports valid over past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
m.contourf(cdf['lon'].values, cdf['lat'].values, cdf['val'].values, rng,
cmap=cmap)
m.drawcounties()
pqstr = "plot c 000000000000 lsr_snowfall_nv.png bogus png"
m.postprocess(view=view, pqstr=pqstr)
m.close()
m = MapPlot(sector='midwest', axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
m.contourf(cdf['lon'].values, cdf['lat'].values, cdf['val'].values, rng,
cmap=cmap)
pqstr = "plot c 000000000000 mw_lsr_snowfall.png bogus png"
m.postprocess(view=view, pqstr=pqstr)
m.close()
def main():
"""Go Main Go"""
# Run for a period of 121 days
ets = datetime.datetime.now() - datetime.timedelta(days=1)
sts = ets - datetime.timedelta(days=121)
# Get the normal accumm
cnc = ncopen(iemre.get_dailyc_ncname(), "r")
lons = cnc.variables["lon"][:]
lats = cnc.variables["lat"][:]
index0 = iemre.daily_offset(sts)
index1 = iemre.daily_offset(ets)
if index1 < index0: # Uh oh, we are spanning a year
clprecip = np.sum(cnc.variables["p01d"][:index1, :, :], 0)
clprecip = clprecip + np.sum(cnc.variables["p01d"][index0:, :, :], 0)
else:
clprecip = np.sum(cnc.variables["p01d"][index0:index1, :, :], 0)
cnc.close()
# Get the observed precip
if sts.year != ets.year: # spanner, darn
onc = ncopen(iemre.get_daily_ncname(sts.year))
obprecip = np.sum(onc.variables["p01d"][index0:, :, :], 0)
onc.close()
onc = ncopen(iemre.get_daily_ncname(ets.year))
obprecip = obprecip + np.sum(onc.variables["p01d"][:index1, :, :], 0)
onc.close()
else:
ncfn = iemre.get_daily_ncname(sts.year)
onc = ncopen(ncfn, "r")
obprecip = np.sum(onc.variables["p01d"][index0:index1, :, :], 0)
onc.close()
lons, lats = np.meshgrid(lons, lats)
# Plot departure from normal
mp = MapPlot(
sector="midwest",
title=("Precipitation Departure %s - %s") %
(sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")),
subtitle="based on IEM Estimates",
)
mp.pcolormesh(lons, lats, (obprecip - clprecip) / 25.4,
np.arange(-10, 10, 1))
mp.postprocess(pqstr="plot c 000000000000 summary/4mon_diff.png bogus png")
mp.close()
# Plot normals
mp = MapPlot(
sector="midwest",
title=("Normal Precipitation:: %s - %s") %
(sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")),
subtitle="based on IEM Estimates",
)
mp.pcolormesh(lons, lats, (clprecip) / 25.4, np.arange(0, 30, 2))
mp.postprocess(
pqstr="plot c 000000000000 summary/4mon_normals.png bogus png")
mp.close()
# Plot Obs
mp = MapPlot(
sector="midwest",
title=("Estimated Precipitation:: %s - %s") %
(sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")),
subtitle="based on IEM Estimates",
)
mp.pcolormesh(lons, lats, (obprecip) / 25.4, np.arange(0, 30, 2))
mp.postprocess(
pqstr="plot c 000000000000 summary/4mon_stage4obs.png bogus png")
mp.close()
utcnow = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
for i, lon in enumerate(tqdm(lons)):
for j, lat in enumerate(lats):
cursor.execute("""
select eventid from sbw where
ST_Covers(geom, ST_GeomFromEWKT('SRID=4326;POINT(%s %s)'))
and status = 'NEW' and phenomena = 'TO' and significance = 'W'
and issue < '2017-01-01' and issue > '2002-01-01'
""", (lon, lat))
val = cursor.rowcount
vals[j, i] = val / 15.0 # Number of years 2002-2016
print("Maximum is: %.1f" % (np.max(vals),))
m = MapPlot(sector='iowa',
title='Avg Number of Storm Based Tornado Warnings per Year',
subtitle=("(2003 through 2016) based on unofficial "
"archives maintained by the IEM, %sx%s analysis grid"
) % (dx, dx))
cmap = plt.get_cmap('jet')
cmap.set_under('white')
cmap.set_over('black')
lons, lats = np.meshgrid(lons, lats)
rng = np.arange(0, 2.1, 0.2)
rng[0] = 0.01
m.pcolormesh(lons, lats, vals,
rng, cmap=cmap, units='count')
m.drawcounties()
m.postprocess(filename='count.png')
m.close()
def main():
"""Go Main Go"""
pgconn = get_dbconn("iem", user="******")
now = utc().replace(hour=0, minute=0, second=0, microsecond=0)
df = read_sql(
"""
WITH lows as (
SELECT c.iemid,
min(tmpf) as calc_low,
min(min_tmpf_6hr) as reported_low
from current_log c JOIN stations s
ON (s.iemid = c.iemid)
WHERE valid > %s and valid < %s
and (s.network ~* 'ASOS' or s.network = 'AWOS')
and s.country = 'US' and s.state not in ('HI', 'AK') GROUP by c.iemid
)
select t.id, t.state, ST_x(t.geom) as lon, ST_y(t.geom) as lat,
least(l.calc_low, l.reported_low) as low from
lows l JOIN stations t on (t.iemid = l.iemid)
""",
pgconn,
params=(now, now.replace(hour=12)),
index_col="id",
)
df = df[df["low"].notnull()]
LOG.debug("found %s observations for %s", len(df.index), now)
for sector in [
"iowa",
"midwest",
"conus",
"SD",
"NE",
"ND",
"KS",
"MN",
"MO",
"WI",
"IL",
]:
mp = MapPlot(
sector=sector if len(sector) > 2 else "state",
state=sector if len(sector) == 2 else "IA",
title="%s ASOS/AWOS 01-12 UTC Low Temperature" %
(now.strftime("%-d %b %Y"), ),
subtitle=("includes available 6z and 12z 6-hr mins, "
"does not include 0z observation"),
axisbg="white",
)
if sector == "iowa" or len(sector) == 2:
df2 = df[df["state"] == ("IA" if len(sector) > 2 else sector)]
labels = df2.index.values
mp.drawcounties()
size = 14
else:
df2 = df
labels = None
size = 10
mp.plot_values(
df2["lon"].values,
df2["lat"].values,
df2["low"],
fmt="%.0f",
labels=labels,
labelbuffer=1,
textsize=size,
)
pqstr = ("plot ac %s summary/%s_asos_12z_low.png "
"%s_asos_12z_low.png png") % (now.strftime("%Y%m%d%H%M"),
sector.lower(), sector.lower())
LOG.debug(pqstr)
mp.postprocess(pqstr=pqstr)
mp.close()
avg( (max_tmpf + min_tmpf)/2.0 ) as avgt , count(*) as cnt
from """ + table + """ c JOIN stations s ON (s.iemid = c.iemid)
WHERE s.network in ('IA_ASOS', 'AWOS') and
day >= %s and day < %s
and max_tmpf > -30 and min_tmpf < 90 GROUP by id, s.network, lon, lat
""", (day1, day2))
for row in icursor:
if row['cnt'] != now.day:
continue
lats.append(row['lat'])
lons.append(row['lon'])
vals.append(row['avgt'])
valmask.append(row['network'] in ['AWOS', 'IA_ASOS'])
if len(vals) < 3:
sys.exit()
m = MapPlot(axisbg='white',
title="Iowa %s Average Temperature" % (now.strftime("%Y %B"), ),
subtitle=("Average of the High + Low ending: %s"
"") % (now.strftime("%d %B"), ))
minval = int(min(vals))
maxval = max([int(max(vals)) + 3, minval + 11])
clevs = np.linspace(minval, maxval, 10, dtype='i')
m.contourf(lons, lats, vals, clevs)
m.drawcounties()
m.plot_values(lons, lats, vals, '%.1f')
pqstr = "plot c 000000000000 summary/mon_mean_T.png bogus png"
m.postprocess(view=False, pqstr=pqstr)
m.close()
def run(basets, endts, view):
"""Generate this plot for the given basets"""
pgconn = get_dbconn("postgis", user="******")
df = read_sql(
"""SELECT state,
max(magnitude) as val, ST_x(geom) as lon, ST_y(geom) as lat
from lsrs WHERE type in ('S') and magnitude >= 0 and
valid > %s and valid < %s GROUP by state, lon, lat
""",
pgconn,
params=(basets, endts),
index_col=None,
)
df.sort_values(by="val", ascending=False, inplace=True)
df["useme"] = False
# First, we need to filter down the in-bound values to get rid of small
cellsize = 0.33
newrows = []
for lat in np.arange(reference.MW_SOUTH, reference.MW_NORTH, cellsize):
for lon in np.arange(reference.MW_WEST, reference.MW_EAST, cellsize):
# Look around this box at 1x
df2 = df[(df["lat"] >= (lat - cellsize))
& (df["lat"] < (lat + cellsize))
& (df["lon"] >= (lon - cellsize))
& (df["lon"] < (lon + cellsize))]
if df2.empty:
# If nothing was found, check 2x
df3 = df[(df["lat"] >= (lat - cellsize * 2.0))
& (df["lat"] < (lat + cellsize * 2.0))
& (df["lon"] >= (lon - cellsize * 2.0))
& (df["lon"] < (lon + cellsize * 2.0))]
if df3.empty:
# If nothing found, place a zero here
newrows.append({
"lon": lon,
"lat": lat,
"val": 0,
"useme": True,
"state": "NA",
})
continue
maxval = df.at[df2.index[0], "val"]
df.loc[df2[df2["val"] > (maxval * 0.8)].index, "useme"] = True
dfall = pd.concat([df, pd.DataFrame(newrows)], ignore_index=True)
df2 = dfall[dfall["useme"]]
xi = np.arange(reference.MW_WEST, reference.MW_EAST, cellsize)
yi = np.arange(reference.MW_SOUTH, reference.MW_NORTH, cellsize)
xi, yi = np.meshgrid(xi, yi)
gridder = Rbf(
df2["lon"].values,
df2["lat"].values,
pd.to_numeric(df2["val"].values, errors="ignore"),
function="thin_plate",
)
vals = gridder(xi, yi)
vals[np.isnan(vals)] = 0
rng = [0.01, 1, 2, 3, 4, 6, 8, 12, 18, 24, 30, 36]
cmap = nwssnow()
mp = MapPlot(
sector="iowa",
axisbg="white",
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )),
)
if df["val"].max() > 0:
mp.contourf(xi, yi, vals, rng, cmap=cmap)
mp.drawcounties()
if not df.empty:
mp.plot_values(
df["lon"].values,
df["lat"].values,
df["val"].values,
fmt="%.1f",
labelbuffer=2,
)
mp.drawcities()
pqstr = "plot c 000000000000 lsr_snowfall.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
# slightly different title to help uniqueness
mp = MapPlot(
sector="iowa",
axisbg="white",
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports valid over past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )),
)
if df["val"].max() > 0:
mp.contourf(xi, yi, vals, rng, cmap=cmap)
mp.drawcounties()
mp.drawcities()
pqstr = "plot c 000000000000 lsr_snowfall_nv.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
mp = MapPlot(
sector="midwest",
axisbg="white",
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )),
)
if df["val"].max() > 0:
mp.contourf(xi, yi, vals, rng, cmap=cmap)
mp.drawcities()
pqstr = "plot c 000000000000 mw_lsr_snowfall.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
def run(utc, routes):
''' Generate the plot for the given UTC time '''
fn = utc.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/hrrr/%H/"
"hrrr.t%Hz.refd.grib2"))
grbs = pygrib.open(fn)
subprocess.call("rm /tmp/hrrr_ref_???.gif",
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
lats = None
lons = None
i = 0
for minute in range(0, 18 * 60 + 1, 15):
now = utc + datetime.timedelta(minutes=minute)
now = now.astimezone(pytz.timezone("America/Chicago"))
grbs.seek(0)
try:
gs = grbs.select(level=1000, forecastTime=minute)
except ValueError:
continue
if lats is None:
lats, lons = gs[0].latlons()
x1, x2, y1, y2 = compute_bounds(lons, lats)
lats = lats[x1:x2, y1:y2]
lons = lons[x1:x2, y1:y2]
reflect = gs[0]['values'][x1:x2, y1:y2]
mp = MapPlot(sector='midwest',
axisbg='tan',
title=('%s UTC NCEP HRRR 1 km AGL Reflectivity') %
(utc.strftime("%-d %b %Y %H"), ),
subtitle=('valid: %s') %
(now.strftime("%-d %b %Y %I:%M %p %Z"), ))
mp.pcolormesh(lons,
lats,
reflect,
np.arange(0, 75, 5),
units='dBZ',
clip_on=False)
mp.postprocess(filename='/tmp/hrrr_ref_%03i.png' % (i, ))
mp.close()
subprocess.call(("convert /tmp/hrrr_ref_%03i.png "
"/tmp/hrrr_ref_%03i.gif") % (i, i),
shell=True)
i += 1
# Generate anim GIF
subprocess.call(("gifsicle --loopcount=0 --delay=50 "
"/tmp/hrrr_ref_???.gif > /tmp/hrrr_ref.gif"),
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
pqstr = ("plot %s %s model/hrrr/hrrr_1km_ref.gif "
"model/hrrr/hrrr_1km_ref_%02i.gif gif") % (
routes, utc.strftime("%Y%m%d%H%M"), utc.hour)
subprocess.call("/home/ldm/bin/pqinsert -p '%s' /tmp/hrrr_ref.gif" %
(pqstr, ),
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
os.remove("/tmp/hrrr_ref.gif")
def draw_map():
"""make maps, not war."""
m = MapPlot(sector="conus", title="4 March 2019 :: DEP Precip Points")
update_grid(m.ax)
m.postprocess(filename="/tmp/map_clipoints.png")
m.close()
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
# Start at 1 AM
now = ts.replace(hour=1, minute=0)
ets = now + datetime.timedelta(hours=24)
interval = datetime.timedelta(hours=1)
currenttime = datetime.datetime.utcnow()
currenttime = currenttime.replace(tzinfo=pytz.timezone("UTC"))
total = None
lastts = None
while now < ets:
gmt = now.astimezone(pytz.timezone("UTC"))
if gmt > currenttime:
break
for prefix in ['GaugeCorr', 'RadarOnly']:
gribfn = gmt.strftime(
("/mnt/a4/data/%Y/%m/%d/mrms/ncep/" + prefix + "_QPE_01H/" +
prefix + "_QPE_01H_00.00_%Y%m%d-%H%M00"
".grib2.gz"))
if os.path.isfile(gribfn):
break
if not os.path.isfile(gribfn):
print("q3_today_total.py MISSING %s" % (gribfn, ))
now += interval
continue
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb['values']
else:
maxgrid = np.maximum(grb['values'], total)
total = np.where(np.logical_and(grb['values'] >= 0, total >= 0),
grb['values'] + total, maxgrid)
lastts = now
now += interval
if lastts is None:
print(('No MRMS Q3 Data found for date: %s') %
(now.strftime("%d %B %Y"), ))
return
lastts = lastts - datetime.timedelta(minutes=1)
subtitle = "Total between 12:00 AM and %s" % (
lastts.strftime("%I:%M %p %Z"), )
routes = 'ac'
if not realtime:
routes = 'a'
clevs = np.arange(0, 0.25, 0.05)
clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs[0] = 0.01
sector = 'iowa'
pqstr = ("plot %s %s00 %s_q2_1d.png %s_q2_1d.png png") % (
routes, ts.strftime("%Y%m%d%H"), sector, sector)
m = MapPlot(title=("%s NCEP MRMS Q3 Today's Precipitation") %
(ts.strftime("%-d %b %Y"), ),
subtitle=subtitle,
sector=sector)
(x, y) = np.meshgrid(mrms.XAXIS, mrms.YAXIS)
m.pcolormesh(x, y, np.flipud(total) / 24.5, clevs, units='inch')
m.drawcounties()
m.postprocess(pqstr=pqstr, view=False)
m.close()
def doit(now, model):
""" Figure out the model runtime we care about """
mcursor.execute("""
SELECT max(runtime at time zone 'UTC') from alldata where station = 'KDSM'
and ftime = %s and model = %s
""", (now, model))
row = mcursor.fetchone()
runtime = row[0]
if runtime is None:
sys.exit()
runtime = runtime.replace(tzinfo=pytz.timezone("UTC"))
# Load up the mos forecast for our given
mcursor.execute("""
SELECT station, tmp FROM alldata
WHERE model = %s and runtime = %s and ftime = %s and tmp < 999
""", (model, runtime, now))
forecast = {}
for row in mcursor:
if row[0][0] == 'K':
forecast[row[0][1:]] = row[1]
# Load up the currents!
icursor.execute("""
SELECT
s.id, s.network, tmpf, ST_x(s.geom) as lon, ST_y(s.geom) as lat
FROM
current c, stations s
WHERE
c.iemid = s.iemid and
(s.network ~* 'ASOS' or s.network = 'AWOS') and s.country = 'US' and
valid + '60 minutes'::interval > now() and
tmpf > -50
""")
lats = []
lons = []
vals = []
#valmask = []
for row in icursor:
if not forecast.has_key(row[0]):
continue
diff = forecast[row[0]] - row[2]
if diff > 20 or diff < -20:
continue
lats.append(row[4])
lons.append(row[3])
vals.append(diff)
#valmask.append((row[1] in ['AWOS', 'IA_AWOS']))
cmap = cm.get_cmap("RdYlBu_r")
cmap.set_under('black')
cmap.set_over('black')
localnow = now.astimezone(pytz.timezone("America/Chicago"))
m = MapPlot(sector='midwest',
title="%s MOS Temperature Bias " % (model,),
subtitle='Model Run: %s Forecast Time: %s' % (
runtime.strftime("%d %b %Y %H %Z"),
localnow.strftime("%d %b %Y %-I %p %Z")))
m.contourf(lons, lats, vals, range(-10, 11, 2), units='F', cmap=cmap)
pqstr = "plot ac %s00 %s_mos_T_bias.png %s_mos_T_bias_%s.png png" % (
now.strftime("%Y%m%d%H"), model.lower(),
model.lower(), now.strftime("%H"))
m.postprocess(pqstr=pqstr, view=False)
m.close()
m = MapPlot(sector='conus',
title="%s MOS Temperature Bias " % (model,),
subtitle='Model Run: %s Forecast Time: %s' % (
runtime.strftime("%d %b %Y %H %Z"),
localnow.strftime("%d %b %Y %-I %p %Z"))
)
m.contourf(lons, lats, vals, range(-10, 11, 2), units='F', cmap=cmap)
pqstr = ("plot ac %s00 conus_%s_mos_T_bias.png "
+"conus_%s_mos_T_bias_%s.png png") % (
now.strftime("%Y%m%d%H"), model.lower(),
model.lower(), now.strftime("%H"))
m.postprocess(pqstr=pqstr, view=False)
def doit(ts):
"""
Generate hourly plot of stage4 data
"""
gmtnow = datetime.datetime.utcnow()
gmtnow = gmtnow.replace(tzinfo=pytz.utc)
routes = "a"
if ((gmtnow - ts).days * 86400.0 + (gmtnow - ts).seconds) < 7200:
routes = "ac"
fn = "/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib" % (
ts.strftime("%Y/%m/%d"),
ts.strftime("%Y%m%d%H"),
)
if not os.path.isfile(fn):
LOG.info("Missing stage4 %s", fn)
return
grbs = pygrib.open(fn)
grib = grbs[1]
lats, lons = grib.latlons()
vals = grib.values / 25.4
cmap = cm.get_cmap("jet")
cmap.set_under("white")
cmap.set_over("black")
clevs = [
0.01,
0.05,
0.1,
0.2,
0.3,
0.4,
0.5,
0.6,
0.7,
0.8,
0.9,
1,
1.5,
2,
3,
]
localtime = ts.astimezone(pytz.timezone("America/Chicago"))
for sector in ["iowa", "midwest", "conus"]:
mp = MapPlot(
sector=sector,
title="Stage IV One Hour Precipitation",
subtitle="Hour Ending %s" %
(localtime.strftime("%d %B %Y %I %p %Z"), ),
)
mp.pcolormesh(lons, lats, vals, clevs, units="inch")
pqstr = "plot %s %s00 %s_stage4_1h.png %s_stage4_1h_%s.png png" % (
routes,
ts.strftime("%Y%m%d%H"),
sector,
sector,
ts.strftime("%H"),
)
if sector == "iowa":
mp.drawcounties()
mp.postprocess(view=False, pqstr=pqstr)
mp.close()
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
# Start at midnight
now = ts.replace(hour=0, minute=0)
ets = now + datetime.timedelta(hours=24)
interval = datetime.timedelta(minutes=5)
currenttime = datetime.datetime.utcnow()
currenttime = currenttime.replace(tzinfo=pytz.timezone("UTC"))
total = None
lastts = None
while now <= ets:
gmt = now.astimezone(pytz.timezone("UTC"))
if gmt > currenttime:
break
fn = gmt.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/"
"GIS/ifc/p05m_%Y%m%d%H%M.png"))
if not os.path.isfile(fn):
now += interval
continue
png = gdal.Open(fn, 0)
data = np.flipud(png.ReadAsArray()) # units are mm per 5 minutes
data = np.where(data > 254, 0, data) / 10.0
if total is None:
total = data
else:
total += data
lastts = now
now += interval
if lastts is None:
print(('No IFC Data found for date: %s'
) % (now.strftime("%d %B %Y"),))
return
lastts = lastts - datetime.timedelta(minutes=1)
subtitle = "Total between 12:00 AM and %s" % (
lastts.strftime("%I:%M %p %Z"),)
routes = 'ac'
if not realtime:
routes = 'a'
clevs = np.arange(0, 0.25, 0.05)
clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs[0] = 0.01
sector = 'iowa'
pqstr = ("plot %s %s00 %s_ifc_1d.png %s_ifc_1d.png png"
) % (routes, ts.strftime("%Y%m%d%H"), sector, sector)
m = MapPlot(title=("%s Iowa Flood Center Today's Precipitation"
) % (ts.strftime("%-d %b %Y"),),
subtitle=subtitle, sector=sector)
xaxis = -97.154167 + np.arange(1741) * 0.004167
yaxis = 40.133331 + np.arange(1057) * 0.004167
(x, y) = np.meshgrid(xaxis, yaxis)
m.pcolormesh(x, y, distance(total, 'MM').value("IN"), clevs, units='inch')
m.drawcounties()
m.postprocess(pqstr=pqstr, view=False)
m.close()
def runYear(year):
sql = """SELECT station, avg(high) as avg_high, avg(low) as avg_low,
avg( (high+low)/2 ) as avg_tmp, max(day)
from alldata_ia WHERE year = %s and station != 'IA0000' and
high is not Null and low is not Null and substr(station,3,1) != 'C'
GROUP by station""" % (year, )
ccursor.execute(sql)
# Plot Average Highs
lats = []
lons = []
vals = []
labels = []
for row in ccursor:
sid = row['station'].upper()
if sid not in nt.sts:
continue
labels.append(sid[2:])
lats.append(nt.sts[sid]['lat'])
lons.append(nt.sts[sid]['lon'])
vals.append(row['avg_high'])
maxday = row['max']
# ---------- Plot the points
m = MapPlot(title="Average Daily High Temperature [F] (%s)" % (year, ),
subtitle='1 January - %s' % (maxday.strftime("%d %B"), ),
axisbg='white')
m.plot_values(lons,
lats,
vals,
labels=labels,
labeltextsize=8,
labelcolor='tan',
fmt='%.1f')
pqstr = "plot m %s bogus %s/summary/avg_high.png png" % (
now.strftime("%Y%m%d%H%M"),
year,
)
m.postprocess(pqstr=pqstr)
m.close()
# Plot Average Lows
lats = []
lons = []
vals = []
labels = []
ccursor.execute(sql)
for row in ccursor:
sid = row['station'].upper()
if sid not in nt.sts:
continue
labels.append(sid[2:])
lats.append(nt.sts[sid]['lat'])
lons.append(nt.sts[sid]['lon'])
vals.append(row['avg_low'])
# ---------- Plot the points
m = MapPlot(title="Average Daily Low Temperature [F] (%s)" % (year, ),
subtitle='1 January - %s' % (maxday.strftime("%d %B"), ),
axisbg='white')
m.plot_values(lons,
lats,
vals,
labels=labels,
labeltextsize=8,
labelcolor='tan',
fmt='%.1f')
pqstr = "plot m %s bogus %s/summary/avg_low.png png" % (
now.strftime("%Y%m%d%H%M"),
year,
)
m.postprocess(pqstr=pqstr)
m.close()
# Plot Average Highs
lats = []
lons = []
vals = []
labels = []
ccursor.execute(sql)
for row in ccursor:
sid = row['station'].upper()
if sid not in nt.sts:
continue
labels.append(sid[2:])
lats.append(nt.sts[sid]['lat'])
lons.append(nt.sts[sid]['lon'])
vals.append(row['avg_tmp'])
# ---------- Plot the points
m = MapPlot(title="Average Daily Temperature [F] (%s)" % (year, ),
subtitle='1 January - %s' % (maxday.strftime("%d %B"), ),
axisbg='white')
m.plot_values(lons,
lats,
vals,
labels=labels,
labeltextsize=8,
labelcolor='tan',
fmt='%.1f')
pqstr = "plot m %s bogus %s/summary/avg_temp.png png" % (
now.strftime("%Y%m%d%H%M"),
year,
)
m.postprocess(pqstr=pqstr)
m.close()
def main():
"""Go Main Go"""
pgconn = get_dbconn("iem", user="******")
df = read_sql(
"""
select station, st_x(geom), st_y(geom), snow_jul1, snow_jul1_normal
from cli_data c JOIN stations t on (t.id = c.station)
WHERE c.valid = 'YESTERDAY' and t.network = 'NWSCLI'
and snow_jul1 is not null and snow_jul1_normal is not null
and t.id not in ('RAP', 'DVN', 'FGF', 'OAX', 'MPX')
""",
pgconn,
index_col="station",
)
df["departure"] = df["snow_jul1"] - df["snow_jul1_normal"]
df["colors"] = df["departure"].apply(lambda x: "#ff0000"
if x < 0 else "#0000ff")
yesterday = datetime.datetime.today() - datetime.timedelta(days=1)
year = yesterday.year if yesterday.month > 6 else yesterday.year - 1
mp = MapPlot(
sector="midwest",
axisbg="white",
title="NWS Total Snowfall (inches) thru %s" %
(yesterday.strftime("%-d %B %Y"), ),
subtitle=("1 July %s - %s") %
(year, datetime.datetime.today().strftime("%-d %B %Y")),
)
mp.plot_values(
df["st_x"].values,
df["st_y"].values,
df["snow_jul1"].values,
fmt="%.1f",
labelbuffer=5,
)
pqstr = ("data ac %s0000 summary/mw_season_snowfall.png "
"mw_season_snowfall.png png") % (
datetime.datetime.today().strftime("%Y%m%d"), )
mp.postprocess(view=False, pqstr=pqstr)
mp.close()
# Depature
mp = MapPlot(
sector="midwest",
axisbg="white",
title="NWS Total Snowfall Departure (inches) thru %s" %
(yesterday.strftime("%-d %B %Y"), ),
subtitle=("1 July %s - %s") %
(year, datetime.datetime.today().strftime("%-d %B %Y")),
)
mp.plot_values(
df["st_x"].values,
df["st_y"].values,
df["departure"].values,
color=df["colors"].values,
fmt="%.1f",
labelbuffer=5,
)
pqstr = ("data ac %s0000 summary/mw_season_snowfall_departure.png "
"mw_season_snowfall_departure.png png") % (
datetime.datetime.today().strftime("%Y%m%d"), )
mp.postprocess(view=False, pqstr=pqstr)
mp.close()
def runYear(year):
"""Hack"""
now = datetime.datetime.now()
nt = NetworkTable("IACLIMATE")
nt.sts["IA0200"]["lon"] = -93.4
nt.sts["IA5992"]["lat"] = 41.65
pgconn = get_dbconn("coop", user="******")
ccursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
sql = """SELECT station, avg(high) as avg_high, avg(low) as avg_low,
avg( (high+low)/2 ) as avg_tmp, max(day)
from alldata_ia WHERE year = %s and station != 'IA0000' and
high is not Null and low is not Null and substr(station,3,1) != 'C'
GROUP by station""" % (
year,
)
ccursor.execute(sql)
# Plot Average Highs
lats = []
lons = []
vals = []
labels = []
for row in ccursor:
sid = row["station"].upper()
if sid not in nt.sts:
continue
labels.append(sid[2:])
lats.append(nt.sts[sid]["lat"])
lons.append(nt.sts[sid]["lon"])
vals.append(row["avg_high"])
maxday = row["max"]
# ---------- Plot the points
mp = MapPlot(
title="Average Daily High Temperature [F] (%s)" % (year,),
subtitle="1 January - %s" % (maxday.strftime("%d %B"),),
axisbg="white",
)
mp.plot_values(
lons,
lats,
vals,
labels=labels,
labeltextsize=8,
labelcolor="tan",
fmt="%.1f",
)
pqstr = "plot m %s bogus %s/summary/avg_high.png png" % (
now.strftime("%Y%m%d%H%M"),
year,
)
mp.postprocess(pqstr=pqstr)
mp.close()
# Plot Average Lows
lats = []
lons = []
vals = []
labels = []
ccursor.execute(sql)
for row in ccursor:
sid = row["station"].upper()
if sid not in nt.sts:
continue
labels.append(sid[2:])
lats.append(nt.sts[sid]["lat"])
lons.append(nt.sts[sid]["lon"])
vals.append(row["avg_low"])
# ---------- Plot the points
mp = MapPlot(
title="Average Daily Low Temperature [F] (%s)" % (year,),
subtitle="1 January - %s" % (maxday.strftime("%d %B"),),
axisbg="white",
)
mp.plot_values(
lons,
lats,
vals,
labels=labels,
labeltextsize=8,
labelcolor="tan",
fmt="%.1f",
)
pqstr = "plot m %s bogus %s/summary/avg_low.png png" % (
now.strftime("%Y%m%d%H%M"),
year,
)
mp.postprocess(pqstr=pqstr)
mp.close()
# Plot Average Highs
lats = []
lons = []
vals = []
labels = []
ccursor.execute(sql)
for row in ccursor:
sid = row["station"].upper()
if sid not in nt.sts:
continue
labels.append(sid[2:])
lats.append(nt.sts[sid]["lat"])
lons.append(nt.sts[sid]["lon"])
vals.append(row["avg_tmp"])
# ---------- Plot the points
mp = MapPlot(
title="Average Daily Temperature [F] (%s)" % (year,),
subtitle="1 January - %s" % (maxday.strftime("%d %B"),),
axisbg="white",
)
mp.plot_values(
lons,
lats,
vals,
labels=labels,
labeltextsize=8,
labelcolor="tan",
fmt="%.1f",
)
pqstr = "plot m %s bogus %s/summary/avg_temp.png png" % (
now.strftime("%Y%m%d%H%M"),
year,
)
mp.postprocess(pqstr=pqstr)
mp.close()
def main(argv):
"""Go Main Go"""
nt = Table("ISUSM")
qdict = loadqc()
idbconn = get_dbconn('isuag', user='******')
icursor = idbconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
pdbconn = get_dbconn('postgis', user='******')
pcursor = pdbconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
day_ago = int(argv[1])
ts = datetime.datetime.now() - datetime.timedelta(days=day_ago)
# Query out the data
soil_obs = []
lats = []
lons = []
icursor.execute("""
SELECT station, tsoil_c_avg_qc from sm_daily
where valid = '%s' and tsoil_c_avg_qc > -40
and station not in ('AHTI4', 'FRUI4')
""" % (ts.strftime("%Y-%m-%d"), ))
for row in icursor:
stid = row['station']
if qdict.get(stid, {}).get('soil4', False):
# print '%s was QCd out' % (stid,)
continue
soil_obs.append(temperature(row['tsoil_c_avg_qc'], 'C').value('F'))
lats.append(nt.sts[stid]['lat'])
lons.append(nt.sts[stid]['lon'])
if len(lats) < 5:
print(("isuag/fancy_4inch found %s obs for %s") %
(len(lats), ts.strftime("%Y-%m-%d")))
return
# Grid it
# numxout = 40
# numyout = 40
# xmin = min(lons) - 2.
# ymin = min(lats) - 2.
# xmax = max(lons) + 2.
# ymax = max(lats) + 2.
# xc = (xmax-xmin)/(numxout-1)
# yc = (ymax-ymin)/(numyout-1)
# xo = xmin + xc * np.arange(0, numxout)
# yo = ymin + yc * np.arange(0, numyout)
# analysis = griddata((lons, lats), soil_obs, (xo, yo) )
# rbfi = Rbf(lons, lats, soil_obs, function='cubic')
# analysis = rbfi(xo, yo)
nn = NearestNDInterpolator((lons, lats), np.array(soil_obs))
# analysis = nn(xo, yo)
# Query out centroids of counties...
pcursor.execute("""SELECT ST_x(ST_centroid(the_geom)) as lon,
ST_y(ST_centroid(the_geom)) as lat
from uscounties WHERE state_name = 'Iowa'
""")
clons = []
clats = []
for row in pcursor:
clats.append(row['lat'])
clons.append(row['lon'])
cobs = nn(clons, clats)
mp = MapPlot(sector='iowa',
title=("Iowa Average 4 inch Soil Temperatures %s") %
(ts.strftime("%b %d %Y"), ),
subtitle=("Based on gridded analysis (black numbers) of "
"ISUSM network observations (red numbers)"))
mp.contourf(clons,
clats,
cobs,
np.arange(10, 101, 5),
cmap=cm.get_cmap('jet'),
units=r'$^\circ$F')
mp.plot_values(lons, lats, soil_obs, fmt='%.0f', color='r', labelbuffer=5)
mp.plot_values(clons, clats, cobs, fmt='%.0f', textsize=11, labelbuffer=5)
# for lo, la, ob in zip(clons, clats, cobs):
# xi, yi = m.map(lo, la)
# txt = m.ax.text(xi, yi, "%.0f" % (ob,))
mp.drawcounties()
routes = "a" if day_ago >= 4 else "ac"
pqstr = ("plot %s %s0000 soilt_day%s.png isuag_county_4inch_soil.png png"
) % (routes, ts.strftime("%Y%m%d"), day_ago)
mp.postprocess(pqstr=pqstr)
mp.close()
def doit(ts, hours):
"""
Create a plot of precipitation stage4 estimates for some day
"""
# Start at 1 AM
ts = ts.replace(minute=0, second=0, microsecond=0)
now = ts - datetime.timedelta(hours=hours-1)
interval = datetime.timedelta(hours=1)
ets = datetime.datetime.utcnow()
ets = ets.replace(tzinfo=pytz.timezone("UTC"))
total = None
while now < ets:
gmt = now.astimezone(pytz.timezone("UTC"))
gribfn = None
for prefix in ['GaugeCorr', 'RadarOnly']:
fn = gmt.strftime((prefix + "_QPE_01H_00.00_%Y%m%d-%H%M00"
".grib2.gz"))
res = requests.get(gmt.strftime(
("http://mtarchive.geol.iastate.edu/%Y/%m/%d/mrms/ncep/" +
prefix + "_QPE_01H/" + fn)), timeout=30)
if res.status_code != 200:
continue
o = open(TMP + "/" + fn, 'wb')
o.write(res.content)
o.close()
gribfn = "%s/%s" % (TMP, fn)
break
if gribfn is None:
print("q3_Xhour.py[%s] MISSING %s" % (hours, now))
now += interval
continue
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb['values']
else:
maxgrid = np.maximum(grb['values'], total)
total = np.where(np.logical_and(grb['values'] >= 0,
total >= 0),
grb['values'] + total, maxgrid)
now += interval
os.unlink(gribfn)
if total is None:
print("q3_Xhour.py no data ts: %s hours: %s" % (ts, hours))
return
# Scale factor is 10
routes = "c"
if ts.minute == 0:
routes = "ac"
pqstr = "plot %s %s iowa_q2_%sh.png q2/iowa_q2_%sh_%s00.png png" % (
routes, ts.strftime("%Y%m%d%H%M"), hours, hours,
ts.strftime("%H"))
lts = ts.astimezone(pytz.timezone("America/Chicago"))
subtitle = 'Total up to %s' % (lts.strftime("%d %B %Y %I:%M %p %Z"),)
m = MapPlot(title=("NCEP MRMS Q3 (RADAR Only) %s Hour "
"Precipitation [inch]") % (hours,),
subtitle=subtitle)
clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
m.contourf(mrms.XAXIS, mrms.YAXIS,
distance(np.flipud(total), 'MM').value('IN'), clevs,
cmap=nwsprecip())
m.drawcounties()
m.postprocess(pqstr=pqstr, view=False)
m.close()
def doit(now, model):
""" Figure out the model runtime we care about """
mcursor.execute(
"""
SELECT max(runtime at time zone 'UTC') from alldata where station = 'KDSM'
and ftime = %s and model = %s
""", (now, model))
row = mcursor.fetchone()
runtime = row[0]
if runtime is None:
sys.exit()
runtime = runtime.replace(tzinfo=pytz.timezone("UTC"))
# Load up the mos forecast for our given
mcursor.execute(
"""
SELECT station, tmp FROM alldata
WHERE model = %s and runtime = %s and ftime = %s and tmp < 999
""", (model, runtime, now))
forecast = {}
for row in mcursor:
if row[0][0] == 'K':
forecast[row[0][1:]] = row[1]
# Load up the currents!
icursor.execute("""
SELECT
s.id, s.network, tmpf, ST_x(s.geom) as lon, ST_y(s.geom) as lat
FROM
current c, stations s
WHERE
c.iemid = s.iemid and
(s.network ~* 'ASOS' or s.network = 'AWOS') and s.country = 'US' and
valid + '60 minutes'::interval > now() and
tmpf > -50
""")
lats = []
lons = []
vals = []
for row in icursor:
if row[0] not in forecast:
continue
diff = forecast[row[0]] - row[2]
if diff > 20 or diff < -20:
continue
lats.append(row[4])
lons.append(row[3])
vals.append(diff)
cmap = cm.get_cmap("RdYlBu_r")
cmap.set_under('black')
cmap.set_over('black')
localnow = now.astimezone(pytz.timezone("America/Chicago"))
m = MapPlot(sector='midwest',
title="%s MOS Temperature Bias " % (model, ),
subtitle=('Model Run: %s Forecast Time: %s') %
(runtime.strftime("%d %b %Y %H %Z"),
localnow.strftime("%d %b %Y %-I %p %Z")))
m.contourf(lons, lats, vals, range(-10, 11, 2), units='F', cmap=cmap)
pqstr = "plot ac %s00 %s_mos_T_bias.png %s_mos_T_bias_%s.png png" % (
now.strftime("%Y%m%d%H"), model.lower(), model.lower(),
now.strftime("%H"))
m.postprocess(pqstr=pqstr, view=False)
m.close()
m = MapPlot(sector='conus',
title="%s MOS Temperature Bias " % (model, ),
subtitle=('Model Run: %s Forecast Time: %s') %
(runtime.strftime("%d %b %Y %H %Z"),
localnow.strftime("%d %b %Y %-I %p %Z")))
m.contourf(lons, lats, vals, range(-10, 11, 2), units='F', cmap=cmap)
pqstr = ("plot ac %s00 conus_%s_mos_T_bias.png "
"conus_%s_mos_T_bias_%s.png png") % (now.strftime("%Y%m%d%H"),
model.lower(), model.lower(),
now.strftime("%H"))
m.postprocess(pqstr=pqstr, view=False)
def main():
"""Go Main Go"""
COOP = get_dbconn('coop', user='******')
ccursor = COOP.cursor()
form = cgi.FieldStorage()
if ("year1" in form and "year2" in form and "month1" in form
and "month2" in form and "day1" in form and "day2" in form):
sts = datetime.datetime(int(form["year1"].value),
int(form["month1"].value),
int(form["day1"].value))
ets = datetime.datetime(int(form["year2"].value),
int(form["month2"].value),
int(form["day2"].value))
else:
sts = datetime.datetime(2011, 5, 1)
ets = datetime.datetime(2011, 10, 1)
baseV = 50
if "base" in form:
baseV = int(form["base"].value)
maxV = 86
if "max" in form:
maxV = int(form["max"].value)
# Make sure we aren't in the future
now = datetime.datetime.today()
if ets > now:
ets = now
st = NetworkTable("IACLIMATE")
# Now we load climatology
#sts = {}
#rs = mesosite.query("SELECT id, x(geom) as lon, y(geom) as lat from stations WHERE \
# network = 'IACLIMATE'").dictresult()
#for i in range(len(rs)):
# sts[ rs[i]["id"].lower() ] = rs[i]
# Compute normal from the climate database
sql = """SELECT station,
sum(gddXX(%s, %s, high, low)) as gdd, count(*)
from alldata_ia WHERE year = %s and day >= '%s' and day < '%s'
and substr(station, 2, 1) != 'C' and station != 'IA0000'
GROUP by station""" % (baseV, maxV, sts.year, sts.strftime("%Y-%m-%d"),
ets.strftime("%Y-%m-%d"))
lats = []
lons = []
gdd50 = []
valmask = []
ccursor.execute(sql)
total_days = (ets - sts).days
for row in ccursor:
id = row[0]
if not st.sts.has_key(id):
continue
if row[2] < (total_days * 0.9):
continue
lats.append(st.sts[id]['lat'])
lons.append(st.sts[id]['lon'])
gdd50.append(float(row[1]))
valmask.append(True)
m = MapPlot(
title=("Iowa %s thru %s GDD(base=%s,max=%s) Accumulation"
"") %
(sts.strftime("%Y: %d %b"),
(ets - datetime.timedelta(days=1)).strftime("%d %b"), baseV, maxV),
axisbg='white')
m.contourf(lons, lats, gdd50, range(int(min(gdd50)), int(max(gdd50)), 25))
m.plot_values(lons, lats, gdd50, fmt='%.0f')
m.drawcounties()
m.postprocess(web=True)
m.close()
def main():
"""Go Main Go"""
pgconn = get_dbconn("coop")
ccursor = pgconn.cursor()
form = cgi.FieldStorage()
if (
"year1" in form
and "year2" in form
and "month1" in form
and "month2" in form
and "day1" in form
and "day2" in form
):
sts = datetime.datetime(
int(form["year1"].value),
int(form["month1"].value),
int(form["day1"].value),
)
ets = datetime.datetime(
int(form["year2"].value),
int(form["month2"].value),
int(form["day2"].value),
)
else:
sts = datetime.datetime(2011, 5, 1)
ets = datetime.datetime(2011, 10, 1)
baseV = 50
if "base" in form:
baseV = int(form["base"].value)
maxV = 86
if "max" in form:
maxV = int(form["max"].value)
# Make sure we aren't in the future
now = datetime.datetime.today()
if ets > now:
ets = now
st = NetworkTable("IACLIMATE")
# Compute normal from the climate database
sql = """
SELECT station,
sum(gddXX(%s, %s, high, low)) as gdd, count(*)
from alldata_ia WHERE year = %s and day >= '%s' and day < '%s'
and substr(station, 2, 1) != 'C' and station != 'IA0000'
GROUP by station
""" % (
baseV,
maxV,
sts.year,
sts.strftime("%Y-%m-%d"),
ets.strftime("%Y-%m-%d"),
)
lats = []
lons = []
gdd50 = []
valmask = []
ccursor.execute(sql)
total_days = (ets - sts).days
for row in ccursor:
sid = row[0]
if sid not in st.sts:
continue
if row[2] < (total_days * 0.9):
continue
lats.append(st.sts[sid]["lat"])
lons.append(st.sts[sid]["lon"])
gdd50.append(float(row[1]))
valmask.append(True)
m = MapPlot(
title=("Iowa %s thru %s GDD(base=%s,max=%s) Accumulation" "")
% (
sts.strftime("%Y: %d %b"),
(ets - datetime.timedelta(days=1)).strftime("%d %b"),
baseV,
maxV,
),
axisbg="white",
)
m.contourf(lons, lats, gdd50, range(int(min(gdd50)), int(max(gdd50)), 25))
m.plot_values(lons, lats, gdd50, fmt="%.0f")
m.drawcounties()
m.postprocess(web=True)
m.close()
def do(valid):
"""Do Something"""
clevs = np.arange(0, 0.25, 0.05)
clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs[0] = 0.01
precip = load_precip(valid)
stage4 = load_precip(valid, 'stage4')
inqcprecip = load_precip(valid, 'inqcprecip')
outqcprecip = load_precip(valid, 'outqcprecip')
multiplier = load_precip(valid, 'multiplier')
yidx = int((43.27 - SOUTH) / 0.01)
xidx = int((-94.39 - WEST) / 0.01)
print(("yidx:%s xidx:%s precip:%.2f stage4: %.2f inqc: %.2f outqc: %.2f "
"mul: %.2f"
) % (yidx, xidx, precip[yidx, xidx], stage4[yidx, xidx],
inqcprecip[yidx, xidx], outqcprecip[yidx, xidx],
multiplier[yidx, xidx]))
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s DEP Quality Controlled Precip Totals'
) % (valid.strftime("%-d %b %Y"), ))
(lons, lats) = np.meshgrid(XS, YS)
mp.pcolormesh(lons, lats, precip / 25.4, clevs,
units='inch')
mp.drawcounties()
mp.postprocess(filename='qc.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s Stage IV Precip Totals'
) % (valid.strftime("%-d %b %Y"), ))
mp.pcolormesh(lons, lats, stage4 / 25.4, clevs,
units='inch')
mp.drawcounties()
mp.postprocess(filename='stageIV.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s High-res total prior to QC'
) % (valid.strftime("%-d %b %Y"), ))
mp.pcolormesh(lons, lats, inqcprecip / 25.4, clevs,
units='inch')
mp.drawcounties()
mp.postprocess(filename='inqcprecip.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s High-res total after QC'
) % (valid.strftime("%-d %b %Y"), ))
mp.pcolormesh(lons, lats, outqcprecip / 25.4, clevs,
units='inch')
mp.drawcounties()
mp.postprocess(filename='outqcprecip.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s QC Change in Precip Out - In'
) % (valid.strftime("%-d %b %Y"), ))
diff = (outqcprecip - inqcprecip) / 25.4
mp.pcolormesh(lons, lats, diff,
np.arange(-1.4, 1.5, 0.1),
cmap=plt.get_cmap('BrBG'),
units='inch')
mp.drawcounties()
mp.postprocess(filename='qcprecipdiff.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s multiplier'
) % (valid.strftime("%-d %b %Y"), ))
mp.pcolormesh(lons, lats, multiplier,
np.arange(0.0, 2.5, 0.2),
cmap=plt.get_cmap('jet'),
units='ratio')
mp.drawcounties()
mp.postprocess(filename='multiplier.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s manually computed QC Precip mul * stage4'
) % (valid.strftime("%-d %b %Y"), ))
mp.pcolormesh(lons, lats, multiplier * stage4 / 25.4,
clevs,
units='inch')
mp.drawcounties()
mp.postprocess(filename='qcmancalc.png')
mp.close()
# Go MRMS
ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (valid.year, )
if not os.path.isfile(ncfn):
return
nc = netCDF4.Dataset(ncfn, 'r')
xidx = int((WEST - nc.variables['lon'][0]) * 100.)
yidx = int((SOUTH - nc.variables['lat'][0]) * 100.)
idx = daily_offset(valid)
mrms = nc.variables['p01d'][idx, yidx:(yidx+800), xidx:(xidx+921)]
nc.close()
def doit(ts, hours):
"""
Create a plot of precipitation stage4 estimates for some day
"""
# Start at 1 AM
ts = ts.replace(minute=0, second=0, microsecond=0)
now = ts - datetime.timedelta(hours=hours-1)
interval = datetime.timedelta(hours=1)
ets = datetime.datetime.utcnow()
ets = ets.replace(tzinfo=pytz.utc)
total = None
while now < ets:
gmt = now.astimezone(pytz.utc)
gribfn = None
for prefix in ['GaugeCorr', 'RadarOnly']:
gribfn = mrms.fetch(prefix+"_QPE_01H", gmt)
if gribfn is None:
continue
break
if gribfn is None:
print("q3_xhour.py[%s] MISSING %s" % (hours, now))
now += interval
continue
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb['values']
else:
maxgrid = np.maximum(grb['values'], total)
total = np.where(np.logical_and(grb['values'] >= 0,
total >= 0),
grb['values'] + total, maxgrid)
now += interval
os.unlink(gribfn)
if total is None:
print("q3_xhour.py no data ts: %s hours: %s" % (ts, hours))
return
# Scale factor is 10
routes = "c"
if ts.minute == 0:
routes = "ac"
pqstr = "plot %s %s iowa_q2_%sh.png q2/iowa_q2_%sh_%s00.png png" % (
routes, ts.strftime("%Y%m%d%H%M"), hours, hours,
ts.strftime("%H"))
lts = ts.astimezone(pytz.timezone("America/Chicago"))
subtitle = 'Total up to %s' % (lts.strftime("%d %B %Y %I:%M %p %Z"),)
mp = MapPlot(title=("NCEP MRMS Q3 (RADAR Only) %s Hour "
"Precipitation [inch]") % (hours,),
subtitle=subtitle)
clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
mp.contourf(mrms.XAXIS, mrms.YAXIS,
distance(np.flipud(total), 'MM').value('IN'), clevs,
cmap=nwsprecip())
mp.drawcounties()
mp.postprocess(pqstr=pqstr, view=False)
mp.close()
def makeplot(ts, routes='ac'):
"""
Generate two plots for a given time GMT
"""
sql = """
SELECT ST_x(geom), ST_y(geom),
CASE WHEN sm is Null THEN -1 ELSE sm END,
CASE WHEN od is Null THEN -1 ELSE od END from
(SELECT grid_idx, avg(soil_moisture) as sm,
avg(optical_depth) as od
from data WHERE valid > '%s+00'::timestamptz - '6 hours'::interval
and valid < '%s+00'::timestamptz + '6 hours'::interval
GROUP by grid_idx) as foo, grid
WHERE foo.grid_idx = grid.idx
""" % (ts.strftime('%Y-%m-%d %H:%M'),
ts.strftime('%Y-%m-%d %H:%M'))
#sql = """
#SELECT x(geom), y(geom), random(), random() from grid
#"""
scursor.execute( sql )
lats = []
lons = []
sm = []
od = []
for row in scursor:
lats.append( float(row[1]) )
lons.append( float(row[0]) )
sm.append( row[2] * 100.)
od.append( row[3] )
if len(lats) == 0:
#print 'Did not find SMOS data for ts: %s' % (ts,)
return
lats = np.array( lats )
lons = np.array( lons )
sm = np.array( sm )
od = np.array( od )
for sector in ['midwest', 'iowa']:
clevs = np.arange(0,71,5)
m = MapPlot(sector=sector,
title = 'SMOS Satellite: Soil Moisture (0-5cm)',
subtitle="Satelite passes around %s UTC" % (
ts.strftime("%d %B %Y %H"),))
if sector == 'iowa':
m.drawcounties()
cmap = cm.get_cmap('jet_r')
cmap.set_under('#EEEEEE')
cmap.set_over("k")
m.hexbin(lons,lats,sm,clevs,units='%', cmap=cmap)
pqstr = "plot %s %s00 smos_%s_sm%s.png smos_%s_sm%s.png png" % (
routes, ts.strftime("%Y%m%d%H"), sector, ts.strftime("%H"),
sector, ts.strftime("%H"))
m.postprocess(pqstr=pqstr)
m.close()
for sector in ['midwest', 'iowa']:
clevs = np.arange(0,1.001,0.05)
m = MapPlot(sector=sector,
title = 'SMOS Satellite: Land Cover Optical Depth (microwave L-band)',
subtitle="Satelite passes around %s UTC" % (
ts.strftime("%d %B %Y %H"),))
if sector == 'iowa':
m.drawcounties()
cmap = cm.get_cmap('jet')
cmap.set_under('#EEEEEE')
cmap.set_over("k")
m.hexbin(lons,lats,od,clevs, cmap=cmap)
pqstr = "plot %s %s00 smos_%s_od%s.png smos_%s_od%s.png png" % (
routes, ts.strftime("%Y%m%d%H"), sector, ts.strftime("%H"),
sector, ts.strftime("%H"))
m.postprocess(pqstr=pqstr)
m.close()
def run(basets, endts, view):
"""Generate this plot for the given basets"""
vals = []
valmask = []
lats = []
lons = []
pcursor.execute("""SELECT state,
max(magnitude) as val, ST_x(geom) as lon, ST_y(geom) as lat
from lsrs WHERE type in ('S') and magnitude >= 0 and
valid > %s and valid < %s GROUP by state, lon, lat""", (basets, endts))
for row in pcursor:
vals.append(row[1])
lats.append(row[3])
lons.append(row[2])
valmask.append(row[0] in ['IA', ])
# Now, we need to add in zeros, lets say we are looking at a .25 degree box
mybuffer = 1.0
for lat in np.arange(reference.MW_SOUTH, reference.MW_NORTH, mybuffer):
for lon in np.arange(reference.MW_WEST, reference.MW_EAST, mybuffer):
found = False
for j in range(len(lats)):
if (lats[j] > (lat-(mybuffer/2.)) and
lats[j] < (lat+(mybuffer/2.)) and
lons[j] > (lon-(mybuffer/2.)) and
lons[j] < (lon+(mybuffer/2.))):
found = True
if not found:
lats.append(lat)
lons.append(lon)
valmask.append(False)
vals.append(0)
rng = [0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 7, 9, 11, 13, 15, 17]
m = MapPlot(sector='iowa', axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
m.contourf(lons, lats, vals, rng)
m.drawcounties()
m.plot_values(lons, lats, vals, fmt='%.1f', valmask=valmask)
pqstr = "plot c 000000000000 lsr_snowfall.png bogus png"
m.postprocess(view=view, pqstr=pqstr)
m.close()
# slightly different title to help uniqueness
m = MapPlot(sector='iowa', axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports valid over past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
m.contourf(lons, lats, vals, rng)
m.drawcounties()
pqstr = "plot c 000000000000 lsr_snowfall_nv.png bogus png"
m.postprocess(view=view, pqstr=pqstr)
m.close()
m = MapPlot(sector='midwest', axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
m.contourf(lons, lats, vals, rng)
pqstr = "plot c 000000000000 mw_lsr_snowfall.png bogus png"
m.postprocess(view=view, pqstr=pqstr)
m.close()
def two(year):
"""Compare yearly totals in a scatter plot"""
coop = psycopg2.connect(database='coop',
host='localhost',
port=5555,
user='******')
ccursor = coop.cursor()
idep = psycopg2.connect(database='idep',
host='localhost',
port=5555,
user='******')
icursor = idep.cursor()
ccursor.execute(
"""
SELECT station, sum(precip) from alldata_ia
WHERE year = %s and station != 'IA0000'
and substr(station, 3, 1) != 'C' GROUP by station ORDER by station ASC
""", (year, ))
nt = NetworkTable("IACLIMATE")
rows = []
for row in ccursor:
station = row[0]
precip = row[1]
if station not in nt.sts:
continue
lon = nt.sts[station]['lon']
lat = nt.sts[station]['lat']
icursor.execute(
"""
select huc_12 from huc12
where ST_Contains(geom,
ST_Transform(ST_SetSRID(ST_Point(%s, %s), 4326), 5070))
and scenario = 0
""", (lon, lat))
if icursor.rowcount == 0:
continue
huc12 = icursor.fetchone()[0]
icursor.execute(
"""
select sum(qc_precip) from results_by_huc12
WHERE valid between %s and %s and huc_12 = %s and scenario = 0
""", (datetime.date(year, 1, 1), datetime.date(year, 12, 31), huc12))
val = icursor.fetchone()[0]
if val is None:
continue
iprecip = distance(val, 'MM').value('IN')
rows.append(
dict(station=station,
precip=precip,
iprecip=iprecip,
lat=lat,
lon=lon))
# print("%s %s %5.2f %5.2f" % (station, huc12, precip, iprecip))
df = pd.DataFrame(rows)
df['diff'] = df['iprecip'] - df['precip']
bias = df['diff'].mean()
print("%s %5.2f %5.2f %5.2f" %
(year, df['iprecip'].mean(), df['precip'].mean(), bias))
m = MapPlot(title=("%s IDEP Precipitation minus IEM Climodat (inch)") %
(year, ),
subtitle=("HUC12 Average minus point observation, "
"Overall bias: %.2f") % (bias, ),
axisbg='white')
m.plot_values(df['lon'], df['lat'], df['diff'], fmt='%.2f', labelbuffer=1)
m.postprocess(filename='%s_map.png' % (year, ))
m.close()
(fig, ax) = plt.subplots(1, 1)
ax.scatter(df['precip'], df['iprecip'])
ax.grid(True)
ylim = ax.get_ylim()
ax.plot([ylim[0], ylim[1]], [ylim[0], ylim[1]], lw=2)
ax.set_xlabel("IEM Climodat Precip")
ax.set_ylabel("IDEP HUC12 Precip")
ax.set_title("%s Precipitation Comparison, bias=%.2f" % (year, bias))
fig.savefig('%s_xy.png' % (year, ))
plt.close()
def makeplot(ts, routes='ac'):
"""
Generate two plots for a given time GMT
"""
sql = """
SELECT ST_x(geom), ST_y(geom),
CASE WHEN sm is Null THEN -1 ELSE sm END,
CASE WHEN od is Null THEN -1 ELSE od END from
(SELECT grid_idx, avg(soil_moisture) as sm,
avg(optical_depth) as od
from data WHERE valid > '%s+00'::timestamptz - '6 hours'::interval
and valid < '%s+00'::timestamptz + '6 hours'::interval
GROUP by grid_idx) as foo, grid
WHERE foo.grid_idx = grid.idx
""" % (ts.strftime('%Y-%m-%d %H:%M'), ts.strftime('%Y-%m-%d %H:%M'))
#sql = """
#SELECT x(geom), y(geom), random(), random() from grid
#"""
scursor.execute(sql)
lats = []
lons = []
sm = []
od = []
for row in scursor:
lats.append(float(row[1]))
lons.append(float(row[0]))
sm.append(row[2] * 100.)
od.append(row[3])
if len(lats) == 0:
# print 'Did not find SMOS data for ts: %s' % (ts,)
return
lats = np.array(lats)
lons = np.array(lons)
sm = np.array(sm)
od = np.array(od)
for sector in ['midwest', 'iowa']:
clevs = np.arange(0, 71, 5)
m = MapPlot(sector=sector,
axisbg='white',
title='SMOS Satellite: Soil Moisture (0-5cm)',
subtitle="Satelite passes around %s UTC" %
(ts.strftime("%d %B %Y %H"), ))
if sector == 'iowa':
m.drawcounties()
cmap = cm.get_cmap('jet_r')
cmap.set_under('#EEEEEE')
cmap.set_over("k")
m.hexbin(lons, lats, sm, clevs, units='%', cmap=cmap)
pqstr = "plot %s %s00 smos_%s_sm%s.png smos_%s_sm%s.png png" % (
routes, ts.strftime("%Y%m%d%H"), sector, ts.strftime("%H"), sector,
ts.strftime("%H"))
m.postprocess(pqstr=pqstr)
m.close()
for sector in ['midwest', 'iowa']:
clevs = np.arange(0, 1.001, 0.05)
m = MapPlot(sector=sector,
axisbg='white',
title=('SMOS Satellite: Land Cover Optical Depth '
'(microwave L-band)'),
subtitle="Satelite passes around %s UTC" %
(ts.strftime("%d %B %Y %H"), ))
if sector == 'iowa':
m.drawcounties()
cmap = cm.get_cmap('jet')
cmap.set_under('#EEEEEE')
cmap.set_over("k")
m.hexbin(lons, lats, od, clevs, cmap=cmap)
pqstr = "plot %s %s00 smos_%s_od%s.png smos_%s_od%s.png png" % (
routes, ts.strftime("%Y%m%d%H"), sector, ts.strftime("%H"), sector,
ts.strftime("%H"))
m.postprocess(pqstr=pqstr)
m.close()
