def plotter(fdict):
""" Go """
import matplotlib
matplotlib.use('agg')
from pyiem.plot import MapPlot
import matplotlib.cm as cm
pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
cursor = pgconn.cursor()
sector = fdict.get('sector', 'IA')
date1 = datetime.datetime.strptime(fdict.get('date1', '2015-01-01'),
'%Y-%m-%d')
date2 = datetime.datetime.strptime(fdict.get('date2', '2015-02-01'),
'%Y-%m-%d')
table = "alldata_%s" % (sector, ) if sector != 'midwest' else "alldata"
cursor.execute("""
WITH obs as (
SELECT station, sday, day, precip from """ + table + """ WHERE
day >= %s and day < %s and precip >= 0 and
substr(station, 3, 1) != 'C' and substr(station, 3, 4) != '0000'),
climo as (
SELECT station, to_char(valid, 'mmdd') as sday, precip from
climate51),
combo as (
SELECT o.station, o.precip - c.precip as d from obs o JOIN climo c ON
(o.station = c.station and o.sday = c.sday)),
deltas as (
SELECT station, sum(d) from combo GROUP by station)
SELECT d.station, d.sum, ST_x(t.geom), ST_y(t.geom) from deltas d
JOIN stations t on (d.station = t.id) WHERE t.network ~* 'CLIMATE'
""", (date1, date2))
rows = []
for row in cursor:
rows.append(dict(station=row[0], delta=row[1], lon=row[2],
lat=row[3]))
df = pd.DataFrame(rows)
lons = np.array(df['lon'])
vals = np.array(df['delta'])
lats = np.array(df['lat'])
sector2 = "state" if sector != 'midwest' else 'midwest'
m = MapPlot(sector=sector2, state=sector, axisbg='white',
title=('%s - %s Precipitation Departure [inch]'
) % (date1.strftime("%d %b %Y"),
date2.strftime("%d %b %Y")),
subtitle='%s vs 1950-2014 Climatology' % (date1.year,))
rng = int(max([0 - np.min(vals), np.max(vals)]))
cmap = cm.get_cmap('RdYlBu')
cmap.set_bad('white')
m.contourf(lons, lats, vals, np.linspace(0 - rng - 0.5, rng + 0.6, 10,
dtype='i'),
cmap=cmap, units='inch')
m.plot_values(lons, lats, vals, fmt='%.2f')
if sector == 'iowa':
m.drawcounties()
return m.fig, df
def plot(data, v):
''' Actually plot this data '''
nt = NetworkTable("ISUSM")
lats = []
lons = []
vals = []
valid = None
for sid in data.keys():
if data[sid][v] is None:
continue
lats.append(nt.sts[sid]['lat'])
lons.append(nt.sts[sid]['lon'])
vals.append(data[sid][v])
valid = data[sid]['valid']
if valid is None:
m = MapPlot(sector='iowa', axisbg='white',
title=('ISU Soil Moisture Network :: %s'
'') % (CTX[v]['title'], ),
figsize=(8.0, 6.4))
m.plot_values([-95, ], [41.99, ], ['No Data Found'], '%s', textsize=30)
m.postprocess(web=True)
return
m = MapPlot(sector='iowa', axisbg='white',
title='ISU Soil Moisture Network :: %s' % (CTX[v]['title'],),
subtitle='valid %s' % (valid.strftime("%-d %B %Y %I:%M %p"),),
figsize=(8.0, 6.4))
m.plot_values(lons, lats, vals, '%.1f')
m.drawcounties()
m.postprocess(web=True)
def main():
"""Go Main"""
pgconn = get_dbconn('postgis')
df = read_postgis("""
select geom, issue from sbw where wfo = 'PUB' and phenomena = 'TO'
and significance = 'W' and status = 'NEW' and issue > '2007-10-01'
and issue < '2019-01-01'
""", pgconn, geom_col='geom', crs={'init': 'epsg:4326', 'no_defs': True})
bounds = df['geom'].total_bounds
# bounds = [-102.90293903, 40.08745967, -97.75622311, 43.35172981]
bbuf = 0.25
mp = MapPlot(
sector='custom', west=bounds[0] - bbuf,
south=bounds[1] - bbuf,
east=bounds[2] + bbuf, north=bounds[3] + bbuf,
continentalcolor='white', # '#b3242c',
title='NWS Pueblo Issued Tornado Warnings [2008-2018]',
subtitle='%s warnings plotted' % (len(df.index), ))
crs_new = ccrs.Mercator()
crs = ccrs.PlateCarree()
new_geometries = [crs_new.project_geometry(ii, src_crs=crs)
for ii in df['geom'].values]
# mp.draw_cwas()
mp.ax.add_geometries(new_geometries, crs=crs_new, lw=0.5,
edgecolor='red', facecolor='None', alpha=1,
zorder=5)
mp.drawcounties()
mp.postprocess(filename='test.png')
def do_month(year, month, routes):
""" Generate a MRMS plot for the month!"""
sts = datetime.datetime(year,month,1)
ets = sts + datetime.timedelta(days=35)
ets = ets.replace(day=1)
today = datetime.datetime.now()
if ets > today:
ets = today
idx0 = iemre.daily_offset(sts)
idx1 = iemre.daily_offset(ets)
nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (year,),
'r')
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
p01d = np.sum(nc.variables['p01d'][idx0:idx1,:,:],0) / 24.5
nc.close()
m = MapPlot(sector='iowa', title='MRMS %s - %s Total Precipitation' % (
sts.strftime("%-d %b"),
(ets - datetime.timedelta(days=1)).strftime("%-d %b %Y")),
subtitle='Data from NOAA MRMS Project')
x,y = np.meshgrid(lons, lats)
bins = [0.01, 0.1, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20]
m.pcolormesh(x, y, p01d, bins, units='inches')
m.drawcounties()
currentfn = "summary/iowa_mrms_q3_month.png"
archivefn = sts.strftime("%Y/%m/summary/iowa_mrms_q3_month.png")
pqstr = "plot %s %s00 %s %s png" % (
routes, sts.strftime("%Y%m%d%H"), currentfn, archivefn)
m.postprocess(pqstr=pqstr)
def runYear(year):
# Grab the data
sql = """SELECT station,
sum(case when precip >= 0.01 then 1 else 0 end) as days, max(day)
from alldata_ia WHERE year = %s and substr(station,3,1) != 'C'
and station != 'IA0000' GROUP by station""" % (year,)
lats = []
lons = []
vals = []
labels = []
ccursor.execute( sql )
for row in ccursor:
sid = row['station'].upper()
if not nt.sts.has_key(sid):
continue
labels.append( sid[2:] )
lats.append( nt.sts[sid]['lat'] )
lons.append( nt.sts[sid]['lon'] )
vals.append( row['days'] )
maxday = row['max']
#---------- Plot the points
m = MapPlot(title="Days with Measurable Precipitation (%s)" % (year,),
subtitle='Map valid January 1 - %s' % (maxday.strftime("%b %d")),
axisbg='white')
m.plot_values(lons, lats, vals, fmt='%.0f', labels=labels,
labeltextsize=8, labelcolor='tan')
m.drawcounties()
pqstr = "plot m %s bogus %s/summary/precip_days.png png" % (
now.strftime("%Y%m%d%H%M"), year,)
m.postprocess(pqstr=pqstr)
def main():
"""Go!"""
title = 'NOAA MRMS Q3: RADAR + Guage Corrected Rainfall Estimates + NWS Storm Reports'
mp = MapPlot(sector='custom',
north=42.3, east=-93.0, south=41.65, west=-94.1,
axisbg='white',
titlefontsize=14,
title=title,
subtitle='Valid: 14 June 2018')
shp = shapefile.Reader('cities.shp')
for record in shp.shapeRecords():
geo = shape(record.shape)
mp.ax.add_geometries([geo], ccrs.PlateCarree(), zorder=Z_OVERLAY2,
facecolor='None', edgecolor='k', lw=2)
grbs = pygrib.open('MRMS_GaugeCorr_QPE_24H_00.00_20180614-200000.grib2')
grb = grbs.message(1)
pcpn = distance(grb['values'], 'MM').value('IN')
lats, lons = grb.latlons()
lons -= 360.
clevs = [0.01, 0.1, 0.3, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
cmap = nwsprecip()
cmap.set_over('k')
mp.pcolormesh(lons, lats, pcpn, clevs, cmap=cmap, latlon=True,
units='inch')
lons, lats, vals, labels = get_data()
mp.drawcounties()
mp.plot_values(lons, lats, vals, "%s", labels=labels,
labelbuffer=1, labelcolor='white')
mp.drawcities(labelbuffer=5, minarea=0.2)
mp.postprocess(filename='test.png')
def doday():
"""
Create a plot of precipitation stage4 estimates for some day
"""
sts = mx.DateTime.DateTime(2013,5,25,12)
ets = mx.DateTime.DateTime(2013,5,31,12)
interval = mx.DateTime.RelativeDateTime(days=1)
now = sts
total = None
while now < ets:
fp = "/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.24h.grib" % (
now.strftime("%Y/%m/%d"),
now.strftime("%Y%m%d%H") )
if os.path.isfile(fp):
lts = now
grbs = pygrib.open(fp)
if total is None:
g = grbs[1]
total = g["values"]
lats, lons = g.latlons()
else:
total += grbs[1]["values"]
grbs.close()
now += interval
m = MapPlot(sector='iowa', title='NOAA Stage IV & Iowa ASOS Precipitation',
subtitle='25-30 May 2013')
m.pcolormesh(lons, lats, total / 25.4, numpy.arange(0,14.1,1), latlon=True,
units='inch')
m.drawcounties()
m.plot_values(dlons, dlats, dvals, '%.02f')
m.postprocess(filename='test.svg')
import iemplot
iemplot.makefeature('test')
def main():
"""Do Something"""
cursor = IEM.cursor()
data = []
cursor.execute("""SELECT ST_x(geom), ST_y(geom), tsf0, tsf1, tsf2, tsf3,
id, rwis_subf from current c JOIN stations t on (t.iemid = c.iemid)
WHERE c.valid > now() - '1 hour'::interval""")
for row in cursor:
val = cln(row[2:6])
if val is None:
continue
d = dict(lat=row[1], lon=row[0], tmpf=val, id=row[6])
if row[7] is not None and not np.isnan(row[7]):
d['dwpf'] = row[7]
data.append(d)
now = datetime.datetime.now()
m = MapPlot(axisbg='white',
title='Iowa RWIS Average Pavement + Sub-Surface Temperature',
subtitle=("Valid: %s (pavement in red, sub-surface in blue)"
"") % (now.strftime("%-d %b %Y %-I:%M %p"),))
m.plot_station(data)
m.drawcounties()
pqstr = ("plot c %s rwis_sf.png rwis_sf.png png"
"") % (datetime.datetime.utcnow().strftime("%Y%m%d%H%M"), )
m.postprocess(view=False, pqstr=pqstr)
def run(year):
cursor.execute("""
WITH obs as (
SELECT station, sum(precip) from alldata_ia where year = %s GROUP by station
), climate as (
SELECT station, sum(precip) from climate51 GROUP by station
)
SELECT o.station, o.sum - c.sum as diff from obs o JOIN climate c
on (c.station = o.station) ORDER by diff ASC
""", (year,))
lats = []
lons = []
vals = []
for row in cursor:
if not nt.sts.has_key(row[0]) or row[0] in rejs or row[0][2] == 'C':
continue
print row
lats.append( nt.sts[row[0]]['lat'])
lons.append( nt.sts[row[0]]['lon'])
vals.append( row[1] )
m = MapPlot(title='%s Precipitation Departure' % (year,))
cmap = cm.get_cmap('BrBG')
#cmap.set_over('blue')
#cmap.set_under('red')
m.contourf(lons, lats, vals, np.arange(-24,24.1,2), cmap=cmap, units='inch')
#m.plot_values(lons, lats, vals, '%.02f')
m.drawcounties()
m.postprocess(filename='%s.png' % (year,))
def main():
"""Go Main"""
grbs = pygrib.open('ds.snow.bin')
# skip 1-off first field
total = None
lats = lons = None
for grb in grbs[1:]:
if lats is None:
lats, lons = grb.latlons()
total = grb['values']
continue
total += grb['values']
# TODO tz-hack here
analtime = grb.analDate - datetime.timedelta(hours=5)
mp = MapPlot(
sector='custom', west=-100, east=-92, north=45, south=41,
axisbg='tan',
title=("NWS Forecasted Accumulated Snowfall "
"thru 7 PM 12 April 2019"),
subtitle='NDFD Forecast Issued %s' % (
analtime.strftime("%-I %p %-d %B %Y"), )
)
cmap = nwssnow()
cmap.set_bad('tan')
mp.pcolormesh(
lons, lats, total * 39.3701,
[0.01, 1, 2, 3, 4, 6, 8, 12, 18, 24, 30, 36],
cmap=cmap,
units='inch')
mp.drawcounties()
mp.drawcities()
mp.postprocess(filename='test.png')
mp.close()
def run(base, ceil, now, fn):
""" Generate the plot """
# Compute normal from the climate database
sql = """SELECT station,
sum(gddxx(%s, %s, high, low)) as gdd
from alldata_ia WHERE year = %s and month in (5,6,7,8,9,10)
and station != 'IA0000' and substr(station,2,1) != 'C'
GROUP by station""" % (base, ceil, now.year)
lats = []
lons = []
gdd50 = []
ccursor.execute(sql)
for row in ccursor:
if row[0] not in nt.sts:
continue
lats.append(nt.sts[row[0]]['lat'])
lons.append(nt.sts[row[0]]['lon'])
gdd50.append(float(row[1]))
m = MapPlot(title=("Iowa 1 May - %s GDD Accumulation"
) % (now.strftime("%-d %B %Y"), ),
subtitle="base %s" % (base,))
bins = np.linspace(min(gdd50)-1, max(gdd50)+1, num=10, dtype=np.int)
m.contourf(lons, lats, gdd50, bins)
m.drawcounties()
pqstr = "plot c 000000000000 summary/%s.png bogus png" % (fn,)
m.postprocess(pqstr=pqstr)
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
We should total files from 1 AM to midnight local time
"""
sts = ts.replace(hour=1)
ets = sts + datetime.timedelta(hours=24)
interval = datetime.timedelta(hours=1)
now = sts
total = None
lts = None
while now < ets:
gmt = now.astimezone(pytz.timezone("UTC"))
fn = gmt.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/"
+"stage4/ST4.%Y%m%d%H.01h.grib"))
if os.path.isfile(fn):
lts = now
grbs = pygrib.open(fn)
if total is None:
g = grbs[1]
total = g["values"]
lats, lons = g.latlons()
else:
total += grbs[1]["values"]
grbs.close()
now += interval
if lts is None and ts.hour > 1:
print 'stage4_today_total.py found no data!'
if lts is None:
return
lts = lts - datetime.timedelta(minutes=1)
subtitle = "Total between 12:00 AM and %s" % (lts.strftime("%I:%M %p %Z"),)
routes = 'ac'
if not realtime:
routes = 'a'
for sector in ['iowa', 'midwest', 'conus']:
pqstr = "plot %s %s00 %s_stage4_1d.png %s_stage4_1d.png png" % (routes,
ts.strftime("%Y%m%d%H"), sector, sector )
m = MapPlot(sector=sector,
title="%s NCEP Stage IV Today's Precipitation" % (
ts.strftime("%-d %b %Y"),),
subtitle=subtitle)
clevs = np.arange(0, 0.25, 0.05)
clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs[0] = 0.01
m.pcolormesh(lons, lats, total / 24.5, clevs, units='inch')
#map.drawstates(zorder=2)
if sector == 'iowa':
m.drawcounties()
m.postprocess(pqstr=pqstr)
m.close()
def plotter(fdict):
""" Go """
import matplotlib
matplotlib.use('agg')
pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
state = fdict.get('state', 'IA')[:2]
varname = fdict.get('var', 'total_precip')
sector = fdict.get('sector', 'state')
opt = fdict.get('opt', 'both')
over = fdict.get('over', 'monthly')
month = int(fdict.get('month', datetime.date.today().month))
df = read_sql("""
WITH data as (
SELECT station, extract(month from valid) as month,
sum(precip) as total_precip, avg(high) as avg_high,
avg(low) as avg_low, avg((high+low)/2.) as avg_temp
from ncdc_climate81 GROUP by station, month)
SELECT station, ST_X(geom) as lon, ST_Y(geom) as lat, month,
total_precip, avg_high, avg_low, avg_temp from data d JOIN stations t
ON (d.station = t.id) WHERE t.network = 'NCDC81' and
t.state in ('IA', 'ND', 'SD', 'NE', 'KS', 'MO', 'IL', 'WI', 'MN', 'MI',
'IN', 'OH', 'KY')
""", pgconn, index_col=['station', 'month'])
if over == 'monthly':
title = "%s %s" % (calendar.month_name[month], PDICT3[varname])
df.reset_index(inplace=True)
df2 = df[df['month'] == month]
else:
title = "Annual %s" % (PDICT3[varname], )
if varname == 'total_precip':
df2 = df.sum(axis=0, level='station')
else:
df2 = df.mean(axis=0, level='station')
df2['lat'] = df['lat'].mean(axis=0, level='station')
df2['lon'] = df['lon'].mean(axis=0, level='station')
m = MapPlot(sector=sector, state=state, axisbg='white',
title=('NCEI 1981-2010 Climatology of %s'
) % (title,),
subtitle=('based on National Centers for '
'Environmental Information (NCEI) 1981-2010'
' Climatology'))
levels = np.linspace(df2[varname].min(), df2[varname].max(), 10)
levels = [round(x, PRECISION[varname]) for x in levels]
if opt in ['both', 'contour']:
m.contourf(df2['lon'].values, df2['lat'].values,
df2[varname].values, levels, units=UNITS[varname])
if sector == 'state':
m.drawcounties()
if opt in ['both', 'values']:
m.plot_values(df2['lon'].values, df2['lat'].values,
df2[varname].values,
fmt='%%.%if' % (PRECISION[varname],))
return m.fig, df
def main():
"""GO!"""
pgconn = get_dbconn('idep')
cursor = pgconn.cursor()
scenario = int(sys.argv[1])
mp = MapPlot(sector='iowa', axisbg='white', nologo=True,
subtitle='1 Jan 2014 thru 31 Dec 2014',
caption='Daily Erosion Project',
title=('Harvest Index 0.8 Change in 2014 Soil Delivery '
'from Baseline'))
cursor.execute("""
with baseline as (
SELECT huc_12, sum(avg_delivery) * 4.463 as loss from results_by_huc12
where valid between '2014-01-01' and '2015-01-01' and
scenario = 0 GROUP by huc_12),
scenario as (
SELECT huc_12, sum(avg_delivery) * 4.463 as loss from results_by_huc12
where valid between '2014-01-01' and '2015-01-01' and
scenario = %s GROUP by huc_12),
agg as (
SELECT b.huc_12, b.loss as baseline_loss, s.loss as scenario_loss from
baseline b LEFT JOIN scenario s on (b.huc_12 = s.huc_12))
SELECT ST_Transform(simple_geom, 4326),
(scenario_loss - baseline_loss) / 1.0 as val, i.huc_12
from huc12 i JOIN agg d on (d.huc_12 = i.huc_12)
WHERE i.states ~* 'IA' ORDER by val DESC
""", (scenario, ))
# bins = np.arange(0, 101, 10)
bins = [-5, -2, -1, -0.5, 0, 0.5, 1, 2, 5]
cmap = plt.get_cmap("BrBG_r")
cmap.set_under('purple')
cmap.set_over('black')
norm = mpcolors.BoundaryNorm(bins, cmap.N)
for row in cursor:
# print "%s,%s" % (row[2], row[1])
polygon = loads(row[0].decode('hex'))
arr = np.asarray(polygon.exterior)
points = mp.ax.projection.transform_points(ccrs.Geodetic(),
arr[:, 0], arr[:, 1])
val = float(row[1])
# We have very small negative numbers that should just be near a
# positive zero
if val < 0 and val > -0.1:
val = 0.001
color = cmap(norm([val, ]))[0]
poly = Polygon(points[:, :2], fc=color, ec='k', zorder=2, lw=0.1)
mp.ax.add_patch(poly)
mp.draw_colorbar(bins, cmap, norm, units='T/a/yr')
mp.drawcounties()
mp.postprocess(filename='test.png')
def do(ts, hours):
"""
Create a plot of precipitation stage4 estimates for some day
"""
ts = ts.replace(minute=0)
sts = ts - datetime.timedelta(hours=hours)
ets = ts
interval = datetime.timedelta(hours=1)
now = sts
total = None
lts = None
while now < ets:
fn = "/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib" % (
now.strftime("%Y/%m/%d"),
now.strftime("%Y%m%d%H") )
if os.path.isfile(fn):
lts = now
grbs = pygrib.open(fn)
if total is None:
g = grbs[1]
total = g["values"]
lats, lons = g.latlons()
else:
total += grbs[1]["values"]
grbs.close()
now += interval
if lts is None and ts.hour > 1:
print 'Missing StageIV data!'
if lts is None:
return
cmap = cm.get_cmap("jet")
cmap.set_under('white')
cmap.set_over('black')
clevs = [0.01,0.1,0.25,0.5,1,2,3,5,8,9.9]
localtime = (ts - datetime.timedelta(minutes=1)).astimezone(
pytz.timezone("America/Chicago"))
for sector in ['iowa', 'midwest', 'conus']:
m = MapPlot(sector=sector,
title='NCEP Stage IV %s Hour Precipitation' % (hours,),
subtitle='Total up to %s' % (
localtime.strftime("%d %B %Y %I %p %Z"),))
m.pcolormesh(lons, lats, total / 24.5, clevs, units='inch')
pqstr = "plot %s %s00 %s_stage4_%sh.png %s_stage4_%sh_%s.png png" % (
'ac', ts.strftime("%Y%m%d%H"), sector, hours,
sector, hours, ts.strftime("%H"))
if sector == 'iowa':
m.drawcounties()
m.postprocess(pqstr=pqstr)
m.close()
def main():
pgconn = get_dbconn('idep')
cursor = pgconn.cursor()
mp = MapPlot(continentalcolor='white', nologo=True, # sector='custom',
# south=36.8, north=45.0, west=-99.2, east=-88.9,
caption='Daily Erosion Project',
subtitle='HUC12 Areal Averaged',
title=("Est. Hillslope Soil Loss between 12 April "
"2008 and 12 June 2008 (inclusive)"))
cursor.execute("""
with agg as (
select huc_12, sum(avg_delivery) * 4.463 as runoff from results_by_huc12
WHERE valid >= '2008-04-12' and valid <= '2008-06-12' and scenario = 0
GROUP by huc_12)
SELECT ST_Transform(geom, 4326), h.huc_12,
coalesce(a.runoff, 0) as runoff from huc12 h
LEFT JOIN agg a on (h.huc_12 = a.huc_12) WHERE h.scenario = 0
and h.states ~* 'IA'
ORDER by runoff DESC
""")
#bins = np.arange(0, 26, 2)
bins = [0.05, 0.25, 0.5, 1, 1.5, 2, 3, 5, 7, 10, 15, 20]
cmap = plt.get_cmap("plasma_r")
cmap.set_under('white')
cmap.set_over('black')
norm = mpcolors.BoundaryNorm(bins, cmap.N)
for row in cursor:
# val = distance(row[2], 'MM').value('IN')
val = row[2]
if val > 30:
print("%s %s" % (row[1], val))
multipoly = loads(row[0].decode('hex'))
for polygon in multipoly:
arr = np.asarray(polygon.exterior)
points = mp.ax.projection.transform_points(ccrs.Geodetic(),
arr[:, 0], arr[:, 1])
color = cmap(norm([val, ]))[0]
poly = Polygon(points[:, :2], fc=color, ec='None', zorder=2, lw=.1)
mp.ax.add_patch(poly)
mp.draw_colorbar(bins, cmap, norm, units='tons/acre')
mp.drawcounties()
plt.savefig('test.pdf')
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
lts = datetime.datetime.utcnow().replace(
tzinfo=pytz.timezone("UTC"))
lts = lts.astimezone(pytz.timezone("America/Chicago"))
# make assumptions about the last valid MRMS data
if realtime:
# Up until :59 after of the last hour
lts = (lts - datetime.timedelta(hours=1)).replace(minute=59)
else:
lts = lts.replace(year=ts.year, month=ts.month, day=ts.day,
hour=23, minute=59)
idx = iemre.daily_offset(ts)
ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (ts.year,)
nc = netCDF4.Dataset(ncfn)
precip = nc.variables['p01d'][idx, :, :]
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
subtitle = "Total between 12:00 AM and %s" % (
lts.strftime("%I:%M %p %Z"),)
routes = 'ac'
if not realtime:
routes = 'a'
# clevs = np.arange(0, 0.25, 0.05)
# clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
# clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
10]
sector = 'iowa'
pqstr = ("plot %s %s00 %s_q2_1d.png %s_q2_1d.png png"
) % (routes, ts.strftime("%Y%m%d%H"), sector, sector)
m = MapPlot(title=("%s NCEP MRMS Q3 Today's Precipitation"
) % (ts.strftime("%-d %b %Y"),),
subtitle=subtitle, sector=sector)
(x, y) = np.meshgrid(lons, lats)
m.pcolormesh(x, y, distance(precip, 'MM').value('IN'), clevs,
cmap=nwsprecip(), units='inch')
m.drawcounties()
m.postprocess(pqstr=pqstr, view=False)
m.close()
def plot_precip_month(valid):
""" Go Main Go
Args:
valid (datetime): The timestamp we are interested in!
"""
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
cursor = pgconn.cursor()
d1 = valid.replace(day=1)
d2 = d1 + datetime.timedelta(days=35)
d2 = d2.replace(day=1)
cursor.execute("""SELECT sum(pday), id, st_x(geom), st_y(geom)
from summary s JOIN stations t on
(t.iemid = s.iemid) WHERE s.day >= %s and s.day < %s
and t.network in ('IA_COOP', 'NE_COOP', 'MO_COOP', 'IL_COOP', 'WI_COOP',
'MN_COOP')
and pday is not null and pday >= 0 and
extract(hour from coop_valid) between 5 and 10
GROUP by id, st_x, st_y""", (d1.date(), d2.date()))
labels = []
vals = []
lats = []
lons = []
for row in cursor:
labels.append(row[1])
vals.append(pretty(row[0]))
lats.append(row[3])
lons.append(row[2])
m = MapPlot(title='%s NWS COOP Month Precipitation Totals [inch]' % (
valid.strftime("%-d %b %Y"),),
subtitle='Reports valid between 6 and 9 AM',
axisbg='white', figsize=(10.24, 7.68))
m.plot_values(lons, lats, vals, fmt='%s', labels=labels,
labelcolor='tan')
m.drawcounties()
pqstr = "plot ac %s0000 coopMonthPlot.gif coopMonthPlot.gif gif" % (
valid.strftime("%Y%m%d"),)
m.postprocess(pqstr=pqstr)
m.close()
pgconn.close()
def plotter(fdict):
""" Go """
import matplotlib
matplotlib.use('agg')
from pyiem.plot import MapPlot
pgconn = psycopg2.connect(dbname='coop', host='iemdb', user='******')
ctx = util.get_autoplot_context(fdict, get_description())
sector = ctx['sector']
varname = ctx['var']
year = ctx['year']
popt = ctx['popt']
threshold = ctx['threshold']
table = "alldata_%s" % (sector,)
df = read_sql("""
WITH data as (
SELECT station, """ + SQLOPT[varname] + """ as doy
from """ + table + """
WHERE year = %s GROUP by station
)
select station, doy, st_x(geom) as lon, st_y(geom) as lat
from data d JOIN stations t on (d.station = t.id) WHERE
t.network = %s and substr(station, 3, 4) != '0000'
and substr(station, 3, 1) != 'C' and doy not in (0, 400) ORDER by doy
""", pgconn, params=(threshold, year, '%sCLIMATE' % (sector,)),
index_col='station')
if len(df.index) == 0:
return "No data found!"
def f(val):
ts = datetime.date(year, 1, 1) + datetime.timedelta(days=(val - 1))
return ts.strftime("%-m/%-d")
df['pdate'] = df['doy'].apply(f)
m = MapPlot(sector='state', state=sector, axisbg='white', nocaption=True,
title="%s %s %s$^\circ$F" % (year, PDICT2[varname], threshold),
subtitle='based on NWS COOP and IEM Daily Estimates')
levs = np.linspace(df['doy'].min() - 3, df['doy'].max() + 3, 7, dtype='i')
levlables = map(f, levs)
if popt == 'contour':
m.contourf(df['lon'], df['lat'], df['doy'], levs, clevlabels=levlables)
m.plot_values(df['lon'], df['lat'], df['pdate'], labelbuffer=5)
m.drawcounties()
return m.fig, df
def main():
"""Go Main!"""
nt = NetworkTable("IACLIMATE")
pgconn = get_dbconn('coop')
df = read_sql("""
with monthly as (
select station, year, month, avg((high+low)/2.) from alldata_ia
WHERE day < '2018-06-01' and high is not null
GROUP by station, year, month),
agg as (
select station, year, month, avg,
lag(avg) OVER (PARTITION by station ORDER by year ASC, month ASC)
from monthly),
agg2 as (
select station, year, month, avg, lag, avg - lag as val,
rank() OVER (PARTITION by station ORDER by avg - lag DESC)
from agg WHERE lag is not null)
select * from agg2 where rank = 1 ORDER by station
""", pgconn, index_col='station')
df['lat'] = 0.
df['lon'] = 0.
for station, _ in df.iterrows():
if station in nt.sts and station != 'IA0000' and station[2] != 'C':
df.at[station, 'lat'] = nt.sts[station]['lat']
df.at[station, 'lon'] = nt.sts[station]['lon']
mp = MapPlot(title="Largest Positive Change in Month to Month Average Temperature",
subtitle=('values in red set record for April to May 2018'), sector='state',
state='IA',
drawstates=True, continentalcolor='white')
df2 = df[df['year'] == 2018]
mp.plot_values(df2['lon'].values,
df2['lat'].values,
df2['val'].values, fmt='%.1f', textsize=12, labelbuffer=5,
color='r')
df2 = df[df['year'] != 2018]
mp.plot_values(df2['lon'].values,
df2['lat'].values,
df2['val'].values, fmt='%.1f', textsize=12, labelbuffer=5,
color='b')
mp.drawcounties()
mp.postprocess(filename='test.png')
mp.close()
def plot_snowdepth(valid):
""" Go Main Go
Args:
valid (datetime): The timestamp we are interested in!
"""
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
cursor = pgconn.cursor()
cursor.execute("""SELECT snowd, id, st_x(geom), st_y(geom)
from summary s JOIN stations t on
(t.iemid = s.iemid) WHERE s.day = %s
and t.network in ('IA_COOP', 'NE_COOP', 'MO_COOP', 'IL_COOP', 'WI_COOP',
'MN_COOP')
and snowd is not null and snowd >= 0 and
extract(hour from coop_valid) between 5 and 10""", (valid.date(),))
labels = []
vals = []
lats = []
lons = []
for row in cursor:
labels.append(row[1])
vals.append(pretty(row[0], precision=0))
lats.append(row[3])
lons.append(row[2])
m = MapPlot(title='%s NWS COOP Snowfall Depth Reports [inch]' % (
valid.strftime("%-d %b %Y"),),
subtitle='Reports valid between 6 and 9 AM',
axisbg='white', figsize=(10.24, 7.68))
m.plot_values(lons, lats, vals, fmt='%s', labels=labels,
labelcolor='tan')
m.drawcounties()
pqstr = "plot ac %s0000 coopSnowDepth.gif coopSnowDepth.gif gif" % (
valid.strftime("%Y%m%d"),)
m.postprocess(pqstr=pqstr)
m.close()
pgconn.close()
def main():
"""Go Main"""
nc = netCDF4.Dataset('/tmp/sfav2_CONUS_2018093012_to_2019021312.nc')
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
data = nc.variables['Data'][:] * 1000. / 25.4
nc.close()
mp = MapPlot(
sector='iowa', continentalcolor='tan',
title=("National Snowfall Analysis - NOHRSC "
"- Season Total Snowfall"),
subtitle='Snowfall up until 7 AM 13 Feb 2019')
cmap = plt.get_cmap('terrain_r')
levs = [0.1, 2, 5, 8, 12, 18, 24, 30, 36, 42, 48]
mp.pcolormesh(
lons, lats, data, levs, cmap=cmap, units='inch', clip_on=False,
spacing='proportional'
)
mp.drawcounties()
mp.drawcities()
mp.postprocess(filename='test.png')
def doit(ts):
"""
Generate hourly plot of stage4 data
"""
routes = "a"
if ((gmtnow - ts).days * 86400. + (gmtnow - ts).seconds) < 7200:
routes = "ac"
fn = "/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib" % (
ts.strftime("%Y/%m/%d"), ts.strftime("%Y%m%d%H") )
if not os.path.isfile(fn):
print 'current/stage4_hourly.py Missing stage4 %s' % (fn,)
return
grbs = pygrib.open(fn)
grib = grbs[1]
lats, lons = grib.latlons()
vals = grib.values / 25.4
cmap = cm.get_cmap("jet")
cmap.set_under('white')
cmap.set_over('black')
clevs = [0.01,0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1,1.5,2,3]
localtime = ts.astimezone(pytz.timezone("America/Chicago"))
for sector in ['iowa', 'midwest', 'conus']:
m = MapPlot(sector=sector,
title='Stage IV One Hour Precipitation',
subtitle='Hour Ending %s' % (
localtime.strftime("%d %B %Y %I %p %Z"),))
m.pcolormesh(lons, lats, vals, clevs, units='inch')
pqstr = "plot %s %s00 %s_stage4_1h.png %s_stage4_1h_%s.png png" % (
routes, ts.strftime("%Y%m%d%H"), sector,
sector, ts.strftime("%H"))
if sector == 'iowa':
m.drawcounties()
m.postprocess(view=False, pqstr=pqstr)
m.close()
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
nc = netCDF4.Dataset("/mesonet/data/iemre/%s_ifc_daily.nc" % (ts.year,))
idx = daily_offset(ts)
xaxis = nc.variables['lon'][:]
yaxis = nc.variables['lat'][:]
total = nc.variables['p01d'][idx, :, :]
nc.close()
lastts = datetime.datetime(ts.year, ts.month, ts.day, 23, 59)
if realtime:
now = datetime.datetime.now() - datetime.timedelta(minutes=60)
lastts = now.replace(minute=59)
subtitle = "Total between 12:00 AM and %s" % (
lastts.strftime("%I:%M %p"),)
routes = 'ac'
if not realtime:
routes = 'a'
clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
10]
pqstr = ("plot %s %s00 iowa_ifc_1d.png iowa_ifc_1d.png png"
) % (routes, ts.strftime("%Y%m%d%H"))
m = MapPlot(title=("%s Iowa Flood Center Today's Precipitation"
) % (ts.strftime("%-d %b %Y"),),
subtitle=subtitle, sector='custom',
west=xaxis[0], east=xaxis[-1],
south=yaxis[0], north=yaxis[-1])
(x, y) = np.meshgrid(xaxis, yaxis)
m.pcolormesh(x, y, distance(total, 'MM').value("IN"), clevs,
cmap=nwsprecip(), units='inch')
m.drawcounties()
m.postprocess(pqstr=pqstr, view=False)
m.close()
def main():
"""Go Main Go"""
now = datetime.datetime.now()
nt = NetworkTable('IACLIMATE')
nt.sts["IA0200"]["lon"] = -93.6
nt.sts["IA5992"]["lat"] = 41.65
coop = get_dbconn('coop', user='******')
# Compute normal from the climate database
sql = """SELECT station, high, low from climate WHERE valid = '2000-%s'
and substr(station,0,3) = 'IA'""" % (now.strftime("%m-%d"), )
obs = []
cursor = coop.cursor(cursor_factory=psycopg2.extras.DictCursor)
cursor.execute(sql)
for row in cursor:
sid = row['station']
if sid[2] == 'C' or sid[2:] == '0000' or sid not in nt.sts:
continue
obs.append(
dict(id=sid[2:],
lat=nt.sts[sid]['lat'],
lon=nt.sts[sid]['lon'],
tmpf=row['high'],
dwpf=row['low']))
mp = MapPlot(title=("Average High + Low Temperature [F] (1893-%s)") %
(now.year, ),
subtitle="For Date: %s" % (now.strftime("%d %b"), ),
axisbg='white')
mp.drawcounties()
mp.plot_station(obs)
pqstr = ("plot ac %s0000 climate/iowa_today_avg_hilo_pt.png "
"coop_avg_temp.png png") % (now.strftime("%Y%m%d"), )
mp.postprocess(view=False, pqstr=pqstr)
mp.close()
def plot_hilo(valid):
""" Go Main Go
Args:
valid (datetime): The timestamp we are interested in!
"""
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
cursor = pgconn.cursor()
cursor.execute("""SELECT max_tmpf, min_tmpf, id, st_x(geom), st_y(geom)
from summary s JOIN stations t on
(t.iemid = s.iemid) WHERE s.day = %s
and t.network in ('IA_COOP', 'NE_COOP', 'MO_COOP', 'IL_COOP', 'WI_COOP',
'MN_COOP')
and max_tmpf is not null and max_tmpf >= -30 and
min_tmpf is not null and min_tmpf < 99 and
extract(hour from coop_valid) between 5 and 10""", (valid.date(),))
data = []
for row in cursor:
data.append(dict(lat=row[4], lon=row[3], tmpf=row[0],
dwpf=row[1], id=row[2]))
m = MapPlot(title=('%s NWS COOP 24 Hour High/Low Temperature [$^\circ$F]'
) % (valid.strftime("%-d %b %Y"),),
subtitle='Reports valid between 6 and 9 AM',
axisbg='white', figsize=(10.24, 7.68))
m.plot_station(data)
m.drawcounties()
pqstr = "plot ac %s0000 coopHighLow.gif coopHighLow.gif gif" % (
valid.strftime("%Y%m%d"),)
m.postprocess(pqstr=pqstr)
m.close()
pgconn.close()
def main():
"""Go"""
pgconn = psycopg2.connect(database='iem', host='localhost', port=5555,
user='******')
df = read_sql("""
SELECT id, network, ST_x(geom) as lon,
ST_y(geom) as lat, min(min_tmpf)
from summary_2017 s JOIN stations t on (t.iemid = s.iemid)
WHERE network IN ('IA_ASOS','AWOS') and min_tmpf > -50 and
day > '2017-08-01' and id not in ('XXX')
GROUP by id, network, lon, lat ORDER by min ASC
""", pgconn, index_col=None)
mp = MapPlot(title=r"2017 Fall Season Minimum Temperature $^\circ$F",
axisbg='white',
subtitle=('Automated Weather Stations ASOS/AWOS, '
'Valid Fall 2017 thru %s'
) % (datetime.datetime.now().strftime("%d %b %Y"),))
# m.contourf(lons, lats, vals, np.arange(-30,1,4))
mp.plot_values(df['lon'].values, df['lat'].values, df['min'].values,
'%.0f', labels=df['id'].values, labelbuffer=3)
mp.drawcounties()
mp.postprocess(filename='test.png')
def do_month(year, month, routes):
""" Generate a MRMS plot for the month!"""
sts = datetime.datetime(year, month, 1)
ets = sts + datetime.timedelta(days=35)
ets = ets.replace(day=1)
today = datetime.datetime.now()
if ets > today:
ets = today
idx0 = iemre.daily_offset(sts)
idx1 = iemre.daily_offset(ets)
nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (year, ),
'r')
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
p01d = np.sum(nc.variables['p01d'][idx0:idx1, :, :], 0) / 24.5
nc.close()
m = MapPlot(sector='iowa',
title='MRMS %s - %s Total Precipitation' %
(sts.strftime("%-d %b"),
(ets - datetime.timedelta(days=1)).strftime("%-d %b %Y")),
subtitle='Data from NOAA MRMS Project')
x, y = np.meshgrid(lons, lats)
bins = [0.01, 0.1, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20]
m.pcolormesh(x, y, p01d, bins, units='inches')
m.drawcounties()
currentfn = "summary/iowa_mrms_q3_month.png"
archivefn = sts.strftime("%Y/%m/summary/iowa_mrms_q3_month.png")
pqstr = "plot %s %s00 %s %s png" % (routes, sts.strftime("%Y%m%d%H"),
currentfn, archivefn)
m.postprocess(pqstr=pqstr)
def main():
"""Go Main Go."""
nt = NetworkTable(['IA_ASOS', 'AWOS'])
pgconn = get_dbconn('iem')
cursor = pgconn.cursor()
lats = []
lons = []
vals = []
cursor.execute("""WITH today as (
SELECT id, max_tmpf from summary_2017 s JOIN stations t
ON (s.iemid = t.iemid) WHERE t.network in ('IA_ASOS', 'AWOS')
and day = '2017-01-18'),
agg as (
SELECT t.id, max(day) from summary s, stations t, today t2
WHERE s.iemid = t.iemid and t.id = t2.id and
t.network in ('IA_ASOS', 'AWOS') and day > '2016-12-01'
and day < '2017-01-18' and s.max_tmpf >= t2.max_tmpf
GROUP by t.id)
SELECT id, max from agg ORDER by max ASC
""")
colors = []
for row in cursor:
lats.append(nt.sts[row[0]]['lat'])
lons.append(nt.sts[row[0]]['lon'])
vals.append(row[1].strftime("%-m/%-d"))
colors.append('r' if row[1] == datetime.date(2016, 12, 25) else 'k')
m = MapPlot(continentalcolor='white',
title=("Iowa ASOS/AWOS Last Date of as Warm of "
"Daily High as 18 Jan 2017"))
m.plot_values(lons, lats, vals, fmt='%s', color=colors, labelbuffer=5)
m.drawcounties()
m.postprocess(filename='170119.png')
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
idx = daily_offset(ts)
with ncopen("/mesonet/data/iemre/%s_ifc_daily.nc" % (ts.year, ),
timeout=300) as nc:
xaxis = nc.variables["lon"][:]
yaxis = nc.variables["lat"][:]
total = nc.variables["p01d"][idx, :, :]
lastts = datetime.datetime(ts.year, ts.month, ts.day, 23, 59)
if realtime:
now = datetime.datetime.now() - datetime.timedelta(minutes=60)
lastts = now.replace(minute=59)
subtitle = "Total between 12:00 AM and %s" % (
lastts.strftime("%I:%M %p"), )
routes = "ac"
if not realtime:
routes = "a"
clevs = [
0.01,
0.1,
0.25,
0.5,
0.75,
1,
1.5,
2,
2.5,
3,
3.5,
4,
5,
6,
8,
10,
]
pqstr = ("plot %s %s00 iowa_ifc_1d.png iowa_ifc_1d.png png") % (
routes,
ts.strftime("%Y%m%d%H"),
)
mp = MapPlot(
title=("%s Iowa Flood Center Today's Precipitation") %
(ts.strftime("%-d %b %Y"), ),
subtitle=subtitle,
sector="custom",
west=xaxis[0],
east=xaxis[-1],
south=yaxis[0],
north=yaxis[-1],
)
(lons, lats) = np.meshgrid(xaxis, yaxis)
mp.pcolormesh(
lons,
lats,
distance(total, "MM").value("IN"),
clevs,
cmap=nwsprecip(),
units="inch",
)
mp.drawcounties()
mp.postprocess(pqstr=pqstr, view=False)
mp.close()
def doit(ts, hours):
"""
Create a plot of precipitation stage4 estimates for some day
"""
# Start at 1 AM
ts = ts.replace(minute=0, second=0, microsecond=0)
now = ts - datetime.timedelta(hours=hours - 1)
interval = datetime.timedelta(hours=1)
ets = datetime.datetime.utcnow()
ets = ets.replace(tzinfo=pytz.utc)
total = None
while now < ets:
gmt = now.astimezone(pytz.utc)
gribfn = None
for prefix in ["MultiSensor_QPE_01H_Pass2", "RadarOnly_QPE_01H"]:
gribfn = mrms.fetch(prefix, gmt)
if gribfn is None:
continue
break
if gribfn is None:
LOG.info("%s MISSING %s", hours, now)
now += interval
continue
fp = gzip.GzipFile(gribfn, "rb")
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, "wb")
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb["values"]
else:
maxgrid = np.maximum(grb["values"], total)
total = np.where(
np.logical_and(grb["values"] >= 0, total >= 0),
grb["values"] + total,
maxgrid,
)
now += interval
os.unlink(gribfn)
if total is None:
LOG.info("no data ts: %s hours: %s", ts, hours)
return
# Scale factor is 10
routes = "c"
if ts.minute == 0:
routes = "ac"
pqstr = "plot %s %s iowa_q2_%sh.png q2/iowa_q2_%sh_%s00.png png" % (
routes,
ts.strftime("%Y%m%d%H%M"),
hours,
hours,
ts.strftime("%H"),
)
lts = ts.astimezone(pytz.timezone("America/Chicago"))
subtitle = "Total up to %s" % (lts.strftime("%d %B %Y %I:%M %p %Z"), )
mp = MapPlot(
title=("NCEP MRMS Q3 (RADAR Only) %s Hour "
"Precipitation [inch]") % (hours, ),
subtitle=subtitle,
)
clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
mp.contourf(
mrms.XAXIS,
mrms.YAXIS,
distance(np.flipud(total), "MM").value("IN"),
clevs,
cmap=nwsprecip(),
)
mp.drawcounties()
mp.postprocess(pqstr=pqstr, view=False)
mp.close()
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
varname = ctx["v"]
df = get_df(ctx)
if df.empty:
raise NoDataFound("No data was found for your query")
mp = MapPlot(
sector=("state" if ctx["t"] == "state" else "cwa"),
state=ctx["state"],
cwa=(ctx["wfo"] if len(ctx["wfo"]) == 3 else ctx["wfo"][1:]),
axisbg="white",
title="%s for %s on %s" % (PDICT2[ctx["v"]], ctx["title"], ctx["day"]),
nocaption=True,
titlefontsize=16,
)
ramp = None
cmap = get_cmap(ctx["cmap"])
extend = "both"
if varname in ["max_gust", "max_sknt"]:
extend = "max"
ramp = np.arange(0, 40, 4)
ramp = np.append(ramp, np.arange(40, 80, 10))
ramp = np.append(ramp, np.arange(80, 120, 20))
# Data QC, cough
if ctx.get("above"):
df = df[df[varname] < ctx["above"]]
if ctx.get("below"):
df = df[df[varname] > ctx["below"]]
# with QC done, we compute ramps
if ramp is None:
ramp = np.linspace(df[varname].min() - 5,
df[varname].max() + 5,
10,
dtype="i")
if ctx["p"] == "both":
mp.contourf(
df["lon"].values,
df["lat"].values,
df[varname].values,
ramp,
units=VARUNITS[varname],
cmap=cmap,
spacing="proportional",
extend=extend,
)
if ctx["t"] == "state":
df2 = df[df[ctx["t"]] == ctx[ctx["t"]]]
else:
df2 = df[df["wfo"] == ctx["wfo"]]
mp.plot_values(
df2["lon"].values,
df2["lat"].values,
df2[varname].values,
"%.1f" if varname in ["max_gust", "max_sknt"] else "%.0f",
labelbuffer=3,
)
mp.drawcounties()
if ctx["t"] == "cwa":
mp.draw_cwas()
return mp.fig, df
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
varname = ctx["v"]
if ctx["t"] == "state":
bnds = reference.state_bounds[ctx["state"]]
title = reference.state_names[ctx["state"]]
else:
bnds = reference.wfo_bounds[ctx["wfo"]]
title = "NWS CWA %s [%s]" % (
ctx["_nt"].sts[ctx["wfo"]]["name"],
ctx["wfo"],
)
df, valid = get_df(ctx, bnds)
if df.empty:
raise NoDataFound("No data was found for your query")
mp = MapPlot(
sector=("state" if ctx["t"] == "state" else "cwa"),
state=ctx["state"],
cwa=(ctx["wfo"] if len(ctx["wfo"]) == 3 else ctx["wfo"][1:]),
axisbg="white",
title="%s for %s" % (PDICT2[ctx["v"]], title),
subtitle=("Map valid: %s UTC") % (valid.strftime("%d %b %Y %H:%M"),),
nocaption=True,
titlefontsize=16,
)
if varname == "vsby":
ramp = np.array([0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 8, 9.9])
valunit = "miles"
elif varname == "feel":
valunit = "F"
df["feel"] = (
apparent_temperature(
df["tmpf"].values * units("degF"),
df["relh"].values * units("percent"),
df["sknt"].values * units("knots"),
)
.to(units("degF"))
.m
)
# Data QC, cough
if ctx.get("above"):
df = df[df[varname] < ctx["above"]]
if ctx.get("below"):
df = df[df[varname] > ctx["below"]]
# with QC done, we compute ramps
if varname != "vsby":
ramp = np.linspace(
df[varname].min() - 5, df[varname].max() + 5, 10, dtype="i"
)
mp.contourf(
df["lon"].values,
df["lat"].values,
df[varname].values,
ramp,
units=valunit,
cmap=get_cmap(ctx["cmap"]),
)
if ctx["t"] == "state":
df2 = df[df["state"] == ctx["state"]]
else:
df2 = df[df["wfo"] == ctx["wfo"]]
mp.plot_values(
df2["lon"].values,
df2["lat"].values,
df2[varname].values,
"%.1f",
labelbuffer=10,
)
mp.drawcounties()
if ctx["t"] == "cwa":
mp.draw_cwas()
return mp.fig, df
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
csector = ctx["csector"]
date = ctx["date"]
z = ctx["z"]
period = ctx["f"]
scale = ctx["scale"]
valid = utc(date.year, date.month, date.day, int(z))
gribfn = valid.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/wpc/"
"p" + period + "m_%Y%m%d%Hf" + period + ".grb"))
if not os.path.isfile(gribfn):
raise NoDataFound("gribfn %s missing" % (gribfn, ))
grbs = pygrib.open(gribfn)
grb = grbs[1]
precip = distance(grb.values, "MM").value("IN")
lats, lons = grb.latlons()
title = ("Weather Prediction Center %s Quantitative "
"Precipitation Forecast") % (PDICT[period])
subtitle = ("%sWPC Forecast %s UTC to %s UTC") % (
("US Drought Monitor Overlaid, " if ctx["opt"] == "both" else ""),
valid.strftime("%d %b %Y %H"),
(valid +
datetime.timedelta(hours=int(period))).strftime("%d %b %Y %H"),
)
mp = MapPlot(
sector=("state" if len(csector) == 2 else csector),
state=ctx["csector"],
title=title,
subtitle=subtitle,
continentalcolor="white",
titlefontsize=16,
)
cmap = plt.get_cmap(ctx["cmap"])
cmap.set_under("#EEEEEE")
cmap.set_over("black")
if scale == "auto":
levs = np.linspace(0, np.max(precip) * 1.1, 10)
levs = [round(lev, 2) for lev in levs]
levs[0] = 0.01
elif scale == "10":
levs = np.arange(0, 10.1, 1.0)
levs[0] = 0.01
elif scale == "7":
levs = np.arange(0, 7.1, 0.5)
levs[0] = 0.01
elif scale == "3.5":
levs = np.arange(0, 3.6, 0.25)
levs[0] = 0.01
mp.pcolormesh(
lons,
lats,
precip,
levs,
cmap=cmap,
units="inch",
clip_on=(ctx["csector"] == "iailin"),
)
if ctx["opt"] == "both":
mp.draw_usdm(valid=valid, filled=False, hatched=True)
if ctx["csector"] == "iailin":
mp.drawcounties()
return mp.fig
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
csector = ctx['csector']
sdate = make_tuesday(ctx['sdate'])
edate = make_tuesday(ctx['edate'])
dlevel = ctx['d']
griddelta = 0.1
mp = MapPlot(sector=('state' if len(csector) == 2 else csector),
state=ctx['csector'],
title=('%s at or above "%s" %s - %s') %
(PDICT2[ctx['w']], PDICT[dlevel],
sdate.strftime("%b %-d, %Y"), edate.strftime("%b %-d, %Y")),
subtitle=('based on weekly US Drought Monitor Analysis, '
'%.2f$^\circ$ grid analysis') % (griddelta, ),
continentalcolor='white',
titlefontsize=14)
# compute the affine
(west, east, south, north) = mp.ax.get_extent(ccrs.PlateCarree())
raster = np.zeros((int(
(north - south) / griddelta), int((east - west) / griddelta)))
lons = np.arange(raster.shape[1]) * griddelta + west
lats = np.arange(0, 0 - raster.shape[0], -1) * griddelta + north
lats = lats[::-1]
affine = Affine(griddelta, 0., west, 0., 0 - griddelta, north)
# get the geopandas data
pgconn = get_dbconn('postgis')
df = read_postgis("""
with d as (
select valid, (ST_Dump(st_simplify(geom, 0.01))).geom from usdm where
valid >= %s and valid <= %s and dm >= %s and
ST_Intersects(geom, ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s %s,
%s %s, %s %s, %s %s))'))
)
select valid, st_collect(geom) as the_geom from d GROUP by valid
""",
pgconn,
params=(sdate, edate, dlevel, west, south, west, north,
east, north, east, south, west, south),
geom_col='the_geom')
if df.empty:
raise NoDataFound("No Data Found, sorry!")
# loop over the cached stats
czs = CachingZonalStats(affine)
czs.compute_gridnav(df['the_geom'], raster)
for nav in czs.gridnav:
if nav is None:
continue
grid = np.ones((nav.ysz, nav.xsz))
grid[nav.mask] = 0.
jslice = slice(nav.y0, nav.y0 + nav.ysz)
islice = slice(nav.x0, nav.x0 + nav.xsz)
raster[jslice, islice] += grid
maxval = 10 if np.max(raster) < 11 else np.max(raster)
ramp = np.linspace(1, maxval + 1, 11, dtype='i')
if ctx['w'] == 'percent':
ramp = np.arange(0, 101, 10)
ramp[0] = 1.
ramp[-1] = 100.1
# we add one since we are rectified to tuesdays, so we have an extra
# week in there
raster = raster / ((edate - sdate).days / 7. + 1.) * 100.
# plot
cmap = stretch_cmap(ctx['cmap'], ramp)
cmap.set_under('white')
cmap.set_bad('white')
mp.pcolormesh(lons,
lats,
np.flipud(raster),
ramp,
cmap=cmap,
units='count' if ctx['w'] == 'weeks' else 'Percent')
if len(csector) == 2:
mp.drawcounties()
mp.drawcities()
rows = []
for j in range(raster.shape[0]):
for i in range(raster.shape[1]):
rows.append(dict(lon=lons[i], lat=lats[j], value=raster[j, i]))
return mp.fig, pd.DataFrame(rows)
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
We should total files from 1 AM to midnight local time
"""
sts = ts.replace(hour=1)
ets = sts + datetime.timedelta(hours=24)
interval = datetime.timedelta(hours=1)
now = sts
total = None
lts = None
while now < ets:
gmt = now.astimezone(pytz.timezone("UTC"))
fn = gmt.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/"
"stage4/ST4.%Y%m%d%H.01h.grib"))
if os.path.isfile(fn):
lts = now
grbs = pygrib.open(fn)
if total is None:
g = grbs[1]
total = g["values"]
lats, lons = g.latlons()
else:
total += grbs[1]["values"]
grbs.close()
now += interval
if lts is None and ts.hour > 1:
print 'stage4_today_total.py found no data!'
if lts is None:
return
lts = lts - datetime.timedelta(minutes=1)
subtitle = "Total between 12:00 AM and %s" % (
lts.strftime("%I:%M %p %Z"), )
routes = 'ac'
if not realtime:
routes = 'a'
for sector in ['iowa', 'midwest', 'conus']:
pqstr = ("plot %s %s00 %s_stage4_1d.png %s_stage4_1d.png png") % (
routes, ts.strftime("%Y%m%d%H"), sector, sector)
m = MapPlot(sector=sector,
title="%s NCEP Stage IV Today's Precipitation" %
(ts.strftime("%-d %b %Y"), ),
subtitle=subtitle)
clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
m.pcolormesh(lons,
lats,
distance(total, 'MM').value('IN'),
clevs,
cmap=nwsprecip(),
units='inch')
# map.drawstates(zorder=2)
if sector == 'iowa':
m.drawcounties()
m.postprocess(pqstr=pqstr)
m.close()
def plotter(fdict):
""" Go """
pgconn = get_dbconn('coop')
ctx = get_autoplot_context(fdict, get_description())
sector = ctx['sector']
date1 = ctx['date1']
date2 = ctx['date2']
varname = ctx['var']
table = "alldata_%s" % (sector, ) if sector != 'midwest' else "alldata"
df = read_sql("""
WITH obs as (
SELECT station, gddxx(%s, 86, high, low) as gdd,
cdd(high, low, 65) as cdd65, hdd(high, low, 65) as hdd65,
sday, high, low, precip,
(high + low)/2. as avg_temp
from """ + table + """ WHERE
day >= %s and day < %s and
substr(station, 3, 1) != 'C' and substr(station, 3, 4) != '0000'),
climo as (
SELECT station, to_char(valid, 'mmdd') as sday, precip, high, low,
gdd""" + str(ctx['gddbase']) + """ as gdd, cdd65, hdd65
from climate51),
combo as (
SELECT o.station, o.precip - c.precip as precip_diff,
o.precip as precip, c.precip as cprecip,
o.avg_temp, o.cdd65, o.hdd65,
o.high, o.low, o.gdd, c.gdd as cgdd,
o.gdd - c.gdd as gdd_diff,
o.cdd65 - c.cdd65 as cdd_diff,
o.hdd65 - c.hdd65 as hdd_diff,
o.avg_temp - (c.high + c.low)/2. as temp_diff
from obs o JOIN climo c ON
(o.station = c.station and o.sday = c.sday)),
agg as (
SELECT station,
avg(avg_temp) as avg_temp,
sum(precip_diff) as precip_depart,
sum(precip) as precip, sum(cprecip) as cprecip,
avg(high) as avg_high_temp,
avg(low) as avg_low_temp,
max(high) as max_high_temp,
min(low) as min_low_temp, sum(gdd_diff) as gdd_depart,
avg(temp_diff) as avg_temp_depart, sum(gdd) as gdd_sum,
sum(cgdd) as cgdd_sum,
sum(cdd65) as cdd_sum,
sum(hdd65) as hdd_sum,
sum(cdd_diff) as cdd_depart,
sum(hdd_diff) as hdd_depart
from combo GROUP by station)
SELECT d.station, t.name,
precip as precip_sum,
avg_temp,
cprecip as cprecip_sum,
precip_depart,
min_low_temp,
avg_temp_depart,
gdd_depart,
gdd_sum,
cgdd_sum,
max_high_temp,
avg_high_temp,
avg_low_temp,
cdd_sum, hdd_sum, cdd_depart, hdd_depart,
ST_x(t.geom) as lon, ST_y(t.geom) as lat
from agg d JOIN stations t on (d.station = t.id)
WHERE t.network ~* 'CLIMATE'
""",
pgconn,
params=(ctx['gddbase'], date1, date2),
index_col='station')
df = df.reindex(df[varname].abs().sort_values(ascending=False).index)
sector2 = "state" if sector != 'midwest' else 'midwest'
datefmt = "%d %b %Y" if varname != 'cgdd_sum' else '%d %b'
subtitle = ''
if varname.find('depart') > -1:
subtitle = ('%s is compared with 1951-%s Climatology'
' to compute departures') % (
date1.year, datetime.date.today().year - 1)
elif varname.startswith('c'):
subtitle = ("Climatology is based on data from 1951-%s") % (
datetime.date.today().year - 1, )
mp = MapPlot(sector=sector2,
state=sector,
axisbg='white',
title=('%s - %s %s [%s]') %
(date1.strftime(datefmt), date2.strftime(datefmt),
PDICT2.get(varname).replace("$base", str(
ctx['gddbase'])), UNITS.get(varname)),
subtitle=subtitle)
fmt = '%.2f'
if varname in ['precip_depart', 'avg_temp_depart']:
rng = df[varname].abs().describe(percentiles=[0.95])['95%']
clevels = np.linspace(0 - rng, rng, 7)
cmap = cm.get_cmap('RdYlBu' if varname ==
'precip_depart' else 'RdYlBu_r')
elif varname in ['precip_sum']:
rng = df[varname].abs().describe(percentiles=[0.95])['95%']
clevels = np.linspace(0, rng, 7)
cmap = cm.get_cmap('plasma_r')
cmap.set_under('white')
cmap.set_over('black')
else:
minv = df[varname].min() - 5
maxv = df[varname].max() + 5
clevels = np.linspace(minv, maxv, 6, dtype='i')
fmt = '%.0f'
cmap = cm.get_cmap('RdYlBu_r')
clevlabels = [fmt % x for x in clevels]
cmap.set_bad('white')
if ctx['p'] == 'contour':
mp.contourf(df['lon'].values,
df['lat'].values,
df[varname].values,
clevels,
clevlabels=clevlabels,
cmap=cmap,
units=UNITS.get(varname))
mp.plot_values(df['lon'].values,
df['lat'].values,
df[varname].values,
fmt=fmt,
labelbuffer=5)
if len(sector) == 2:
mp.drawcounties()
if ctx['usdm'] == 'yes':
mp.draw_usdm(date2, filled=False, hatched=True)
return mp.fig, df
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
ts = ctx["ts"].replace(tzinfo=pytz.utc)
hour = int(ctx["hour"])
ilabel = ctx["ilabel"] == "yes"
plot = MapPlot(
sector=ctx["t"],
continentalcolor="white",
state=ctx["state"],
cwa=ctx["wfo"],
title=("NWS RFC %s Hour Flash Flood Guidance on %s UTC") %
(hour, ts.strftime("%-d %b %Y %H")),
subtitle=("Estimated amount of %s Rainfall "
"needed for non-urban Flash Flooding to commence") %
(HOURS[ctx["hour"]], ),
)
cmap = plt.get_cmap(ctx["cmap"])
bins = [
0.01,
0.6,
0.8,
1.0,
1.2,
1.4,
1.6,
1.8,
2.0,
2.25,
2.5,
2.75,
3.0,
3.5,
4.0,
5.0,
]
if ts.year < 2019:
column = "hour%02i" % (hour, )
pgconn = get_dbconn("postgis")
df = read_sql(
"""
WITH data as (
SELECT ugc, rank() OVER (PARTITION by ugc ORDER by valid DESC),
hour01, hour03, hour06, hour12, hour24
from ffg WHERE valid >= %s and valid <= %s)
SELECT *, substr(ugc, 3, 1) as ztype from data where rank = 1
""",
pgconn,
params=(ts - datetime.timedelta(hours=24), ts),
index_col="ugc",
)
df2 = df[df["ztype"] == "C"]
plot.fill_ugcs(
df2[column].to_dict(),
bins,
cmap=cmap,
plotmissing=False,
ilabel=ilabel,
)
df2 = df[df["ztype"] == "Z"]
plot.fill_ugcs(
df2[column].to_dict(),
bins,
cmap=cmap,
plotmissing=False,
units="inches",
ilabel=ilabel,
)
else:
# use grib data
ts -= datetime.timedelta(hours=(ts.hour % 6))
ts = ts.replace(minute=0)
fn = None
for offset in range(0, 24, 4):
ts2 = ts - datetime.timedelta(hours=offset)
testfn = ts2.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/ffg/"
"5kmffg_%Y%m%d00.grib2"))
if os.path.isfile(testfn):
fn = testfn
break
if fn is None:
raise NoDataFound("No valid grib data found!")
grbs = pygrib.index(fn, "stepRange")
grb = grbs.select(stepRange="0-%s" % (hour, ))[0]
lats, lons = grb.latlons()
data = (masked_array(grb.values,
data_units=units("mm")).to(units("inch")).m)
plot.pcolormesh(lons, lats, data, bins, cmap=cmap)
if ilabel:
plot.drawcounties()
df = pd.DataFrame()
return plot.fig, df
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
csector = ctx['csector']
date = ctx['date']
z = ctx['z']
period = ctx['f']
scale = ctx['scale']
valid = utc(date.year, date.month, date.day, int(z))
gribfn = valid.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/wpc/"
"p" + period + "m_%Y%m%d%Hf" + period + ".grb"))
if not os.path.isfile(gribfn):
raise NoDataFound("gribfn %s missing" % (gribfn, ))
grbs = pygrib.open(gribfn)
grb = grbs[1]
precip = distance(grb.values, 'MM').value('IN')
lats, lons = grb.latlons()
title = ("Weather Prediction Center %s Quantitative "
"Precipitation Forecast") % (PDICT[period])
subtitle = ("%sWPC Forecast %s UTC to %s UTC") % (
("US Drought Monitor Overlaid, " if ctx['opt'] == 'both' else ''),
valid.strftime("%d %b %Y %H"),
(valid +
datetime.timedelta(hours=int(period))).strftime("%d %b %Y %H"))
mp = MapPlot(sector=('state' if len(csector) == 2 else csector),
state=ctx['csector'],
title=title,
subtitle=subtitle,
continentalcolor='white',
titlefontsize=16)
cmap = plt.get_cmap(ctx['cmap'])
cmap.set_under('#EEEEEE')
cmap.set_over('black')
if scale == 'auto':
levs = np.linspace(0, np.max(precip) * 1.1, 10)
levs = [round(lev, 2) for lev in levs]
levs[0] = 0.01
elif scale == '10':
levs = np.arange(0, 10.1, 1.)
levs[0] = 0.01
elif scale == '7':
levs = np.arange(0, 7.1, 0.5)
levs[0] = 0.01
elif scale == '3.5':
levs = np.arange(0, 3.6, 0.25)
levs[0] = 0.01
mp.pcolormesh(lons,
lats,
precip,
levs,
cmap=cmap,
units='inch',
clip_on=(ctx['csector'] == 'iailin'))
if ctx['opt'] == 'both':
mp.draw_usdm(valid=valid, filled=False, hatched=True)
if ctx['csector'] == 'iailin':
mp.drawcounties()
return mp.fig
def plotter(fdict):
""" Go """
pgconn = get_dbconn("coop")
ctx = get_autoplot_context(fdict, get_description())
state = ctx["state"]
varname = ctx["var"]
sector = ctx["sector"]
threshold = ctx["threshold"]
opt = ctx["opt"]
month = ctx["month"]
p1syear = ctx["p1syear"]
p1eyear = ctx["p1eyear"]
p1yearreq = p1eyear - p1syear
p2syear = ctx["p2syear"]
p2eyear = ctx["p2eyear"]
p2yearreq = p2eyear - p2syear
opt1 = ctx["opt1"]
if month == "all":
months = range(1, 13)
elif month == "fall":
months = [9, 10, 11]
elif month == "winter":
months = [12, 1, 2]
elif month == "spring":
months = [3, 4, 5]
elif month == "summer":
months = [6, 7, 8]
elif month == "gs":
months = [5, 6, 7, 8, 9]
else:
ts = datetime.datetime.strptime("2000-" + month + "-01", "%Y-%b-%d")
# make sure it is length two for the trick below in SQL
months = [ts.month]
table = "alldata"
if sector == "state":
# optimization
table = "alldata_%s" % (state, )
df = read_sql(
"""
WITH period1 as (
SELECT station, year, sum(precip) as total_precip,
avg((high+low) / 2.) as avg_temp, avg(high) as avg_high,
avg(low) as avg_low,
sum(gddxx(50, 86, high, low)) as sum_gdd,
sum(case when high > 86 then high - 86 else 0 end) as sum_sdd,
sum(case when high >= %s then 1 else 0 end) as days_high_above
from """ + table + """ WHERE year >= %s and year < %s
and month in %s GROUP by station, year),
period2 as (
SELECT station, year, sum(precip) as total_precip,
avg((high+low) / 2.) as avg_temp, avg(high) as avg_high,
avg(low) as avg_low,
sum(gddxx(50, 86, high, low)) as sum_gdd,
sum(case when high > 86 then high - 86 else 0 end) as sum_sdd,
sum(case when high >= %s then 1 else 0 end) as days_high_above
from """ + table + """ WHERE year >= %s and year < %s
and month in %s GROUP by station, year),
p1agg as (
SELECT station, avg(total_precip) as precip,
avg(avg_temp) as avg_temp, avg(avg_high) as avg_high,
avg(avg_low) as avg_low, avg(sum_sdd) as sdd,
avg(sum_gdd) as gdd,
avg(days_high_above) as avg_days_high_above,
count(*) as count
from period1 GROUP by station),
p2agg as (
SELECT station, avg(total_precip) as precip,
avg(avg_temp) as avg_temp, avg(avg_high) as avg_high,
avg(avg_low) as avg_low, avg(sum_sdd) as sdd,
avg(sum_gdd) as gdd,
avg(days_high_above) as avg_days_high_above,
count(*) as count
from period2 GROUP by station),
agg as (
SELECT p2.station,
p2.precip as p2_total_precip, p1.precip as p1_total_precip,
p2.gdd as p2_gdd, p1.gdd as p1_gdd,
p2.sdd as p2_sdd, p1.sdd as p1_sdd,
p2.avg_temp as p2_avg_temp, p1.avg_temp as p1_avg_temp,
p1.avg_high as p1_avg_high, p2.avg_high as p2_avg_high,
p1.avg_low as p1_avg_low, p2.avg_low as p2_avg_low,
p1.avg_days_high_above as p1_days_high_above,
p2.avg_days_high_above as p2_days_high_above
from p1agg p1 JOIN p2agg p2 on
(p1.station = p2.station)
WHERE p1.count >= %s and p2.count >= %s)
SELECT station, ST_X(geom) as lon, ST_Y(geom) as lat,
d.* from agg d JOIN stations t ON (d.station = t.id)
WHERE t.network ~* 'CLIMATE'
and substr(station, 3, 1) != 'C' and substr(station, 3, 4) != '0000'
""",
pgconn,
params=[
threshold,
p1syear,
p1eyear,
tuple(months),
threshold,
p2syear,
p2eyear,
tuple(months),
p1yearreq,
p2yearreq,
],
index_col=None,
)
if df.empty:
raise NoDataFound("No Data Found.")
df["total_precip"] = df["p2_total_precip"] - df["p1_total_precip"]
df["avg_temp"] = df["p2_avg_temp"] - df["p1_avg_temp"]
df["avg_high"] = df["p2_avg_high"] - df["p1_avg_high"]
df["avg_low"] = df["p2_avg_low"] - df["p1_avg_low"]
df["gdd"] = df["p2_gdd"] - df["p1_gdd"]
df["sdd"] = df["p2_sdd"] - df["p1_sdd"]
df["days_high_above"] = df["p2_days_high_above"] - df["p1_days_high_above"]
column = varname
title = "%s %s" % (MDICT[month], PDICT3[varname])
title = title.replace("[Threshold]", "%.1f" % (threshold, ))
if opt1 == "p1":
column = "p1_%s" % (varname, )
title = "%.0f-%.0f %s" % (p1syear, p1eyear, title)
else:
title = ("%.0f-%.0f minus %.0f-%.0f %s Difference") % (
p2syear,
p2eyear,
p1syear,
p1eyear,
title,
)
# Reindex so that most extreme values are first
df = df.reindex(df[column].abs().sort_values(ascending=False).index)
# drop 5% most extreme events, too much?
df2 = df.iloc[int(len(df.index) * 0.05):]
mp = MapPlot(
sector=sector,
state=state,
axisbg="white",
title=title,
subtitle=("based on IEM Archives"),
titlefontsize=12,
)
if opt1 == "diff":
# Create 9 levels centered on zero
abval = df2[column].abs().max()
levels = centered_bins(abval)
else:
levels = [
round(v, PRECISION[varname])
for v in np.percentile(df2[column].values, range(0, 101, 10))
]
if opt in ["both", "contour"]:
mp.contourf(
df2["lon"].values,
df2["lat"].values,
df2[column].values,
levels,
cmap=plt.get_cmap(ctx["cmap"]),
units=UNITS[varname],
)
if sector == "state":
mp.drawcounties()
if opt in ["both", "values"]:
mp.plot_values(
df2["lon"].values,
df2["lat"].values,
df2[column].values,
fmt="%%.%if" % (PRECISION[varname], ),
labelbuffer=5,
)
return mp.fig, df
def run(basets, endts, view):
"""Generate this plot for the given basets"""
pgconn = get_dbconn('postgis', user='******')
df = read_sql("""SELECT state,
max(magnitude) as val, ST_x(geom) as lon, ST_y(geom) as lat
from lsrs WHERE type in ('S') and magnitude >= 0 and
valid > %s and valid < %s GROUP by state, lon, lat
""",
pgconn,
params=(basets, endts),
index_col=None)
df.sort_values(by='val', ascending=False, inplace=True)
df['useme'] = False
# First, we need to filter down the in-bound values to get rid of small
cellsize = 0.33
newrows = []
for lat in np.arange(reference.MW_SOUTH, reference.MW_NORTH, cellsize):
for lon in np.arange(reference.MW_WEST, reference.MW_EAST, cellsize):
# Look around this box at 1x
df2 = df[(df['lat'] >= (lat - cellsize))
& (df['lat'] < (lat + cellsize)) &
(df['lon'] >= (lon - cellsize)) & (df['lon'] <
(lon + cellsize))]
if df2.empty:
# If nothing was found, check 2x
df3 = df[(df['lat'] >= (lat - cellsize * 2.))
& (df['lat'] < (lat + cellsize * 2.)) &
(df['lon'] >=
(lon - cellsize * 2.)) & (df['lon'] <
(lon + cellsize * 2.))]
if df3.empty:
# If nothing found, place a zero here
newrows.append({
'lon': lon,
'lat': lat,
'val': 0,
'useme': True,
'state': 'NA'
})
continue
maxval = df.at[df2.index[0], 'val']
df.loc[df2[df2['val'] > (maxval * 0.8)].index, 'useme'] = True
dfall = pd.concat([df, pd.DataFrame(newrows)], ignore_index=True)
df2 = dfall[dfall['useme']]
xi = np.arange(reference.MW_WEST, reference.MW_EAST, cellsize)
yi = np.arange(reference.MW_SOUTH, reference.MW_NORTH, cellsize)
xi, yi = np.meshgrid(xi, yi)
gridder = Rbf(df2['lon'].values,
df2['lat'].values,
pd.to_numeric(df2['val'].values, errors='ignore'),
function='thin_plate')
vals = gridder(xi, yi)
vals[np.isnan(vals)] = 0
rng = [0.01, 1, 2, 3, 4, 6, 8, 12, 18, 24, 30, 36]
cmap = nwssnow()
mp = MapPlot(sector='iowa',
axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
if df['val'].max() > 0:
mp.contourf(xi, yi, vals, rng, cmap=cmap)
mp.drawcounties()
if not df.empty:
mp.plot_values(df['lon'].values,
df['lat'].values,
df['val'].values,
fmt='%.1f',
labelbuffer=2)
mp.drawcities()
pqstr = "plot c 000000000000 lsr_snowfall.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
# slightly different title to help uniqueness
mp = MapPlot(sector='iowa',
axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports valid over past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
if df['val'].max() > 0:
mp.contourf(xi, yi, vals, rng, cmap=cmap)
mp.drawcounties()
mp.drawcities()
pqstr = "plot c 000000000000 lsr_snowfall_nv.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
mp = MapPlot(sector='midwest',
axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
if df['val'].max() > 0:
mp.contourf(xi, yi, vals, rng, cmap=cmap)
mp.drawcities()
pqstr = "plot c 000000000000 mw_lsr_snowfall.png bogus png"
mp.postprocess(view=view, pqstr=pqstr)
mp.close()
def doit(now):
"""
Generate some plots for the COOP data!
"""
st = NetworkTable([
"IA_COOP",
"MO_COOP",
"KS_COOP",
"NE_COOP",
"SD_COOP",
"ND_ASOS",
"MN_COOP",
"WI_COOP",
"IL_COOP",
"IN_COOP",
"OH_COOP",
"MI_COOP",
])
pgconn = get_dbconn("iem", user="******")
icursor = pgconn.cursor()
clevs = [0, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1, 2, 3, 4, 5, 10]
# We'll assume all COOP data is 12z, sigh for now
sql = """SELECT id, pday, network
from summary_%s s JOIN stations t ON (t.iemid = s.iemid)
WHERE day = '%s' and
t.network ~* 'COOP' and pday >= 0""" % (
now.year,
now.strftime("%Y-%m-%d"),
)
lats = []
lons = []
vals = []
icursor.execute(sql)
iamax = 0.0
for row in icursor:
sid = row[0]
if sid not in st.sts:
continue
# labels.append( id[2:] )
lats.append(st.sts[sid]["lat"])
lons.append(st.sts[sid]["lon"])
vals.append(row[1])
if row[2] == "IA_COOP" and row[1] > iamax:
iamax = row[1]
# Plot Iowa
mp = MapPlot(
sector="iowa",
title="24 Hour NWS COOP Precipitation [inch]",
subtitle=("Ending %s at roughly 12Z") % (now.strftime("%d %B %Y"), ),
)
mp.contourf(lons, lats, vals, clevs, units="inch")
pqstr = ("plot ac %s0000 iowa_coop_12z_precip.png "
"iowa_coop_12z_precip.png png") % (now.strftime("%Y%m%d"), )
mp.drawcounties()
mp.postprocess(pqstr=pqstr)
mp.close()
mp = MapPlot(
sector="midwest",
title="24 Hour NWS COOP Precipitation [inch]",
subtitle=("Ending %s at roughly 12Z") % (now.strftime("%d %B %Y"), ),
)
mp.contourf(lons, lats, vals, clevs, units="inch")
pqstr = ("plot ac %s0000 midwest_coop_12z_precip.png "
"midwest_coop_12z_precip.png png") % (now.strftime("%Y%m%d"), )
mp.postprocess(pqstr=pqstr)
mp.close()
def plotter(fdict):
""" Go """
import matplotlib
matplotlib.use('agg')
from pyiem.plot import MapPlot, nwsprecip
ctx = util.get_autoplot_context(fdict, get_description())
ptype = ctx['ptype']
sdate = ctx['sdate']
edate = ctx['edate']
src = ctx['src']
if sdate.year != edate.year:
raise Exception('Sorry, do not support multi-year plots yet!')
days = (edate - sdate).days
sector = ctx['sector']
idx0 = iemre.daily_offset(sdate)
idx1 = iemre.daily_offset(edate) + 1
if src == 'mrms':
ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (sdate.year, )
ncvar = 'p01d'
source = 'NOAA MRMS Q3'
subtitle = 'NOAA MRMS Project, GaugeCorr and RadarOnly'
else:
ncfn = "/mesonet/data/prism/%s_daily.nc" % (sdate.year, )
ncvar = 'ppt'
source = 'OSU PRISM'
subtitle = ('PRISM Climate Group, Oregon State Univ., '
'http://prism.oregonstate.edu, created 4 Feb 2004.')
if not os.path.isfile(ncfn):
raise Exception("No data for that year, sorry.")
nc = netCDF4.Dataset(ncfn, 'r')
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
if (idx1 - idx0) < 32:
p01d = distance(np.sum(nc.variables[ncvar][idx0:idx1, :, :], 0),
'MM').value('IN')
else:
# Too much data can overwhelm this app, need to chunk it
for i in range(idx0, idx1, 10):
i2 = min([i + 10, idx1])
if idx0 == i:
p01d = distance(np.sum(nc.variables[ncvar][i:i2, :, :], 0),
'MM').value('IN')
else:
p01d += distance(np.sum(nc.variables[ncvar][i:i2, :, :], 0),
'MM').value('IN')
nc.close()
if sdate == edate:
title = sdate.strftime("%-d %B %Y")
else:
title = "%s to %s (inclusive)" % (sdate.strftime("%-d %b %Y"),
edate.strftime("%-d %b %Y"))
if sector == 'midwest':
state = None
else:
state = sector
sector = 'state'
m = MapPlot(sector=sector,
state=state,
axisbg='white',
nocaption=True,
title='%s:: %s Total Precip' % (source, title),
subtitle='Data from %s' % (subtitle, ))
if np.ma.is_masked(np.max(p01d)):
return 'Data Unavailable'
clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
if days > 6:
clevs = [0.01, 0.3, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 10, 15, 20]
if days > 29:
clevs = [0.01, 0.5, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35]
if days > 90:
clevs = [0.01, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35, 40]
x, y = np.meshgrid(lons, lats)
cmap = nwsprecip()
cmap.set_over('k')
if ptype == 'c':
m.contourf(x, y, p01d, clevs, cmap=cmap, label='inches')
else:
m.pcolormesh(x, y, p01d, clevs, cmap=cmap, label='inches')
if sector != 'midwest':
m.drawcounties()
m.drawcities()
return m.fig
def main(argv):
"""Go Main Go"""
nt = Table("ISUSM")
qdict = loadqc()
idbconn = get_dbconn("isuag", user="******")
pdbconn = get_dbconn("postgis", user="******")
day_ago = int(argv[1])
ts = datetime.date.today() - datetime.timedelta(days=day_ago)
hlons, hlats, hvals = do_nam(ts)
nam = temperature(hvals, "K").value("F")
window = np.ones((3, 3))
nam = convolve2d(nam, window / window.sum(), mode="same", boundary="symm")
# mp = MapPlot(sector='midwest')
# mp.pcolormesh(hlons, hlats, nam,
# range(20, 90, 5))
# mp.postprocess(filename='test.png')
# sys.exit()
# Query out the data
df = read_sql(
"""
WITH ranges as (
select station, count(*), min(tsoil_c_avg_qc),
max(tsoil_c_avg_qc) from sm_hourly WHERE
valid >= %s and valid < %s and tsoil_c_avg_qc > -40
and tsoil_c_avg_qc < 50 GROUP by station
)
SELECT d.station, d.tsoil_c_avg_qc,
r.max as hourly_max_c, r.min as hourly_min_c, r.count
from sm_daily d JOIN ranges r on (d.station = r.station)
where valid = %s and tsoil_c_avg_qc > -40 and r.count > 19
""",
idbconn,
params=(ts, ts + datetime.timedelta(days=1), ts),
index_col="station",
)
for col, newcol in zip(
["tsoil_c_avg_qc", "hourly_min_c", "hourly_max_c"],
["ob", "min", "max"],
):
df[newcol] = temperature(df[col].values, "C").value("F")
df.drop(col, axis=1, inplace=True)
for stid, row in df.iterrows():
df.at[stid, "ticket"] = qdict.get(stid, {}).get("soil4", False)
x, y = get_idx(hlons, hlats, nt.sts[stid]["lon"], nt.sts[stid]["lat"])
df.at[stid, "nam"] = nam[x, y]
df.at[stid, "lat"] = nt.sts[stid]["lat"]
df.at[stid, "lon"] = nt.sts[stid]["lon"]
# ticket is an object type from above
df = df[~df["ticket"].astype("bool")]
df["diff"] = df["ob"] - df["nam"]
bias = df["diff"].mean()
nam = nam + bias
print("fancy_4inch NAM bias correction of: %.2fF applied" % (bias, ))
# apply nam bias to sampled data
df["nam"] += bias
df["diff"] = df["ob"] - df["nam"]
# we are going to require data be within 1 SD of sampled or 5 deg
std = 5.0 if df["nam"].std() < 5.0 else df["nam"].std()
for station in df[df["diff"].abs() > std].index.values:
print(("fancy_4inch %s QC'd %s out std: %.2f, ob:%.1f nam:%.1f") % (
ts.strftime("%Y%m%d"),
station,
std,
df.at[station, "ob"],
df.at[station, "nam"],
))
df.drop(station, inplace=True)
# Query out centroids of counties...
cdf = read_sql(
"""SELECT ST_x(ST_centroid(the_geom)) as lon,
ST_y(ST_centroid(the_geom)) as lat
from uscounties WHERE state_name = 'Iowa'
""",
pdbconn,
index_col=None,
)
for i, row in cdf.iterrows():
x, y = get_idx(hlons, hlats, row["lon"], row["lat"])
cdf.at[i, "nam"] = nam[x, y]
mp = MapPlot(
sector="iowa",
title=("Average 4 inch Depth Soil Temperatures for %s") %
(ts.strftime("%b %d, %Y"), ),
subtitle=("County est. based on bias adj. "
"NWS NAM Model (black numbers), "
"ISUSM network observations (red numbers)"),
)
mp.pcolormesh(
hlons,
hlats,
nam,
np.arange(10, 101, 5),
cmap=cm.get_cmap("jet"),
units=r"$^\circ$F",
)
mp.plot_values(df["lon"],
df["lat"],
df["ob"],
fmt="%.0f",
color="r",
labelbuffer=5)
mp.plot_values(
cdf["lon"],
cdf["lat"],
cdf["nam"],
fmt="%.0f",
textsize=11,
labelbuffer=5,
)
mp.drawcounties()
routes = "a" if day_ago >= 4 else "ac"
pqstr = ("plot %s %s0000 soilt_day%s.png isuag_county_4inch_soil.png png"
) % (routes, ts.strftime("%Y%m%d"), day_ago)
mp.postprocess(pqstr=pqstr)
mp.close()
def plotter(fdict):
""" Go """
pgconn = get_dbconn('coop')
ctx = get_autoplot_context(fdict, get_description())
state = ctx['state']
varname = ctx['var']
sector = ctx['sector']
threshold = ctx['threshold']
opt = ctx['opt']
month = ctx['month']
p1syear = ctx['p1syear']
p1eyear = ctx['p1eyear']
p1yearreq = (p1eyear - p1syear)
p2syear = ctx['p2syear']
p2eyear = ctx['p2eyear']
p2yearreq = (p2eyear - p2syear)
opt1 = ctx['opt1']
if month == 'all':
months = range(1, 13)
elif month == 'fall':
months = [9, 10, 11]
elif month == 'winter':
months = [12, 1, 2]
elif month == 'spring':
months = [3, 4, 5]
elif month == 'summer':
months = [6, 7, 8]
elif month == 'gs':
months = [5, 6, 7, 8, 9]
else:
ts = datetime.datetime.strptime("2000-"+month+"-01", '%Y-%b-%d')
# make sure it is length two for the trick below in SQL
months = [ts.month]
table = "alldata"
if sector == 'state':
# optimization
table = "alldata_%s" % (state,)
df = read_sql("""
WITH period1 as (
SELECT station, year, sum(precip) as total_precip,
avg((high+low) / 2.) as avg_temp, avg(high) as avg_high,
avg(low) as avg_low,
sum(gddxx(50, 86, high, low)) as sum_gdd,
sum(case when high > 86 then high - 86 else 0 end) as sum_sdd,
sum(case when high >= %s then 1 else 0 end) as days_high_above
from """ + table + """ WHERE year >= %s and year < %s
and month in %s GROUP by station, year),
period2 as (
SELECT station, year, sum(precip) as total_precip,
avg((high+low) / 2.) as avg_temp, avg(high) as avg_high,
avg(low) as avg_low,
sum(gddxx(50, 86, high, low)) as sum_gdd,
sum(case when high > 86 then high - 86 else 0 end) as sum_sdd,
sum(case when high >= %s then 1 else 0 end) as days_high_above
from """ + table + """ WHERE year >= %s and year < %s
and month in %s GROUP by station, year),
p1agg as (
SELECT station, avg(total_precip) as precip,
avg(avg_temp) as avg_temp, avg(avg_high) as avg_high,
avg(avg_low) as avg_low, avg(sum_sdd) as sdd,
avg(sum_gdd) as gdd,
avg(days_high_above) as avg_days_high_above,
count(*) as count
from period1 GROUP by station),
p2agg as (
SELECT station, avg(total_precip) as precip,
avg(avg_temp) as avg_temp, avg(avg_high) as avg_high,
avg(avg_low) as avg_low, avg(sum_sdd) as sdd,
avg(sum_gdd) as gdd,
avg(days_high_above) as avg_days_high_above,
count(*) as count
from period2 GROUP by station),
agg as (
SELECT p2.station,
p2.precip as p2_total_precip, p1.precip as p1_total_precip,
p2.gdd as p2_gdd, p1.gdd as p1_gdd,
p2.sdd as p2_sdd, p1.sdd as p1_sdd,
p2.avg_temp as p2_avg_temp, p1.avg_temp as p1_avg_temp,
p1.avg_high as p1_avg_high, p2.avg_high as p2_avg_high,
p1.avg_low as p1_avg_low, p2.avg_low as p2_avg_low,
p1.avg_days_high_above as p1_days_high_above,
p2.avg_days_high_above as p2_days_high_above
from p1agg p1 JOIN p2agg p2 on
(p1.station = p2.station)
WHERE p1.count >= %s and p2.count >= %s)
SELECT station, ST_X(geom) as lon, ST_Y(geom) as lat,
d.* from agg d JOIN stations t ON (d.station = t.id)
WHERE t.network ~* 'CLIMATE'
and substr(station, 3, 1) != 'C' and substr(station, 3, 4) != '0000'
""", pgconn, params=[threshold, p1syear, p1eyear, tuple(months),
threshold, p2syear, p2eyear, tuple(months),
p1yearreq, p2yearreq],
index_col=None)
df['total_precip'] = df['p2_total_precip'] - df['p1_total_precip']
df['avg_temp'] = df['p2_avg_temp'] - df['p1_avg_temp']
df['avg_high'] = df['p2_avg_high'] - df['p1_avg_high']
df['avg_low'] = df['p2_avg_low'] - df['p1_avg_low']
df['gdd'] = df['p2_gdd'] - df['p1_gdd']
df['sdd'] = df['p2_sdd'] - df['p1_sdd']
df['days_high_above'] = (df['p2_days_high_above'] -
df['p1_days_high_above'])
column = varname
title = "%s %s" % (MDICT[month], PDICT3[varname])
title = title.replace("[Threshold]", '%.1f' % (threshold,))
if opt1 == 'p1':
column = 'p1_%s' % (varname,)
title = '%.0f-%.0f %s' % (p1syear, p1eyear, title)
else:
title = ('%.0f-%.0f minus %.0f-%.0f %s Difference'
) % (p2syear, p2eyear, p1syear, p1eyear, title)
# Reindex so that most extreme values are first
df = df.reindex(df[column].abs().sort_values(ascending=False).index)
# drop 5% most extreme events, too much?
df2 = df.iloc[int(len(df.index) * 0.05):]
mp = MapPlot(sector=sector, state=state, axisbg='white', title=title,
subtitle=('based on IEM Archives'),
titlefontsize=12)
if opt1 == 'diff':
# Create 9 levels centered on zero
abval = df2[column].abs().max()
levels = centered_bins(abval)
else:
levels = [round(v, PRECISION[varname])
for v in np.percentile(df2[column].values, range(0, 101,
10))]
if opt in ['both', 'contour']:
mp.contourf(df2['lon'].values, df2['lat'].values,
df2[column].values, levels,
cmap=plt.get_cmap(('seismic_r' if varname == 'total_precip'
else 'seismic')),
units=UNITS[varname])
if sector == 'state':
mp.drawcounties()
if opt in ['both', 'values']:
mp.plot_values(df2['lon'].values, df2['lat'].values,
df2[column].values,
fmt='%%.%if' % (PRECISION[varname],), labelbuffer=5)
return mp.fig, df
def main():
"""Go Main Go"""
now = datetime.datetime.now()
qdict = loadqc()
pgconn = get_dbconn("iem", user="******")
icursor = pgconn.cursor()
sql = """
select s.id,
ST_x(s.geom) as lon, ST_y(s.geom) as lat,
max_tmpf as high, s.network
from summary c, stations s
WHERE c.iemid = s.iemid and day = 'TODAY' and max_tmpf > -40
and s.network in ('IA_ASOS', 'AWOS', 'IL_ASOS','MO_ASOS','KS_ASOS',
'NE_ASOS','SD_ASOS','MN_ASOS','WI_ASOS') ORDER by high ASC
"""
lats = []
lons = []
vals = []
valmask = []
labels = []
icursor.execute(sql)
dsm = None
for row in icursor:
if row[0] == "DSM":
dsm = row[3]
if qdict.get(row[0], {}).get("tmpf") is not None:
continue
lats.append(row[2])
lons.append(row[1])
vals.append(row[3])
labels.append(row[0])
valmask.append(row[4] in ["AWOS", "IA_ASOS"])
if len(lats) < 4:
return
mp = MapPlot(
sector="iowa",
title=("%s Iowa ASOS/AWOS High Temperature")
% (now.strftime("%-d %b %Y"),),
subtitle="map valid: %s" % (now.strftime("%d %b %Y %-I:%M %p"),),
)
# m.debug = True
if dsm is None:
dsm = vals[0]
bottom = int(dsm) - 15
top = int(dsm) + 15
bins = np.linspace(bottom, top, 11)
cmap = cm.get_cmap("jet")
mp.contourf(lons, lats, vals, bins, units="F", cmap=cmap)
mp.plot_values(
lons,
lats,
vals,
"%.0f",
valmask=valmask,
labels=labels,
labelbuffer=10,
)
mp.drawcounties()
pqstr = "plot ac %s summary/iowa_asos_high.png iowa_asos_high.png png" % (
now.strftime("%Y%m%d%H%M"),
)
mp.postprocess(pqstr=pqstr)
mp.close()
def plotter(fdict):
""" Go """
import matplotlib
matplotlib.use('agg')
from pyiem.plot import MapPlot
import matplotlib.cm as cm
pgconn = get_dbconn('coop')
ctx = get_autoplot_context(fdict, get_description())
sector = ctx['sector']
date1 = ctx['date1']
date2 = ctx['date2']
varname = ctx['var']
usdm = ctx['usdm']
table = "alldata_%s" % (sector, ) if sector != 'midwest' else "alldata"
df = read_sql("""
WITH obs as (
SELECT station, gddxx(50, 86, high, low) as gdd50,
sday, high, low, precip from """ + table + """ WHERE
day >= %s and day < %s and
substr(station, 3, 1) != 'C' and substr(station, 3, 4) != '0000'),
climo as (
SELECT station, to_char(valid, 'mmdd') as sday, precip, high, low,
gdd50 from climate51),
combo as (
SELECT o.station, o.precip - c.precip as precip_diff,
o.precip as precip, c.precip as cprecip,
o.high, o.low, o.gdd50, c.gdd50 as cgdd50,
o.gdd50 - c.gdd50 as gdd50_diff,
(o.high + o.low)/2. - (c.high + c.low)/2. as temp_diff
from obs o JOIN climo c ON
(o.station = c.station and o.sday = c.sday)),
agg as (
SELECT station, sum(precip_diff) as precip_depart,
sum(precip) as precip, sum(cprecip) as cprecip,
min(low) as min_low_temp, sum(gdd50_diff) as gdd_depart,
avg(temp_diff) as avg_temp_depart, sum(gdd50) as gdd_sum,
sum(cgdd50) as cgdd_sum
from combo GROUP by station)
SELECT d.station, t.name,
d.precip as precip_sum, d.cprecip as cprecip_sum,
d.precip_depart, d.min_low_temp, d.avg_temp_depart,
d.gdd_depart, d.gdd_sum, d.cgdd_sum,
ST_x(t.geom) as lon, ST_y(t.geom) as lat
from agg d JOIN stations t on (d.station = t.id)
WHERE t.network ~* 'CLIMATE'
""",
pgconn,
params=(date1, date2),
index_col='station')
df = df.reindex(df[varname].abs().sort_values(ascending=False).index)
sector2 = "state" if sector != 'midwest' else 'midwest'
datefmt = "%d %b %Y" if varname != 'cgdd_sum' else '%d %b'
subtitle = ("Climatology is based on data from 1951-%s") % (
datetime.date.today().year - 1, )
if varname != 'cgdd_sum':
subtitle = ('%s is compared with 1951-%s Climatology'
' to compute departures') % (
date1.year, datetime.date.today().year - 1)
mp = MapPlot(sector=sector2,
state=sector,
axisbg='white',
title=('%s - %s %s [%s]') %
(date1.strftime(datefmt), date2.strftime(datefmt),
PDICT2.get(varname), UNITS.get(varname)),
subtitle=subtitle)
if varname in ['precip_depart', 'avg_temp_depart']:
rng = df[varname].abs().describe(percentiles=[0.95])['95%']
clevels = np.linspace(0 - rng, rng, 7)
fmt = '%.2f'
else:
minv = df[varname].min() - 5
maxv = df[varname].max() + 5
clevels = np.linspace(minv, maxv, 6, dtype='i')
fmt = '%.0f'
clevlabels = [fmt % x for x in clevels]
cmap = cm.get_cmap('RdYlBu' if varname == 'precip_depart' else 'RdYlBu_r')
cmap.set_bad('white')
mp.contourf(df['lon'].values,
df['lat'].values,
df[varname].values,
clevels,
clevlabels=clevlabels,
cmap=cmap,
units=UNITS.get(varname))
mp.plot_values(df['lon'].values,
df['lat'].values,
df[varname].values,
fmt=fmt,
labelbuffer=10)
if sector == 'IA':
mp.drawcounties()
if usdm == 'yes':
mp.draw_usdm(date2, filled=False, hatched=True)
return mp.fig, df
def plotter(fdict):
""" Go """
pgconn = get_dbconn("coop")
ctx = get_autoplot_context(fdict, get_description())
state = ctx["state"][:2]
sector = ctx["sector"]
opt = ctx["opt"]
p1syear = ctx["p1syear"]
p1eyear = ctx["p1eyear"]
p2syear = ctx["p2syear"]
p2eyear = ctx["p2eyear"]
varname = ctx["var"]
table = "alldata"
if sector == "state":
table = "alldata_%s" % (state,)
df = read_sql(
"""
WITH season1 as (
SELECT station, year,
min(case when month > 7 and low < 32 then
extract(doy from day) else 366 end) as first_freeze,
max(case when month < 7 and low < 32 then
extract(doy from day) else 0 end) as last_freeze
from """
+ table
+ """ WHERE
year >= %s and year <= %s GROUP by station, year),
season2 as (
SELECT station, year,
min(case when month > 7 and low < 32 then
extract(doy from day) else 366 end) as first_freeze,
max(case when month < 7 and low < 32 then
extract(doy from day) else 0 end) as last_freeze
from """
+ table
+ """ WHERE
year >= %s and year <= %s GROUP by station, year),
agg as (
SELECT p1.station, avg(p1.first_freeze) as p1_first_fall,
avg(p1.last_freeze) as p1_last_spring,
avg(p2.first_freeze) as p2_first_fall,
avg(p2.last_freeze) as p2_last_spring
from season1 as p1 JOIN season2 as p2 on (p1.station = p2.station)
GROUP by p1.station)
SELECT station, ST_X(geom) as lon, ST_Y(geom) as lat,
d.* from agg d JOIN stations t ON (d.station = t.id)
WHERE t.network ~* 'CLIMATE'
and substr(station, 3, 1) != 'C' and substr(station, 3, 4) != '0000'
""",
pgconn,
params=[p1syear, p1eyear, p2syear, p2eyear],
index_col="station",
)
if df.empty:
raise NoDataFound("No Data Found")
df["p1_season"] = df["p1_first_fall"] - df["p1_last_spring"]
df["p2_season"] = df["p2_first_fall"] - df["p2_last_spring"]
df["season_delta"] = df["p2_season"] - df["p1_season"]
df["spring_delta"] = df["p2_last_spring"] - df["p1_last_spring"]
df["fall_delta"] = df["p2_first_fall"] - df["p1_first_fall"]
# Reindex so that most extreme values are first
df = df.reindex(
df[varname + "_delta"].abs().sort_values(ascending=False).index
)
title = PDICT3[varname]
mp = MapPlot(
sector=sector,
state=state,
axisbg="white",
title=("%.0f-%.0f minus %.0f-%.0f %s Difference")
% (p2syear, p2eyear, p1syear, p1eyear, title),
subtitle=("based on IEM Archives"),
titlefontsize=14,
)
# Create 9 levels centered on zero
abval = df[varname + "_delta"].abs().max()
levels = centered_bins(abval)
if opt in ["both", "contour"]:
mp.contourf(
df["lon"].values,
df["lat"].values,
df[varname + "_delta"].values,
levels,
cmap=get_cmap(ctx["cmap"]),
units="days",
)
if sector == "state":
mp.drawcounties()
if opt in ["both", "values"]:
mp.plot_values(
df["lon"].values,
df["lat"].values,
df[varname + "_delta"].values,
fmt="%.1f",
labelbuffer=5,
)
return mp.fig, df
def main(argv):
"""Go Main Go"""
v = argv[1]
agg = argv[2]
ts = datetime.date(2008, 1, 1)
ts2 = datetime.date(2017, 12, 31)
scenario = 0
# suggested for runoff and precip
if V2UNITS[v] in ['mm', 'inches']:
colors = ['#ffffa6', '#9cf26d', '#76cc94', '#6399ba', '#5558a1']
# suggested for detachment
elif v in ['avg_loss', 'avg_loss_metric']:
colors = ['#cbe3bb', '#c4ff4d', '#ffff4d', '#ffc44d', '#ff4d4d',
'#c34dee']
# suggested for delivery
elif v in ['avg_delivery', 'avg_delivery_metric']:
colors = ['#ffffd2', '#ffff4d', '#ffe0a5', '#eeb74d', '#ba7c57',
'#96504d']
cmap = mpcolors.ListedColormap(colors, 'james')
cmap.set_under('white')
cmap.set_over('black')
pgconn = get_dbconn('idep')
title = "for %s" % (ts.strftime("%-d %B %Y"),)
if ts != ts2:
title = "between %s and %s" % (ts.strftime("%-d %b %Y"),
ts2.strftime("%-d %b %Y"))
mp = MapPlot(axisbg='#EEEEEE', nologo=True, sector='iowa',
nocaption=True,
title=("DEP %s %s %s"
) % (V2NAME[v.replace("_metric", "")],
"Yearly Average" if agg == 'avg' else 'Total',
title),
caption='Daily Erosion Project')
df = read_postgis("""
WITH data as (
SELECT huc_12, extract(year from valid) as yr,
sum(""" + v.replace("_metric", "") + """) as d from results_by_huc12
WHERE scenario = %s and valid >= %s and valid <= %s
GROUP by huc_12, yr),
agg as (
SELECT huc_12, """ + agg + """(d) as d from data GROUP by huc_12)
SELECT ST_Transform(simple_geom, 4326) as geo, coalesce(d.d, 0) as data
from huc12 i LEFT JOIN agg d
ON (i.huc_12 = d.huc_12) WHERE i.scenario = %s and i.states ~* 'IA'
""", pgconn, params=(scenario, ts, ts2, scenario), geom_col='geo',
index_col=None)
df['data'] = df['data'] * V2MULTI[v]
if df['data'].max() < 0.01:
bins = [0.01, 0.02, 0.03, 0.04, 0.05]
else:
bins = np.array(V2RAMP[v]) * (10. if agg == 'sum' else 1.)
norm = mpcolors.BoundaryNorm(bins, cmap.N)
# m.ax.add_geometries(df['geo'], ccrs.PlateCarree())
for _, row in df.iterrows():
c = cmap(norm([row['data'], ]))[0]
arr = np.asarray(row['geo'].exterior)
points = mp.ax.projection.transform_points(ccrs.Geodetic(),
arr[:, 0], arr[:, 1])
p = Polygon(points[:, :2], fc=c, ec='k', zorder=2, lw=0.1)
mp.ax.add_patch(p)
mp.drawcounties()
mp.drawcities()
lbl = [round(_, 2) for _ in bins]
u = "%s, Avg: %.2f" % (V2UNITS[v], df['data'].mean())
mp.draw_colorbar(bins, cmap, norm,
clevlabels=lbl, units=u,
title="%s :: %s" % (V2NAME[v], V2UNITS[v]))
plt.savefig('%s_%s_%s%s.png' % (ts.year, ts2.year, v,
"_sum" if agg == 'sum' else ''))
def do(valid):
"""Do Something"""
clevs = np.arange(0, 0.25, 0.05)
clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs[0] = 0.01
precip = load_precip(valid)
stage4 = load_precip(valid, 'stage4')
inqcprecip = load_precip(valid, 'inqcprecip')
outqcprecip = load_precip(valid, 'outqcprecip')
multiplier = load_precip(valid, 'multiplier')
yidx = int((43.27 - SOUTH) / 0.01)
xidx = int((-94.39 - WEST) / 0.01)
print(("yidx:%s xidx:%s precip:%.2f stage4: %.2f inqc: %.2f outqc: %.2f "
"mul: %.2f"
) % (yidx, xidx, precip[yidx, xidx], stage4[yidx, xidx],
inqcprecip[yidx, xidx], outqcprecip[yidx, xidx],
multiplier[yidx, xidx]))
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s DEP Quality Controlled Precip Totals'
) % (valid.strftime("%-d %b %Y"), ))
(lons, lats) = np.meshgrid(XS, YS)
mp.pcolormesh(lons, lats, precip / 25.4, clevs,
units='inch')
mp.drawcounties()
mp.postprocess(filename='qc.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s Stage IV Precip Totals'
) % (valid.strftime("%-d %b %Y"), ))
mp.pcolormesh(lons, lats, stage4 / 25.4, clevs,
units='inch')
mp.drawcounties()
mp.postprocess(filename='stageIV.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s High-res total prior to QC'
) % (valid.strftime("%-d %b %Y"), ))
mp.pcolormesh(lons, lats, inqcprecip / 25.4, clevs,
units='inch')
mp.drawcounties()
mp.postprocess(filename='inqcprecip.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s High-res total after QC'
) % (valid.strftime("%-d %b %Y"), ))
mp.pcolormesh(lons, lats, outqcprecip / 25.4, clevs,
units='inch')
mp.drawcounties()
mp.postprocess(filename='outqcprecip.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s QC Change in Precip Out - In'
) % (valid.strftime("%-d %b %Y"), ))
diff = (outqcprecip - inqcprecip) / 25.4
mp.pcolormesh(lons, lats, diff,
np.arange(-1.4, 1.5, 0.1),
cmap=plt.get_cmap('BrBG'),
units='inch')
mp.drawcounties()
mp.postprocess(filename='qcprecipdiff.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s multiplier'
) % (valid.strftime("%-d %b %Y"), ))
mp.pcolormesh(lons, lats, multiplier,
np.arange(0.0, 2.5, 0.2),
cmap=plt.get_cmap('jet'),
units='ratio')
mp.drawcounties()
mp.postprocess(filename='multiplier.png')
mp.close()
mp = MapPlot(sector=SECTOR, axisbg='white',
title=('%s manually computed QC Precip mul * stage4'
) % (valid.strftime("%-d %b %Y"), ))
mp.pcolormesh(lons, lats, multiplier * stage4 / 25.4,
clevs,
units='inch')
mp.drawcounties()
mp.postprocess(filename='qcmancalc.png')
mp.close()
# Go MRMS
ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (valid.year, )
if not os.path.isfile(ncfn):
return
nc = netCDF4.Dataset(ncfn, 'r')
xidx = int((WEST - nc.variables['lon'][0]) * 100.)
yidx = int((SOUTH - nc.variables['lat'][0]) * 100.)
idx = daily_offset(valid)
mrms = nc.variables['p01d'][idx, yidx:(yidx+800), xidx:(xidx+921)]
nc.close()
def run(basets, endts, view):
"""Generate this plot for the given basets"""
df = read_sql("""SELECT state,
max(magnitude) as val, ST_x(geom) as lon, ST_y(geom) as lat
from lsrs WHERE type in ('S') and magnitude >= 0 and
valid > %s and valid < %s GROUP by state, lon, lat
""",
POSTGIS,
params=(basets, endts),
index_col=None)
df['used'] = False
df['textplot'] = True
df.sort_values(by='val', ascending=False, inplace=True)
# Now, we need to add in zeros, lets say we are looking at a .25 degree box
mybuffer = 0.75
newrows = []
for lat in np.arange(reference.MW_SOUTH, reference.MW_NORTH, mybuffer):
for lon in np.arange(reference.MW_WEST, reference.MW_EAST, mybuffer):
df2 = df[(df['lat'] >= lat) & (df['lat'] < (lat + mybuffer)) &
(df['lon'] >= lon) & (df['lon'] < (lon + mybuffer))]
if len(df2.index) == 0:
newrows.append(
dict(lon=(lon + mybuffer / 2.),
lat=(lat + mybuffer / 2.),
val=0,
used=True,
textplot=False))
continue
maxval = df.at[df2.index[0], 'val']
df.loc[df2[df2['val'] > (maxval * 0.5)].index, 'used'] = True
df.loc[df2[df2['val'] < (maxval * 0.5)].index, 'textplot'] = False
dfnew = pd.DataFrame(newrows)
df = df.append(dfnew)
cdf = df[df['used']]
tdf = df[df['textplot']]
rng = [0.01, 1, 2, 3, 4, 6, 8, 12, 18, 24, 30, 36]
cmap = nwssnow()
m = MapPlot(sector='iowa',
axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
m.contourf(cdf['lon'].values,
cdf['lat'].values,
cdf['val'].values,
rng,
cmap=cmap)
m.drawcounties()
m.plot_values(tdf['lon'].values,
tdf['lat'].values,
tdf['val'].values,
fmt='%.1f')
m.drawcities()
pqstr = "plot c 000000000000 lsr_snowfall.png bogus png"
m.postprocess(view=view, pqstr=pqstr)
m.close()
# slightly different title to help uniqueness
m = MapPlot(sector='iowa',
axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports valid over past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
m.contourf(cdf['lon'].values,
cdf['lat'].values,
cdf['val'].values,
rng,
cmap=cmap)
m.drawcounties()
m.drawcities()
pqstr = "plot c 000000000000 lsr_snowfall_nv.png bogus png"
m.postprocess(view=view, pqstr=pqstr)
m.close()
m = MapPlot(sector='midwest',
axisbg='white',
title="Local Storm Report Snowfall Total Analysis",
subtitle=("Reports past 12 hours: %s"
"" % (endts.strftime("%d %b %Y %I:%M %p"), )))
m.contourf(cdf['lon'].values,
cdf['lat'].values,
cdf['val'].values,
rng,
cmap=cmap)
m.drawcities()
pqstr = "plot c 000000000000 mw_lsr_snowfall.png bogus png"
m.postprocess(view=view, pqstr=pqstr)
m.close()
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
varname = ctx['v']
if ctx['t'] == 'state':
bnds = reference.state_bounds[ctx['state']]
title = reference.state_names[ctx['state']]
else:
bnds = reference.wfo_bounds[ctx['wfo']]
title = "NWS CWA %s [%s]" % (ctx['_nt'].sts[ctx['wfo']]['name'],
ctx['wfo'])
df, valid = get_df(ctx, bnds)
if df.empty:
raise NoDataFound("No data was found for your query")
mp = MapPlot(sector=('state' if ctx['t'] == 'state' else 'cwa'),
state=ctx['state'],
cwa=(ctx['wfo'] if len(ctx['wfo']) == 3 else ctx['wfo'][1:]),
axisbg='white',
title='%s for %s' % (PDICT2[ctx['v']], title),
subtitle=('Map valid: %s UTC') %
(valid.strftime("%d %b %Y %H:%M"), ),
nocaption=True,
titlefontsize=16)
if varname == 'vsby':
ramp = np.array([0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 8, 9.9])
valunit = 'miles'
elif varname == 'feel':
valunit = 'F'
df['feel'] = apparent_temperature(df['tmpf'].values * units('degF'),
df['relh'].values * units('percent'),
df['sknt'].values *
units('knots')).to(units('degF')).m
# Data QC, cough
if ctx.get('above'):
df = df[df[varname] < ctx['above']]
if ctx.get('below'):
df = df[df[varname] > ctx['below']]
# with QC done, we compute ramps
if varname != 'vsby':
ramp = np.linspace(df[varname].min() - 5,
df[varname].max() + 5,
10,
dtype='i')
mp.contourf(df['lon'].values,
df['lat'].values,
df[varname].values,
ramp,
units=valunit,
cmap=plt.get_cmap(ctx['cmap']))
if ctx['t'] == 'state':
df2 = df[df['state'] == ctx['state']]
else:
df2 = df[df['wfo'] == ctx['wfo']]
mp.plot_values(df2['lon'].values,
df2['lat'].values,
df2[varname].values,
'%.1f',
labelbuffer=10)
mp.drawcounties()
if ctx['t'] == 'cwa':
mp.draw_cwas()
return mp.fig, df
def plotter(fdict):
""" Go """
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
ctx = get_autoplot_context(fdict, get_description())
state = ctx['state'][:2]
sector = ctx['sector']
opt = ctx['opt']
p1syear = ctx['p1syear']
p1eyear = ctx['p1eyear']
p2syear = ctx['p2syear']
p2eyear = ctx['p2eyear']
varname = ctx['var']
table = "alldata"
if sector == 'state':
table = "alldata_%s" % (state, )
df = read_sql("""
WITH season1 as (
SELECT station, year,
min(case when month > 7 and low < 32 then
extract(doy from day) else 366 end) as first_freeze,
max(case when month < 7 and low < 32 then
extract(doy from day) else 0 end) as last_freeze
from """ + table + """ WHERE
year >= %s and year <= %s GROUP by station, year),
season2 as (
SELECT station, year,
min(case when month > 7 and low < 32 then
extract(doy from day) else 366 end) as first_freeze,
max(case when month < 7 and low < 32 then
extract(doy from day) else 0 end) as last_freeze
from """ + table + """ WHERE
year >= %s and year <= %s GROUP by station, year),
agg as (
SELECT p1.station, avg(p1.first_freeze) as p1_first_fall,
avg(p1.last_freeze) as p1_last_spring,
avg(p2.first_freeze) as p2_first_fall,
avg(p2.last_freeze) as p2_last_spring
from season1 as p1 JOIN season2 as p2 on (p1.station = p2.station)
GROUP by p1.station)
SELECT station, ST_X(geom) as lon, ST_Y(geom) as lat,
d.* from agg d JOIN stations t ON (d.station = t.id)
WHERE t.network in ('IACLIMATE', 'NDCLIMATE', 'SDCLIMATE', 'NECLIMATE',
'KSCLIMATE', 'MOCLIMATE', 'ILCLIMATE', 'WICLIMATE', 'MNCLIMATE',
'MICLIMATE', 'INCLIMATE', 'OHCLIMATE', 'KYCLIMATE')
and substr(station, 3, 1) != 'C' and substr(station, 3, 4) != '0000'
""",
pgconn,
params=[p1syear, p1eyear, p2syear, p2eyear],
index_col='station')
df['p1_season'] = df['p1_first_fall'] - df['p1_last_spring']
df['p2_season'] = df['p2_first_fall'] - df['p2_last_spring']
df['season_delta'] = df['p2_season'] - df['p1_season']
df['spring_delta'] = df['p2_last_spring'] - df['p1_last_spring']
df['fall_delta'] = df['p2_first_fall'] - df['p1_first_fall']
# Reindex so that most extreme values are first
df = df.reindex(df[varname +
'_delta'].abs().sort_values(ascending=False).index)
title = PDICT3[varname]
m = MapPlot(sector=sector,
state=state,
axisbg='white',
title=('%.0f-%.0f minus %.0f-%.0f %s Difference') %
(p2syear, p2eyear, p1syear, p1eyear, title),
subtitle=('based on IEM Archives'),
titlefontsize=14)
# Create 9 levels centered on zero
abval = df[varname + '_delta'].abs().max()
levels = centered_bins(abval)
if opt in ['both', 'contour']:
m.contourf(df['lon'].values,
df['lat'].values,
df[varname + '_delta'].values,
levels,
cmap=plt.get_cmap('seismic'),
units='days')
if sector == 'state':
m.drawcounties()
if opt in ['both', 'values']:
m.plot_values(df['lon'].values,
df['lat'].values,
df[varname + '_delta'].values,
fmt='%.1f',
labelbuffer=5)
return m.fig, df
def plotter(fdict):
""" Go """
pgconn = get_dbconn("coop")
ctx = get_autoplot_context(fdict, get_description())
sector = ctx["sector"]
date1 = ctx["date1"]
date2 = ctx["date2"]
varname = ctx["var"]
table = "alldata_%s" % (sector, ) if len(sector) == 2 else "alldata"
state_limiter = ""
if sector == "iailin":
state_limiter = (
" and network in ('IACLIMATE', 'ILCLIMATE', 'INCLIMATE') ")
df = read_sql(
"""
WITH obs as (
SELECT station, gddxx(%s, 86, high, low) as gdd,
cdd(high, low, 65) as cdd65, hdd(high, low, 65) as hdd65,
sday, high, low, precip,
(high + low)/2. as avg_temp
from """ + table + """ WHERE
day >= %s and day < %s and
substr(station, 3, 1) != 'C' and substr(station, 3, 4) != '0000'),
climo as (
SELECT station, to_char(valid, 'mmdd') as sday, precip, high, low,
gdd""" + str(ctx["gddbase"]) + """ as gdd, cdd65, hdd65
from climate51),
combo as (
SELECT o.station, o.precip - c.precip as precip_diff,
o.precip as precip, c.precip as cprecip,
o.avg_temp, o.cdd65, o.hdd65,
o.high, o.low, o.gdd, c.gdd as cgdd,
o.gdd - c.gdd as gdd_diff,
o.cdd65 - c.cdd65 as cdd_diff,
o.hdd65 - c.hdd65 as hdd_diff,
o.avg_temp - (c.high + c.low)/2. as temp_diff
from obs o JOIN climo c ON
(o.station = c.station and o.sday = c.sday)),
agg as (
SELECT station,
avg(avg_temp) as avg_temp,
sum(precip_diff) as precip_depart,
sum(precip) / sum(cprecip) * 100. as precip_percent,
sum(precip) as precip, sum(cprecip) as cprecip,
avg(high) as avg_high_temp,
avg(low) as avg_low_temp,
max(high) as max_high_temp,
min(low) as min_low_temp, sum(gdd_diff) as gdd_depart,
sum(gdd) / greatest(1, sum(cgdd)) * 100. as gdd_percent,
avg(temp_diff) as avg_temp_depart, sum(gdd) as gdd_sum,
sum(cgdd) as cgdd_sum,
sum(cdd65) as cdd_sum,
sum(hdd65) as hdd_sum,
sum(cdd_diff) as cdd_depart,
sum(hdd_diff) as hdd_depart
from combo GROUP by station)
SELECT d.station, t.name,
precip as precip_sum,
avg_temp,
cprecip as cprecip_sum,
precip_depart,
precip_percent,
min_low_temp,
avg_temp_depart,
gdd_depart,
gdd_sum,
gdd_percent,
cgdd_sum,
max_high_temp,
avg_high_temp,
avg_low_temp,
cdd_sum, hdd_sum, cdd_depart, hdd_depart,
ST_x(t.geom) as lon, ST_y(t.geom) as lat
from agg d JOIN stations t on (d.station = t.id)
WHERE t.network ~* 'CLIMATE' """ + state_limiter + """
""",
pgconn,
params=(ctx["gddbase"], date1, date2),
index_col="station",
)
if df.empty:
raise NoDataFound("No Data Found.")
df = df.reindex(df[varname].abs().sort_values(ascending=False).index)
datefmt = "%d %b %Y" if varname != "cgdd_sum" else "%d %b"
subtitle = ""
if varname.find("depart") > -1:
subtitle = ("%s is compared with 1951-%s Climatology"
" to compute departures") % (
date1.year, datetime.date.today().year - 1)
elif varname.startswith("c"):
subtitle = ("Climatology is based on data from 1951-%s") % (
datetime.date.today().year - 1, )
mp = MapPlot(
sector="state" if len(sector) == 2 else sector,
state=sector,
axisbg="white",
title="%s - %s %s [%s]" % (
date1.strftime(datefmt),
date2.strftime(datefmt),
PDICT2.get(varname).replace("$base", str(ctx["gddbase"])),
UNITS.get(varname),
),
subtitle=subtitle,
)
fmt = "%.2f"
cmap = cm.get_cmap(ctx["cmap"])
if varname in ["precip_depart", "avg_temp_depart", "gdd_depart"]:
rng = df[varname].abs().describe(percentiles=[0.95])["95%"]
clevels = np.linspace(0 - rng,
rng,
7,
dtype="i" if varname == "gdd_depart" else "f")
if varname == "gdd_depart":
fmt = "%.0f"
elif varname in ["precip_sum"]:
rng = df[varname].abs().describe(percentiles=[0.95])["95%"]
clevels = np.linspace(0, rng, 7)
cmap.set_under("white")
cmap.set_over("black")
elif varname.endswith("_percent"):
clevels = np.array([10, 25, 50, 75, 100, 125, 150, 175, 200])
fmt = "%.0f"
else:
minv = df[varname].min() - 5
maxv = df[varname].max() + 5
clevels = np.linspace(minv, maxv, 6, dtype="i")
fmt = "%.0f"
clevlabels = [fmt % x for x in clevels]
cmap.set_bad("white")
if ctx["p"] == "contour":
mp.contourf(
df["lon"].values,
df["lat"].values,
df[varname].values,
clevels,
clevlabels=clevlabels,
cmap=cmap,
units=UNITS.get(varname),
)
if ctx["c"] == "yes":
mp.plot_values(
df["lon"].values,
df["lat"].values,
df[varname].values,
fmt=fmt,
labelbuffer=5,
)
if len(sector) == 2 or sector == "iailin":
mp.drawcounties()
if ctx["usdm"] == "yes":
mp.draw_usdm(date2, filled=False, hatched=True)
return mp.fig, df
def main():
"""Go Main Go."""
pgconn = get_dbconn('iem')
cursor = pgconn.cursor()
cursor.execute("""SELECT ST_x(geom) as lon, ST_y(geom) as lat,
pday from summary_2016 s JOIN stations t on (s.iemid = t.iemid)
where day = '2016-08-24' and network in ('WI_COOP', 'MN_COOP', 'IA_COOP')
and pday > 0 ORDER by pday DESC""")
llons = []
llats = []
vals = []
for row in cursor:
llons.append(row[0])
llats.append(row[1])
vals.append("%.2f" % (row[2], ))
pgconn = get_dbconn('postgis')
cursor = pgconn.cursor()
cursor.execute("""SELECT ST_x(geom) as lon, ST_y(geom) as lat,
max(magnitude) from lsrs_2016
where wfo in ('DMX', 'DVN', 'ARX') and typetext = 'HEAVY RAIN' and
valid > '2016-08-23' GROUP by lon, lat ORDER by max DESC""")
for row in cursor:
llons.append(row[0])
llats.append(row[1])
vals.append("%.2f" % (row[2], ))
img = Image.open("p24h_201608241200.png")
data = np.flipud(np.asarray(img))
# 7000,3500 == -130,-60,55,25 === -100 to -90 then 38 to 45
sample = data[1800:2501, 3000:4501]
sample = np.where(sample == 255, 0, sample)
data = sample * 0.01
data = np.where(sample > 100, 1. + (sample - 100) * 0.05, data)
data = np.where(sample > 180, 5. + (sample - 180) * 0.2, data)
lons = np.arange(-100, -84.99, 0.01)
lats = np.arange(38, 45.01, 0.01)
x, y = np.meshgrid(lons, lats)
buff = 0.5
m = MapPlot(sector='custom', projection='aea', west=-93.2,
east=-90.3, south=42.5,
north=44.,
title='NOAA MRMS 24 Hour RADAR-Only Precipitation Estimate',
subtitle=("MRMS valid 7 AM 23 Aug 2016 to 7 AM 24 Aug 2016, "
"NWS Local Storm + COOP Reports Overlaid"))
clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
10]
m.contourf(x[:, :-1], y[:, :-1], data, clevs, cmap=nwsprecip())
nt = NetworkTable("IA_ASOS")
lo = []
la = []
va = []
for sid in nt.sts.keys():
lo.append(nt.sts[sid]['lon'])
la.append(nt.sts[sid]['lat'])
va.append(nt.sts[sid]['name'])
# m.plot_values(lo, la, va, fmt='%s', textsize=10, color='black')
m.drawcounties(zorder=4, linewidth=1.)
m.drawcities(labelbuffer=25, textsize=10, color='white',
outlinecolor='#000000')
m.textmask[:, :] = 0
m.plot_values(llons, llats, vals, fmt='%s', labelbuffer=5)
m.postprocess(filename='test.png')
def main():
"""GO!"""
pgconn = psycopg2.connect(database='idep',
host='localhost',
port=5555,
user='******')
cursor = pgconn.cursor()
scenario = int(sys.argv[1])
mp = MapPlot(sector='iowa',
axisbg='white',
nologo=True,
subtitle='1 Jan 2014 thru 31 Dec 2014',
caption='Daily Erosion Project',
title=('Harvest Index 0.8 Change in 2014 Soil Delivery '
'from Baseline'))
cursor.execute(
"""
with baseline as (
SELECT huc_12, sum(avg_delivery) * 4.463 as loss from results_by_huc12
where valid between '2014-01-01' and '2015-01-01' and
scenario = 0 GROUP by huc_12),
scenario as (
SELECT huc_12, sum(avg_delivery) * 4.463 as loss from results_by_huc12
where valid between '2014-01-01' and '2015-01-01' and
scenario = %s GROUP by huc_12),
agg as (
SELECT b.huc_12, b.loss as baseline_loss, s.loss as scenario_loss from
baseline b LEFT JOIN scenario s on (b.huc_12 = s.huc_12))
SELECT ST_Transform(simple_geom, 4326),
(scenario_loss - baseline_loss) / 1.0 as val, i.huc_12
from huc12 i JOIN agg d on (d.huc_12 = i.huc_12)
WHERE i.states ~* 'IA' ORDER by val DESC
""", (scenario, ))
# bins = np.arange(0, 101, 10)
bins = [-5, -2, -1, -0.5, 0, 0.5, 1, 2, 5]
cmap = plt.get_cmap("BrBG_r")
cmap.set_under('purple')
cmap.set_over('black')
norm = mpcolors.BoundaryNorm(bins, cmap.N)
for row in cursor:
# print "%s,%s" % (row[2], row[1])
polygon = loads(row[0].decode('hex'))
arr = np.asarray(polygon.exterior)
points = mp.ax.projection.transform_points(ccrs.Geodetic(), arr[:, 0],
arr[:, 1])
val = float(row[1])
# We have very small negative numbers that should just be near a
# positive zero
if val < 0 and val > -0.1:
val = 0.001
color = cmap(norm([
val,
]))[0]
poly = Polygon(points[:, :2], fc=color, ec='k', zorder=2, lw=0.1)
mp.ax.add_patch(poly)
mp.draw_colorbar(bins, cmap, norm, units='T/a/yr')
mp.drawcounties()
mp.postprocess(filename='test.png')
""")
bins = np.arange(0, 18, 2)
# cmap = cm.get_cmap('BrBG')
# cmap.set_under('orange')
# cmap.set_over('black')
cmap = james()
cmap.set_under("white")
cmap.set_over("black")
norm = mpcolors.BoundaryNorm(bins, cmap.N)
patches = []
for row in cursor:
geom = loads(row[0].decode("hex"))
for polygon in geom:
a = np.asarray(polygon.exterior)
x, y = m.map(a[:, 0], a[:, 1])
a = zip(x, y)
# c = cmap( norm([float(row[1]),]) )[0]
diff = data.get(row[1], 0) - data1.get(row[1], 0)
diff = data.get(row[1], 0)
c = cmap(norm([float(diff)]))[0]
p = Polygon(a, fc=c, ec="None", zorder=2, lw=0.1)
patches.append(p)
m.ax.add_collection(PatchCollection(patches, match_original=True))
m.draw_colorbar(bins, cmap, norm, units="%")
m.drawcounties()
m.postprocess(filename="test.png")
def doit(ts):
"""
Generate hourly plot of stage4 data
"""
gmtnow = datetime.datetime.utcnow()
gmtnow = gmtnow.replace(tzinfo=pytz.utc)
routes = "a"
if ((gmtnow - ts).days * 86400.0 + (gmtnow - ts).seconds) < 7200:
routes = "ac"
fn = "/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib" % (
ts.strftime("%Y/%m/%d"),
ts.strftime("%Y%m%d%H"),
)
if not os.path.isfile(fn):
LOG.info("Missing stage4 %s", fn)
return
grbs = pygrib.open(fn)
grib = grbs[1]
lats, lons = grib.latlons()
vals = grib.values / 25.4
cmap = get_cmap("jet")
cmap.set_under("white")
cmap.set_over("black")
clevs = [
0.01,
0.05,
0.1,
0.2,
0.3,
0.4,
0.5,
0.6,
0.7,
0.8,
0.9,
1,
1.5,
2,
3,
]
localtime = ts.astimezone(pytz.timezone("America/Chicago"))
for sector in ["iowa", "midwest", "conus"]:
mp = MapPlot(
sector=sector,
title="Stage IV One Hour Precipitation",
subtitle="Hour Ending %s" %
(localtime.strftime("%d %B %Y %I %p %Z"), ),
)
mp.pcolormesh(lons, lats, vals, clevs, units="inch")
pqstr = "plot %s %s00 %s_stage4_1h.png %s_stage4_1h_%s.png png" % (
routes,
ts.strftime("%Y%m%d%H"),
sector,
sector,
ts.strftime("%H"),
)
if sector == "iowa":
mp.drawcounties()
mp.postprocess(view=False, pqstr=pqstr)
mp.close()
from pyiem.plot import MapPlot
import datetime
import psycopg2
import numpy as np
IEM = psycopg2.connect(database='iem', host='iemdb', user='******')
icursor = IEM.cursor()
vals = []
lats = []
lons = []
valmask = []
icursor.execute("""SELECT id, network, ST_x(geom) as lon,
ST_y(geom) as lat, min(min_tmpf)
from summary_2015 s JOIN stations t on (t.iemid = s.iemid)
WHERE network IN ('IA_ASOS','AWOS') and min_tmpf > -50 and
day > '2012-08-01' and id not in ('XXX')
GROUP by id, network, lon, lat ORDER by min ASC""")
for row in icursor:
vals.append(row[4])
lats.append(row[3])
lons.append(row[2])
valmask.append(row[1] in ['IA_ASOS', 'AWOS'])
print row[4], row[0]
m = MapPlot(title="2014-2015 Winter Minimum Temperature $^\circ$F", axisbg='white',
subtitle='Automated Weather Stations, Valid Fall 2014 - %s' % (datetime.datetime.now().strftime("%d %b %Y"),)
)
#m.contourf(lons, lats, vals, np.arange(-30,1,4))
m.plot_values(lons, lats, vals, '%.0f', valmask)
m.drawcounties()
m.postprocess(filename='150223.png')
def plotter(fdict):
""" Go """
pgconn = get_dbconn('coop')
ctx = get_autoplot_context(fdict, get_description())
state = ctx['state'][:2]
varname = ctx['var']
sector = ctx['sector']
opt = ctx['opt']
over = ctx['over']
month = ctx['month']
df = read_sql("""
WITH data as (
SELECT station, extract(month from valid) as month,
sum(precip) as total_precip, avg(high) as avg_high,
avg(low) as avg_low, avg((high+low)/2.) as avg_temp
from ncdc_climate81 GROUP by station, month)
SELECT station, ST_X(geom) as lon, ST_Y(geom) as lat, month,
total_precip, avg_high, avg_low, avg_temp from data d JOIN stations t
ON (d.station = t.id) WHERE t.network = 'NCDC81'
""",
pgconn,
index_col=['station', 'month'])
if df.empty:
raise ValueError("No data was found for query, sorry.")
if over == 'monthly':
title = "%s %s" % (calendar.month_name[month], PDICT3[varname])
df.reset_index(inplace=True)
df2 = df[df['month'] == month]
else:
title = "Annual %s" % (PDICT3[varname], )
if varname == 'total_precip':
df2 = df.sum(axis=0, level='station')
else:
df2 = df.mean(axis=0, level='station')
df2['lat'] = df['lat'].mean(axis=0, level='station')
df2['lon'] = df['lon'].mean(axis=0, level='station')
mp = MapPlot(sector=sector,
state=state,
axisbg='white',
title=('NCEI 1981-2010 Climatology of %s') % (title, ),
subtitle=('based on National Centers for '
'Environmental Information (NCEI) 1981-2010'
' Climatology'))
levels = np.linspace(df2[varname].min(), df2[varname].max(), 10)
levels = [round(x, PRECISION[varname]) for x in levels]
if opt in ['both', 'contour']:
mp.contourf(df2['lon'].values,
df2['lat'].values,
df2[varname].values,
levels,
units=UNITS[varname])
if sector == 'state':
mp.drawcounties()
if opt in ['both', 'values']:
mp.plot_values(df2['lon'].values,
df2['lat'].values,
df2[varname].values,
fmt='%%.%if' % (PRECISION[varname], ))
return mp.fig, df
