def test_loadqc(self):
"""Make sure we exercise the loadqc stuff"""
q = loadqc()
self.assertEquals(len(q), 0)
q = loadqc(cursor=self.pcursor)
self.assertEquals(len(q), 0)
self.pcursor.execute("""
INSERT into tt_base(s_mid, sensor, status) VALUES
('BOGUS', 'tmpf', 'OPEN')
""")
q = loadqc(cursor=self.pcursor)
self.assertEquals(len(q), 1)
def test_loadqc(pcursor):
"""Make sure we exercise the loadqc stuff"""
q = loadqc()
assert not q
q = loadqc(cursor=pcursor)
assert not q
pcursor.execute("""
INSERT into tt_base(s_mid, sensor, status, entered) VALUES
('BOGUS', 'tmpf', 'OPEN', '2019-03-27')
""")
q = loadqc(cursor=pcursor, date=datetime.date(2019, 3, 27))
assert q
def test_loadqc(pcursor):
"""Make sure we exercise the loadqc stuff"""
q = loadqc()
assert not q
q = loadqc(cursor=pcursor)
assert not q
pcursor.execute("""
INSERT into tt_base(s_mid, sensor, status, entered) VALUES
('BOGUS', 'tmpf', 'OPEN', '2019-03-27')
""")
q = loadqc(cursor=pcursor, date=datetime.date(2019, 3, 27))
assert q
def generate_rr5():
"""Create the RR5 Data"""
qcdict = loadqc()
data = ("\n\n\n"
": Iowa State University Soil Moisture Network\n"
": Data contact Daryl Herzmann [email protected]\n"
": File generated %s UTC\n"
) % (datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M"),)
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
cursor = pgconn.cursor()
cursor.execute("""SELECT id, valid, tmpf, c1tmpf, c2tmpf, c3tmpf, c4tmpf
from current c JOIN stations t
on (t.iemid = c.iemid) WHERE t.network = 'ISUSM' and
valid > (now() - '90 minutes'::interval)""")
for row in cursor:
q = qcdict.get(row[0], dict())
if 'tmpf' in q or row[2] is None:
tmpf = "M"
else:
tmpf = "%.1f" % (row[2],)
data += (".A %s %s C DH%s/TA %s%s%s%s%s\n"
) % (row[0], row[1].strftime("%Y%m%d"),
row[1].strftime("%H%M"), tmpf, mt(row[3], '4', q),
mt(row[4], '12', q), mt(row[5], '24', q),
mt(row[6], '50', q))
return data
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
ctx['qc'] = loadqc(date=ctx['date'])
ctx['pgconn'] = get_dbconn('isuag')
ctx['nt'] = NetworkTable("ISUSM")
# Adjust stations to make some room
ctx['nt'].sts['BOOI4']['lon'] -= 0.15
ctx['nt'].sts['BOOI4']['lat'] -= 0.15
ctx['nt'].sts['AHTI4']['lon'] += 0.25
ctx['nt'].sts['AHTI4']['lat'] += 0.25
title = 'TBD'
subtitle = 'TBD'
if ctx['opt'] == '1':
title = 'ISU Soil Moisture Max/Min 4 Inch Soil Temperature'
subtitle = 'based on available hourly observations'
data, df = plot1(ctx)
elif ctx['opt'] == '2':
title = 'ISU Soil Moisture Max/Min Air Temperature'
subtitle = 'based on available daily summary data'
data, df = plot2(ctx)
elif ctx['opt'] == '3':
title = 'ISU Soil Moisture Average 4 Inch Soil Temperature'
subtitle = 'based on available daily summary data'
data, df = plot3(ctx)
elif ctx['opt'] == '4':
title = 'ISU Soil Moisture Solar Radiation [MJ]'
subtitle = 'based on available daily summary data'
data, df = plot4(ctx)
elif ctx['opt'] == '5':
title = 'ISU Soil Moisture Potential Evapotranspiration [inch]'
subtitle = 'based on available daily summary data'
data, df = plot5(ctx, "dailyet")
elif ctx['opt'] == '6':
title = 'ISU Soil Moisture Precipitation [inch]'
subtitle = (
'based on available daily summary data, liquid equiv of snow '
'estimated'
)
data, df = plot5(ctx, "rain_mm_tot")
elif ctx['opt'] == '7':
title = 'ISU Soil Moisture Peak Wind Gust [MPH]'
subtitle = 'based on available daily summary data'
data, df = plot7(ctx)
elif ctx['opt'] == '8':
title = 'ISU Soil Moisture Average Wind Speed [MPH]'
subtitle = 'based on available daily summary data'
data, df = plot8(ctx)
tle = ctx['date'].strftime("%b %-d, %Y")
mp = MapPlot(
sector='iowa', continentalcolor='white', nocaption=True,
title='%s %s' % (tle, title),
subtitle=subtitle)
mp.drawcounties('#EEEEEE')
mp.plot_station(data, fontsize=12)
return mp.fig, df
def get_data(ts):
""" Get the data for this timestamp """
qcdict = loadqc()
nt = NetworkTable("ISUSM")
data = {"type": "FeatureCollection",
"crs": {"type": "EPSG",
"properties": {"code": 4326,
"coordinate_order": [1, 0]}},
"features": []}
# Fetch the daily values
iemcursor.execute("""
SELECT id, pday, max_tmpf, min_tmpf from summary s JOIN stations t
on (t.iemid = s.iemid) WHERE t.network = 'ISUSM' and day = %s
""", (ts.date(),))
daily = {}
for row in iemcursor:
daily[row[0]] = {'pday': row[1], 'max_tmpf': row[2],
'min_tmpf': row[3]}
cursor.execute("""
SELECT * from sm_hourly where valid = %s
""", (ts,))
for i, row in enumerate(cursor):
sid = row['station']
lon = nt.sts[sid]['lon']
lat = nt.sts[sid]['lat']
q = qcdict.get(sid, {})
data['features'].append({"type": "Feature",
"id": sid, "properties": {
"encrh_avg": "%s%%" % safe(row['encrh_avg'], 1) if row['encrh_avg'] > 0 else "M",
"rh": "%.0f%%" % (row["rh"],),
"hrprecip" : safe_p(row['rain_mm_tot']) if not q.get('precip', False) else 'M',
"et": safe_p(row['etalfalfa']),
"bat": safe(row['battv_min'], 2),
"radmj": safe(row['slrmj_tot'], 2),
"tmpf": safe_t(row['tair_c_avg']),
"high": safe_t(daily.get(sid, {}).get('max_tmpf', None), 'F'),
"low": safe_t(daily.get(sid, {}).get('min_tmpf', None), 'F'),
"pday": safe(daily.get(sid, {}).get('pday', None), 2) if not q.get('precip', False) else 'M',
"soil04t": safe_t(row['tsoil_c_avg']) if not q.get('soil4', False) else 'M',
"soil12t": safe_t(row['t12_c_avg']) if not q.get('soil12', False) else 'M',
"soil24t": safe_t(row['t24_c_avg']) if not q.get('soil24', False) else 'M',
"soil50t": safe_t(row['t50_c_avg']) if not q.get('soil50', False) else 'M',
"soil12m": safe_m(row['vwc_12_avg']) if not q.get('soil12', False) else 'M',
"soil24m": safe_m(row['vwc_24_avg']) if not q.get('soil24', False) else 'M',
"soil50m": safe_m(row['vwc_50_avg']) if not q.get('soil50', False) else 'M',
"gust": safe(row['ws_mph_max'], 1),
"wind": "%s@%.0f" % (drct2txt(row['winddir_d1_wvt']),
row['ws_mps_s_wvt'] * 2.23),
'name': nt.sts[sid]['name']
},
"geometry": {"type": "Point",
"coordinates": [lon, lat]
}
})
sys.stdout.write(json.dumps(data))
def iemtracker(df, edate):
"""Figure out what should be QC'd out."""
qcdict = loadqc(date=edate)
for idx, row in df.iterrows():
qc = qcdict.get(row["station"], {})
if qc.get("precip"):
df.at[idx, "precip"] = np.nan
if qc.get("soil4"):
df.at[idx, "sgdd"] = np.nan
if qc.get("tmpf"):
df.at[idx, "gdd"] = np.nan
return df
def get_data(ts):
""" Get the data for this timestamp """
qcdict = loadqc()
nt = NetworkTable("ISUSM")
data = {"type": "FeatureCollection",
"crs": {"type": "EPSG",
"properties": {"code": 4326,
"coordinate_order": [1, 0]}},
"features": []}
# Fetch the daily values
iemcursor.execute("""
SELECT id, pday, max_tmpf, min_tmpf from summary s JOIN stations t
on (t.iemid = s.iemid) WHERE t.network = 'ISUSM' and day = %s
""", (ts.date(),))
daily = {}
for row in iemcursor:
daily[row[0]] = {'pday': row[1], 'max_tmpf': row[2],
'min_tmpf': row[3]}
cursor.execute("""
SELECT * from sm_hourly where valid = %s
""", (ts,))
for i, row in enumerate(cursor):
lon = nt.sts[row['station']]['lon']
lat = nt.sts[row['station']]['lat']
q = qcdict.get(row['station'], {})
data['features'].append({"type": "Feature", "id": i, "properties": {
"encrh_avg": "%s%%" % safe(row['encrh_avg'], 1) if row['encrh_avg'] > 5 else "M",
"rh": "%.0f%%" % (row["rh"],),
"hrprecip" : safe_p(row['rain_mm_tot']) if not q.get('precip', False) else 'M',
"et": safe_p(row['etalfalfa']),
"bat": safe(row['battv_min'], 2),
"radmj": safe(row['slrmj_tot'], 2),
"tmpf": safe_t(row['tair_c_avg']),
"high": safe_t(daily.get(row['station'], {}).get('max_tmpf', None), 'F'),
"low": safe_t(daily.get(row['station'], {}).get('min_tmpf', None), 'F'),
"pday": safe(daily.get(row['station'], {}).get('pday', None), 2) if not q.get('precip', False) else 'M',
"soil04t": safe_t(row['tsoil_c_avg']) if not q.get('soil4', False) else 'M',
"soil12t": safe_t(row['t12_c_avg']),
"soil24t": safe_t(row['t24_c_avg']),
"soil50t": safe_t(row['t50_c_avg']),
"soil12m": safe_m(row['vwc_12_avg']),
"soil24m": safe_m(row['vwc_24_avg']),
"soil50m": safe_m(row['vwc_50_avg']),
"wind": "%s@%.0f" % (drct2txt(row['winddir_d1_wvt']),
row['ws_mps_s_wvt'] * 2.23)
},
"geometry": {"type": "Point",
"coordinates": [lon, lat]
}
})
sys.stdout.write(json.dumps(data))
def generate_rr5():
"""Create the RR5 Data"""
qcdict = loadqc()
data = (
"\n\n\n"
": Iowa State University Soil Moisture Network\n"
": Data contact Daryl Herzmann [email protected]\n"
": File generated %s UTC\n"
) % (datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M"),)
pgconn = get_dbconn("iem", user="******")
cursor = pgconn.cursor()
cursor.execute(
"""
SELECT id, valid, tmpf, c1tmpf, c2tmpf, c3tmpf, c4tmpf,
c2smv, c3smv, c4smv, phour
from current c JOIN stations t
on (t.iemid = c.iemid) WHERE t.network = 'ISUSM' and
valid > (now() - '90 minutes'::interval)
"""
)
for row in cursor:
q = qcdict.get(row[0], dict())
if "tmpf" in q or row[2] is None:
tmpf = "M"
else:
tmpf = "%.1f" % (row[2],)
if "precip" in q or row[10] is None:
precip = "M"
else:
precip = "%.2f" % (row[10],)
data += (".A %s %s C DH%s/TA %s%s%s%s%s\n" ".A1 %s%s%s/PPHRP %s\n") % (
row[0],
row[1].strftime("%Y%m%d"),
row[1].strftime("%H%M"),
tmpf,
mt("TV", row[3], "4", q),
mt("TV", row[4], "12", q),
mt("TV", row[5], "24", q),
mt("TV", row[6], "50", q),
mt("MV", max([0, 0 if row[7] is None else row[7]]), "12", q),
mt("MV", max([0, 0 if row[8] is None else row[8]]), "24", q),
mt("MV", max([0, 0 if row[9] is None else row[9]]), "50", q),
precip,
)
return data
def generate_rr5():
"""Create the RR5 Data"""
qcdict = loadqc()
data = (
"\n\n\n"
": Iowa State University Soil Moisture Network\n"
": Data contact Daryl Herzmann [email protected]\n"
": File generated %s UTC\n"
) % (datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M"),)
pgconn = psycopg2.connect(database="iem", host="iemdb", user="******")
cursor = pgconn.cursor()
cursor.execute(
"""SELECT id, valid, tmpf, c1tmpf, c2tmpf, c3tmpf, c4tmpf,
c2smv, c3smv, c4smv, phour
from current c JOIN stations t
on (t.iemid = c.iemid) WHERE t.network = 'ISUSM' and
valid > (now() - '90 minutes'::interval)"""
)
for row in cursor:
q = qcdict.get(row[0], dict())
if "tmpf" in q or row[2] is None:
tmpf = "M"
else:
tmpf = "%.1f" % (row[2],)
if "precip" in q or row[10] is None:
precip = "M"
else:
precip = "%.2f" % (row[10],)
data += (".A %s %s C DH%s/TA %s%s%s%s%s\n" ".A1 %s%s%s/PPH %s\n") % (
row[0],
row[1].strftime("%Y%m%d"),
row[1].strftime("%H%M"),
tmpf,
mt("TV", row[3], "4", q),
mt("TV", row[4], "12", q),
mt("TV", row[5], "24", q),
mt("TV", row[6], "50", q),
mt("MW", max([0, row[7]]), "12", q),
mt("MW", max([0, row[8]]), "24", q),
mt("MW", max([0, row[9]]), "50", q),
precip,
)
return data
def generate_rr5():
"""Create the RR5 Data"""
qcdict = loadqc()
data = ("\n\n\n"
": Iowa State University Soil Moisture Network\n"
": Data contact Daryl Herzmann [email protected]\n"
": File generated %s UTC\n"
) % (datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M"),)
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
cursor = pgconn.cursor()
cursor.execute("""SELECT id, valid, tmpf, c1tmpf, c2tmpf, c3tmpf, c4tmpf,
c2smv, c3smv, c4smv, phour
from current c JOIN stations t
on (t.iemid = c.iemid) WHERE t.network = 'ISUSM' and
valid > (now() - '90 minutes'::interval)""")
for row in cursor:
q = qcdict.get(row[0], dict())
if 'tmpf' in q or row[2] is None:
tmpf = "M"
else:
tmpf = "%.1f" % (row[2], )
if 'precip' in q or row[10] is None:
precip = "M"
else:
precip = "%.2f" % (row[10], )
data += (".A %s %s C DH%s/TA %s%s%s%s%s\n"
".A1 %s%s%s/PPHRP %s\n"
) % (row[0], row[1].strftime("%Y%m%d"),
row[1].strftime("%H%M"), tmpf,
mt('TV', row[3], '4', q), mt('TV', row[4], '12', q),
mt('TV', row[5], '24', q), mt('TV', row[6], '50', q),
mt('MW', max([0, row[7]]), '12', q),
mt('MW', max([0, row[8]]), '24', q),
mt('MW', max([0, row[9]]), '50', q),
precip
)
return data
def main():
"""Go Main Go."""
nt = network.Table("AWOS")
qdict = loadqc()
pgconn = get_dbconn("iem", user="******")
icursor = pgconn.cursor()
utc = datetime.datetime.utcnow()
ets = utc.replace(
tzinfo=pytz.utc, hour=0, minute=0, second=0, microsecond=0
)
sts12z = ets + datetime.timedelta(hours=-12)
sts6z = ets + datetime.timedelta(hours=-18)
sts24h = ets + datetime.timedelta(days=-1)
fmt = "%-6s:%-19s: %3s / %3s / %5s / %4s / %2s\n"
out = open("/tmp/awos_rtp.shef", "w")
out.write(
"""
.BR DMX %s Z DH00/TAIRVS/TAIRVI/PPDRVZ/SFDRVZ/SDIRVZ
: IOWA AWOS RTP FIRST GUESS PROCESSED BY THE IEM
: 06Z TO 00Z HIGH TEMPERATURE FOR %s
: 06Z TO 00Z LOW TEMPERATURE FOR %s
: 00Z YESTERDAY TO 00Z TODAY RAINFALL
: ...BASED ON REPORTED OBS...
"""
% (
ets.strftime("%m%d"),
sts12z.strftime("%d %b %Y").upper(),
sts12z.strftime("%d %b %Y").upper(),
)
)
# We get 18 hour highs
highs = {}
sql = """SELECT t.id as station,
round(max(tmpf)::numeric,0) as max_tmpf,
count(tmpf) as obs FROM current_log c, stations t
WHERE t.iemid = c.iemid and t.network = 'AWOS' and valid > %s
and valid < %s
and tmpf > -99 GROUP by t.id """
args = (sts6z, ets)
icursor.execute(sql, args)
for row in icursor:
if qdict.get(row[0], {}).get("tmpf"):
continue
highs[row[0]] = row[1]
# 00 UTC to 00 UTC Preciptation
pcpn = {}
icursor.execute(
"""
select id as station, sum(precip) from
(select t.id, extract(hour from valid) as hour,
max(phour) as precip from current_log c, stations t
WHERE t.network = 'AWOS' and t.iemid = c.iemid
and valid >= %s and valid < %s
GROUP by t.id, hour) as foo
GROUP by id""",
(sts24h, ets),
)
for row in icursor:
if qdict.get(row[0], {}).get("precip") or row[1] is None:
continue
pcpn[row[0]] = "%5.2f" % (row[1],)
lows = {}
icursor.execute(
"""SELECT t.id as station,
round(min(tmpf)::numeric,0) as min_tmpf,
count(tmpf) as obs FROM current_log c, stations t
WHERE t.iemid = c.iemid and t.network = 'AWOS' and valid > %s and
valid < %s and tmpf > -99 GROUP by t,id""",
(sts6z, ets),
)
for row in icursor:
if qdict.get(row[0], {}).get("tmpf"):
continue
lows[row[0]] = row[1]
ids = list(nt.sts.keys())
ids.sort()
for sid in ids:
myP = pcpn.get(sid, "M")
myH = highs.get(sid, "M")
myL = lows.get(sid, "M")
out.write(fmt % (sid, nt.sts[sid]["name"], myH, myL, myP, "M", "M"))
out.write(".END\n")
out.close()
cmd = (
"pqinsert -p 'plot ac %s0000 awos_rtp_00z.shef "
"awos_rtp_00z.shef shef' /tmp/awos_rtp.shef"
) % (ets.strftime("%Y%m%d"),)
subprocess.call(cmd, shell=True)
def main():
"""Go Main Go"""
nt = network.Table("AWOS")
qdict = loadqc()
pgconn = get_dbconn('iem', user='******')
icursor = pgconn.cursor()
# We run at 12z
now12z = datetime.datetime.utcnow()
now12z = now12z.replace(hour=12, minute=0, second=0, microsecond=0,
tzinfo=pytz.utc)
today6z = now12z.replace(hour=6)
today0z = now12z.replace(hour=0)
yesterday6z = today6z - datetime.timedelta(days=1)
yesterday12z = now12z - datetime.timedelta(days=1)
fmt = "%-6s:%-19s: %3s / %3s / %5s / %4s / %2s\n"
shef_fn = "/tmp/awos_rtp.shef"
out = open(shef_fn, 'w')
out.write(("\n"
"\n"
"\n"
".BR DMX %s Z DH06/TAIRVX/DH12/TAIRVP/PPDRVZ/SFDRVZ/SDIRVZ\n"
": IOWA AWOS RTP FIRST GUESS PROCESSED BY THE IEM\n"
": 06Z to 06Z HIGH TEMPERATURE FOR %s\n"
": 00Z TO 12Z TODAY LOW TEMPERATURE\n"
": 12Z YESTERDAY TO 12Z TODAY RAINFALL\n"
": ...BASED ON REPORTED OBS...\n"
) % (now12z.strftime("%m%d"),
yesterday6z.strftime("%d %b %Y").upper()))
# 6z to 6z high temperature
highs = {}
sql = """SELECT id,
round(max(tmpf)::numeric,0) as max_tmpf,
count(tmpf) as obs FROM current_log c, stations t
WHERE t.iemid = c.iemid and t.network = 'AWOS' and valid >= %s
and valid < %s
and tmpf > -99 GROUP by id """
args = (yesterday6z, today6z)
icursor.execute(sql, args)
for row in icursor:
if qdict.get(row[0], {}).get('tmpf'):
continue
highs[row[0]] = row[1]
# 12z to 12z precip
pcpn = {}
sql = """
select id, sum(precip) from
(select id, extract(hour from valid) as hour,
max(phour) as precip from current_log c, stations t
WHERE t.network = 'AWOS' and t.iemid = c.iemid
and valid >= %s and valid < %s
GROUP by id, hour) as foo
GROUP by id
"""
args = (yesterday12z, now12z)
icursor.execute(sql, args)
for row in icursor:
if qdict.get(row[0], {}).get('precip'):
continue
pcpn[row[0]] = "%5.2f" % (row[1],)
# 0z to 12z low temperature
lows = {}
sql = """
SELECT id, round(min(tmpf)::numeric,0) as min_tmpf,
count(tmpf) as obs FROM
current_log c JOIN stations t on (t.iemid = c.iemid)
WHERE t.network = 'AWOS' and valid >= %s
and valid < %s and tmpf > -99 GROUP by id
"""
args = (today0z, now12z)
icursor.execute(sql, args)
for row in icursor:
if qdict.get(row[0], {}).get('tmpf'):
continue
lows[row[0]] = row[1]
ids = list(nt.sts.keys())
ids.sort()
for myid in ids:
out.write(fmt % (myid, nt.sts[myid]["name"], highs.get(myid, 'M'),
lows.get(myid, 'M'), pcpn.get(myid, 'M'), "M", "M"))
out.write(".END\n")
out.close()
cmd = ("/home/ldm/bin/pqinsert -p 'plot ac %s0000 awos_rtp.shef "
"awos_rtp.shef shef' %s") % (now12z.strftime("%Y%m%d"), shef_fn)
subprocess.call(cmd, shell=True)
os.unlink(shef_fn)
def get_data(ts):
""" Get the data for this timestamp """
iemcursor = IEM.cursor()
cursor = ISUAG.cursor(cursor_factory=psycopg2.extras.DictCursor)
qcdict = loadqc()
nt = NetworkTable("ISUSM", only_online=False)
data = {"type": "FeatureCollection", "features": []}
# Fetch the daily values
iemcursor.execute(
"""
SELECT id, pday, max_tmpf, min_tmpf from summary s JOIN stations t
on (t.iemid = s.iemid) WHERE t.network = 'ISUSM' and day = %s
""",
(ts.date(),),
)
daily = {}
for row in iemcursor:
daily[row[0]] = {
"pday": row[1],
"max_tmpf": row[2],
"min_tmpf": row[3],
}
cursor.execute(
"""
SELECT h.station,
h.encrh_avg,
coalesce(m.rh_avg_qc, h.rh_qc) as rh,
h.rain_mm_tot,
etalfalfa,
battv_min,
coalesce(m.slrkj_tot_qc * 3600 / 1000000, h.slrmj_tot_qc) as slrmj_tot,
coalesce(m.tair_c_avg, h.tair_c_avg) as tair_c_avg,
coalesce(m.tsoil_c_avg_qc, h.tsoil_c_avg_qc) as tsoil_c_avg_qc,
coalesce(m.t12_c_avg_qc, h.t12_c_avg_qc) as t12_c_avg_qc,
coalesce(m.t24_c_avg_qc, h.t24_c_avg_qc) as t24_c_avg_qc,
coalesce(m.t50_c_avg_qc, h.t50_c_avg_qc) as t50_c_avg_qc,
coalesce(m.calcvwc12_avg_qc, h.calc_vwc_12_avg_qc)
as calc_vwc_12_avg_qc,
coalesce(m.calcvwc24_avg_qc, h.calc_vwc_24_avg_qc)
as calc_vwc_24_avg_qc,
coalesce(m.calcvwc50_avg_qc, h.calc_vwc_50_avg_qc)
as calc_vwc_50_avg_qc,
coalesce(m.ws_mph_max, h.ws_mph_max) as ws_mph_max,
coalesce(m.winddir_d1_wvt, h.winddir_d1_wvt) as winddir_d1_wvt,
coalesce(m.ws_mph_s_wvt * 0.447, h.ws_mps_s_wvt)as ws_mps_s_wvt
from sm_hourly h LEFT JOIN sm_minute m on (h.station = m.station and
h.valid = m.valid)
where h.valid = %s
""",
(ts,),
)
for row in cursor:
sid = row["station"]
if sid not in nt.sts:
continue
lon = nt.sts[sid]["lon"]
lat = nt.sts[sid]["lat"]
q = qcdict.get(sid, {})
data["features"].append(
{
"type": "Feature",
"id": sid,
"properties": {
"encrh_avg": (
"%s%%" % safe(row["encrh_avg"], 1)
if row["encrh_avg"] is not None
and row["encrh_avg"] > 0
else "M"
),
"rh": "%s%%" % (safe(row["rh"], 0),),
"hrprecip": (
safe_p(row["rain_mm_tot"])
if not q.get("precip", False)
else "M"
),
"et": safe_p(row["etalfalfa"]),
"bat": safe(row["battv_min"], 2),
"radmj": safe(row["slrmj_tot"], 2),
"tmpf": safe_t(row["tair_c_avg"]),
"high": safe_t(
daily.get(sid, {}).get("max_tmpf", None), "F"
),
"low": safe_t(
daily.get(sid, {}).get("min_tmpf", None), "F"
),
"pday": (
safe(daily.get(sid, {}).get("pday", None), 2)
if not q.get("precip", False)
else "M"
),
"soil04t": (
safe_t(row["tsoil_c_avg_qc"])
if not q.get("soil4", False)
else "M"
),
"soil12t": (
safe_t(row["t12_c_avg_qc"])
if not q.get("soil12", False)
else "M"
),
"soil24t": (
safe_t(row["t24_c_avg_qc"])
if not q.get("soil24", False)
else "M"
),
"soil50t": (
safe_t(row["t50_c_avg_qc"])
if not q.get("soil50", False)
else "M"
),
"soil12m": (
safe_m(row["calc_vwc_12_avg_qc"])
if not q.get("soil12", False)
else "M"
),
"soil24m": (
safe_m(row["calc_vwc_24_avg_qc"])
if not q.get("soil24", False)
else "M"
),
"soil50m": (
safe_m(row["calc_vwc_50_avg_qc"])
if not q.get("soil50", False)
else "M"
),
"gust": safe(row["ws_mph_max"], 1),
"wind": ("%s@%.0f")
% (
drct2text(row["winddir_d1_wvt"]),
row["ws_mps_s_wvt"] * 2.23,
),
"name": nt.sts[sid]["name"],
},
"geometry": {"type": "Point", "coordinates": [lon, lat]},
}
)
return json.dumps(data)
import os
import subprocess
import datetime
import sys
import tempfile
import pyiem.tracker as tracker
qc = tracker.loadqc()
import psycopg2
IEM = psycopg2.connect(database="iem", host="iemdb", user="******")
icursor = IEM.cursor()
icursor.execute(
"""SELECT t.id as station from current c, stations t
WHERE t.network = 'KCCI' and
valid > 'TODAY' and t.iemid = c.iemid ORDER by gust DESC"""
)
data = {}
data["timestamp"] = datetime.datetime.now()
i = 1
for row in icursor:
if i == 6:
break
if qc.get(row[0], {}).get("wind", False):
continue
data["sid%s" % (i,)] = row[0]
i += 1
if "sid5" not in data:
"""Generate a First Guess RTP that the bureau can use for their product
"""
import datetime
import subprocess
import pytz
import psycopg2
from pyiem.tracker import loadqc
from pyiem import network
nt = network.Table("AWOS")
qdict = loadqc()
IEM = psycopg2.connect(database='iem', host='iemdb', user='******')
icursor = IEM.cursor()
utc = datetime.datetime.utcnow()
ets = utc.replace(tzinfo=pytz.timezone("UTC"),
hour=0,
minute=0,
second=0,
microsecond=0)
sts12z = ets + datetime.timedelta(hours=-12)
sts6z = ets + datetime.timedelta(hours=-18)
sts24h = ets + datetime.timedelta(days=-1)
fmt = "%-6s:%-19s: %3s / %3s / %5s / %4s / %2s\n"
out = open("/tmp/awos_rtp.shef", 'w')
out.write("""
.BR DMX %s Z DH00/TAIRVS/TAIRVI/PPDRVZ/SFDRVZ/SDIRVZ
def main(argv):
"""Go Main Go"""
nt = Table("ISUSM")
qdict = loadqc()
idbconn = get_dbconn('isuag', user='******')
icursor = idbconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
pdbconn = get_dbconn('postgis', user='******')
pcursor = pdbconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
day_ago = int(argv[1])
ts = datetime.datetime.now() - datetime.timedelta(days=day_ago)
# Query out the data
soil_obs = []
lats = []
lons = []
icursor.execute("""
SELECT station, tsoil_c_avg_qc from sm_daily
where valid = '%s' and tsoil_c_avg_qc > -40
and station not in ('AHTI4', 'FRUI4')
""" % (ts.strftime("%Y-%m-%d"), ))
for row in icursor:
stid = row['station']
if qdict.get(stid, {}).get('soil4', False):
# print '%s was QCd out' % (stid,)
continue
soil_obs.append(temperature(row['tsoil_c_avg_qc'], 'C').value('F'))
lats.append(nt.sts[stid]['lat'])
lons.append(nt.sts[stid]['lon'])
if len(lats) < 5:
print(("isuag/fancy_4inch found %s obs for %s") %
(len(lats), ts.strftime("%Y-%m-%d")))
return
# Grid it
# numxout = 40
# numyout = 40
# xmin = min(lons) - 2.
# ymin = min(lats) - 2.
# xmax = max(lons) + 2.
# ymax = max(lats) + 2.
# xc = (xmax-xmin)/(numxout-1)
# yc = (ymax-ymin)/(numyout-1)
# xo = xmin + xc * np.arange(0, numxout)
# yo = ymin + yc * np.arange(0, numyout)
# analysis = griddata((lons, lats), soil_obs, (xo, yo) )
# rbfi = Rbf(lons, lats, soil_obs, function='cubic')
# analysis = rbfi(xo, yo)
nn = NearestNDInterpolator((lons, lats), np.array(soil_obs))
# analysis = nn(xo, yo)
# Query out centroids of counties...
pcursor.execute("""SELECT ST_x(ST_centroid(the_geom)) as lon,
ST_y(ST_centroid(the_geom)) as lat
from uscounties WHERE state_name = 'Iowa'
""")
clons = []
clats = []
for row in pcursor:
clats.append(row['lat'])
clons.append(row['lon'])
cobs = nn(clons, clats)
mp = MapPlot(sector='iowa',
title=("Iowa Average 4 inch Soil Temperatures %s") %
(ts.strftime("%b %d %Y"), ),
subtitle=("Based on gridded analysis (black numbers) of "
"ISUSM network observations (red numbers)"))
mp.contourf(clons,
clats,
cobs,
np.arange(10, 101, 5),
cmap=cm.get_cmap('jet'),
units=r'$^\circ$F')
mp.plot_values(lons, lats, soil_obs, fmt='%.0f', color='r', labelbuffer=5)
mp.plot_values(clons, clats, cobs, fmt='%.0f', textsize=11, labelbuffer=5)
# for lo, la, ob in zip(clons, clats, cobs):
# xi, yi = m.map(lo, la)
# txt = m.ax.text(xi, yi, "%.0f" % (ob,))
mp.drawcounties()
routes = "a" if day_ago >= 4 else "ac"
pqstr = ("plot %s %s0000 soilt_day%s.png isuag_county_4inch_soil.png png"
) % (routes, ts.strftime("%Y%m%d"), day_ago)
mp.postprocess(pqstr=pqstr)
mp.close()
import os
import subprocess
import datetime
import tempfile
import pyiem.tracker as tracker
qc = tracker.loadqc()
import psycopg2
IEM = psycopg2.connect(database="iem", host='iemdb')
icursor = IEM.cursor()
now = datetime.datetime.now()
icursor.execute("""SELECT s.id as station from summary_%s c, stations s WHERE
s.network = 'KCCI' and s.iemid = c.iemid and
day = 'TODAY' ORDER by min_tmpf ASC""" % (now.year, ))
data = {}
data['timestamp'] = now
i = 1
for row in icursor:
if i == 6:
break
if qc.get(row[0], {}).get('tmpf', False):
continue
data['sid%s' % (i, )] = row[0]
i += 1
fd, path = tempfile.mkstemp()
os.write(fd, open('top5lows.tpl', 'r').read() % data)
os.close(fd)
subprocess.call("/home/ldm/bin/pqinsert -p 'auto_top5lows.scn' %s" % (path, ),
def main():
"""Go Main Go"""
now = datetime.datetime.now()
qdict = loadqc()
pgconn = get_dbconn("iem", user="******")
icursor = pgconn.cursor()
sql = """
select s.id,
ST_x(s.geom) as lon, ST_y(s.geom) as lat,
max_tmpf as high, s.network
from summary c, stations s
WHERE c.iemid = s.iemid and day = 'TODAY' and max_tmpf > -40
and s.network in ('IA_ASOS', 'AWOS', 'IL_ASOS','MO_ASOS','KS_ASOS',
'NE_ASOS','SD_ASOS','MN_ASOS','WI_ASOS') ORDER by high ASC
"""
lats = []
lons = []
vals = []
valmask = []
labels = []
icursor.execute(sql)
dsm = None
for row in icursor:
if row[0] == "DSM":
dsm = row[3]
if qdict.get(row[0], {}).get("tmpf") is not None:
continue
lats.append(row[2])
lons.append(row[1])
vals.append(row[3])
labels.append(row[0])
valmask.append(row[4] in ["AWOS", "IA_ASOS"])
if len(lats) < 4:
return
mp = MapPlot(
sector="iowa",
title=("%s Iowa ASOS/AWOS High Temperature")
% (now.strftime("%-d %b %Y"),),
subtitle="map valid: %s" % (now.strftime("%d %b %Y %-I:%M %p"),),
)
# m.debug = True
if dsm is None:
dsm = vals[0]
bottom = int(dsm) - 15
top = int(dsm) + 15
bins = np.linspace(bottom, top, 11)
cmap = cm.get_cmap("jet")
mp.contourf(lons, lats, vals, bins, units="F", cmap=cmap)
mp.plot_values(
lons,
lats,
vals,
"%.0f",
valmask=valmask,
labels=labels,
labelbuffer=10,
)
mp.drawcounties()
pqstr = "plot ac %s summary/iowa_asos_high.png iowa_asos_high.png png" % (
now.strftime("%Y%m%d%H%M"),
)
mp.postprocess(pqstr=pqstr)
mp.close()
def main(argv):
"""Go Main Go"""
nt = Table("ISUSM")
qdict = loadqc()
idbconn = get_dbconn("isuag", user="******")
pdbconn = get_dbconn("postgis", user="******")
day_ago = int(argv[1])
ts = datetime.date.today() - datetime.timedelta(days=day_ago)
hlons, hlats, hvals = do_nam(ts)
nam = temperature(hvals, "K").value("F")
window = np.ones((3, 3))
nam = convolve2d(nam, window / window.sum(), mode="same", boundary="symm")
# mp = MapPlot(sector='midwest')
# mp.pcolormesh(hlons, hlats, nam,
# range(20, 90, 5))
# mp.postprocess(filename='test.png')
# sys.exit()
# Query out the data
df = read_sql(
"""
WITH ranges as (
select station, count(*), min(tsoil_c_avg_qc),
max(tsoil_c_avg_qc) from sm_hourly WHERE
valid >= %s and valid < %s and tsoil_c_avg_qc > -40
and tsoil_c_avg_qc < 50 GROUP by station
)
SELECT d.station, d.tsoil_c_avg_qc,
r.max as hourly_max_c, r.min as hourly_min_c, r.count
from sm_daily d JOIN ranges r on (d.station = r.station)
where valid = %s and tsoil_c_avg_qc > -40 and r.count > 19
""",
idbconn,
params=(ts, ts + datetime.timedelta(days=1), ts),
index_col="station",
)
for col, newcol in zip(
["tsoil_c_avg_qc", "hourly_min_c", "hourly_max_c"],
["ob", "min", "max"],
):
df[newcol] = temperature(df[col].values, "C").value("F")
df.drop(col, axis=1, inplace=True)
for stid, row in df.iterrows():
df.at[stid, "ticket"] = qdict.get(stid, {}).get("soil4", False)
x, y = get_idx(hlons, hlats, nt.sts[stid]["lon"], nt.sts[stid]["lat"])
df.at[stid, "nam"] = nam[x, y]
df.at[stid, "lat"] = nt.sts[stid]["lat"]
df.at[stid, "lon"] = nt.sts[stid]["lon"]
# ticket is an object type from above
df = df[~df["ticket"].astype("bool")]
df["diff"] = df["ob"] - df["nam"]
bias = df["diff"].mean()
nam = nam + bias
print("fancy_4inch NAM bias correction of: %.2fF applied" % (bias, ))
# apply nam bias to sampled data
df["nam"] += bias
df["diff"] = df["ob"] - df["nam"]
# we are going to require data be within 1 SD of sampled or 5 deg
std = 5.0 if df["nam"].std() < 5.0 else df["nam"].std()
for station in df[df["diff"].abs() > std].index.values:
print(("fancy_4inch %s QC'd %s out std: %.2f, ob:%.1f nam:%.1f") % (
ts.strftime("%Y%m%d"),
station,
std,
df.at[station, "ob"],
df.at[station, "nam"],
))
df.drop(station, inplace=True)
# Query out centroids of counties...
cdf = read_sql(
"""SELECT ST_x(ST_centroid(the_geom)) as lon,
ST_y(ST_centroid(the_geom)) as lat
from uscounties WHERE state_name = 'Iowa'
""",
pdbconn,
index_col=None,
)
for i, row in cdf.iterrows():
x, y = get_idx(hlons, hlats, row["lon"], row["lat"])
cdf.at[i, "nam"] = nam[x, y]
mp = MapPlot(
sector="iowa",
title=("Average 4 inch Depth Soil Temperatures for %s") %
(ts.strftime("%b %d, %Y"), ),
subtitle=("County est. based on bias adj. "
"NWS NAM Model (black numbers), "
"ISUSM network observations (red numbers)"),
)
mp.pcolormesh(
hlons,
hlats,
nam,
np.arange(10, 101, 5),
cmap=cm.get_cmap("jet"),
units=r"$^\circ$F",
)
mp.plot_values(df["lon"],
df["lat"],
df["ob"],
fmt="%.0f",
color="r",
labelbuffer=5)
mp.plot_values(
cdf["lon"],
cdf["lat"],
cdf["nam"],
fmt="%.0f",
textsize=11,
labelbuffer=5,
)
mp.drawcounties()
routes = "a" if day_ago >= 4 else "ac"
pqstr = ("plot %s %s0000 soilt_day%s.png isuag_county_4inch_soil.png png"
) % (routes, ts.strftime("%Y%m%d"), day_ago)
mp.postprocess(pqstr=pqstr)
mp.close()
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
ctx["qc"] = loadqc(date=ctx["date"])
ctx["pgconn"] = get_dbconn("isuag")
ctx["nt"] = NetworkTable("ISUSM")
if not ctx["nt"].sts:
raise NoDataFound("No station metadata found.")
# Adjust stations to make some room
ctx["nt"].sts["BOOI4"]["lon"] -= 0.15
ctx["nt"].sts["BOOI4"]["lat"] -= 0.15
ctx["nt"].sts["AHTI4"]["lon"] += 0.25
ctx["nt"].sts["AHTI4"]["lat"] += 0.25
title = "TBD"
subtitle = "TBD"
if ctx["opt"] == "1":
title = "ISU Soil Moisture Max/Min 4 Inch Soil Temperature"
subtitle = "based on available hourly observations"
data, df = plot1(ctx)
elif ctx["opt"] == "2":
title = "ISU Soil Moisture Max/Min Air Temperature"
subtitle = "based on available daily summary data"
data, df = plot2(ctx)
elif ctx["opt"] == "3":
title = "ISU Soil Moisture Average 4 Inch Soil Temperature"
subtitle = "based on available daily summary data"
data, df = plot3(ctx)
elif ctx["opt"] == "4":
title = "ISU Soil Moisture Solar Radiation [MJ]"
subtitle = "based on available daily summary data"
data, df = plot4(ctx)
elif ctx["opt"] == "5":
title = "ISU Soil Moisture Potential Evapotranspiration [inch]"
subtitle = "based on available daily summary data"
data, df = plot5(ctx, "dailyet")
elif ctx["opt"] == "6":
title = "ISU Soil Moisture Precipitation [inch]"
subtitle = (
"based on available daily summary data, liquid equiv of snow "
"estimated")
data, df = plot5(ctx, "rain_mm_tot")
elif ctx["opt"] == "7":
title = "ISU Soil Moisture Peak Wind Gust [MPH]"
subtitle = "based on available daily summary data"
data, df = plot7(ctx)
elif ctx["opt"] == "8":
title = "ISU Soil Moisture Average Wind Speed [MPH]"
subtitle = "based on available daily summary data"
data, df = plot8(ctx)
tle = ctx["date"].strftime("%b %-d, %Y")
mp = MapPlot(
sector="iowa",
continentalcolor="white",
nocaption=True,
title="%s %s" % (tle, title),
subtitle=subtitle,
)
mp.drawcounties("#EEEEEE")
mp.plot_station(data, fontsize=12)
return mp.fig, df
"""
Generate a RTP product for the weather bureau as my database as more AWOS
obs than what they get
"""
import datetime
import pytz
import os
import subprocess
from pyiem.tracker import loadqc
from pyiem import network
nt = network.Table("AWOS")
qdict = loadqc()
import psycopg2
IEM = psycopg2.connect(database='iem', host='iemdb', user='******')
icursor = IEM.cursor()
# We run at 12z
now12z = datetime.datetime.utcnow()
now12z = now12z.replace(hour=12, minute=0, second=0, microsecond=0,
tzinfo=pytz.timezone("UTC"))
today6z = now12z.replace(hour=6)
today0z = now12z.replace(hour=0)
yesterday6z = today6z - datetime.timedelta(days=1)
yesterday12z = now12z - datetime.timedelta(days=1)
fmt = "%-6s:%-19s: %3s / %3s / %5s / %4s / %2s\n"
SHEF_FN = "/tmp/awos_rtp.shef"
out = open(SHEF_FN, 'w')
def get_data(ts):
""" Get the data for this timestamp """
iemcursor = IEM.cursor()
cursor = ISUAG.cursor(cursor_factory=psycopg2.extras.DictCursor)
qcdict = loadqc()
nt = NetworkTable("ISUSM")
data = {"type": "FeatureCollection",
"features": []}
# Fetch the daily values
iemcursor.execute("""
SELECT id, pday, max_tmpf, min_tmpf from summary s JOIN stations t
on (t.iemid = s.iemid) WHERE t.network = 'ISUSM' and day = %s
""", (ts.date(),))
daily = {}
for row in iemcursor:
daily[row[0]] = {'pday': row[1], 'max_tmpf': row[2],
'min_tmpf': row[3]}
cursor.execute("""
SELECT * from sm_hourly where valid = %s
""", (ts,))
for row in cursor:
sid = row['station']
lon = nt.sts[sid]['lon']
lat = nt.sts[sid]['lat']
q = qcdict.get(sid, {})
data['features'].append(
{"type": "Feature",
"id": sid,
"properties": {"encrh_avg": ("%s%%" % safe(row['encrh_avg'], 1)
if row['encrh_avg'] is not None
and row['encrh_avg'] > 0 else "M"),
"rh": "%s%%" % (safe(row["rh"], 0),),
"hrprecip": (safe_p(row['rain_mm_tot'])
if not q.get('precip', False)
else 'M'),
"et": safe_p(row['etalfalfa']),
"bat": safe(row['battv_min'], 2),
"radmj": safe(row['slrmj_tot'], 2),
"tmpf": safe_t(row['tair_c_avg']),
"high": safe_t(daily.get(sid,
{}).get('max_tmpf',
None), 'F'),
"low": safe_t(daily.get(sid,
{}).get('min_tmpf',
None), 'F'),
"pday": (safe(daily.get(sid,
{}).get('pday', None),
2)
if not q.get('precip', False) else 'M'),
"soil04t": (safe_t(row['tsoil_c_avg_qc'])
if not q.get('soil4', False) else 'M'),
"soil12t": (safe_t(row['t12_c_avg_qc'])
if not q.get('soil12', False)
else 'M'),
"soil24t": (safe_t(row['t24_c_avg_qc'])
if not q.get('soil24', False)
else 'M'),
"soil50t": (safe_t(row['t50_c_avg_qc'])
if not q.get('soil50', False)
else 'M'),
"soil12m": (safe_m(row['calc_vwc_12_avg_qc'])
if not q.get('soil12', False)
else 'M'),
"soil24m": (safe_m(row['calc_vwc_24_avg_qc'])
if not q.get('soil24', False)
else 'M'),
"soil50m": (safe_m(row['calc_vwc_50_avg_qc'])
if not q.get('soil50', False)
else 'M'),
"gust": safe(row['ws_mph_max'], 1),
"wind": ("%s@%.0f"
) % (drct2text(row['winddir_d1_wvt']),
row['ws_mps_s_wvt'] * 2.23),
'name': nt.sts[sid]['name']
},
"geometry": {"type": "Point",
"coordinates": [lon, lat]}
})
ssw(json.dumps(data))