def do(netname, pname):
"""Do something please"""
mcursor = MESOSITE.cursor()
mcursor.execute("""
SELECT id, network, name from webcams where
network = %s
and online ORDER by id ASC
""", (netname, ))
NT = NetworkTable(None)
obs = {}
for row in mcursor:
NT.sts[row[0]] = dict(id=row[0], network=row[1], name=row[2],
tzname='America/Chicago')
fn = "%s/%s.jpg" % (mydir, row[0])
if not os.path.isfile(fn):
print 'Missing webcam file: %s' % (fn,)
continue
ticks = os.stat(fn)[stat.ST_MTIME]
valid = (datetime.datetime(1970, 1, 1) +
datetime.timedelta(seconds=ticks))
valid = valid.replace(tzinfo=pytz.timezone("UTC"))
obs[row[0]] = dict(valid=valid)
# Abort out if no obs are found
if len(obs) == 0:
return
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor(), 10)
tracker.process_network(obs, pname, NT, threshold)
tracker.send_emails()
IEM.commit()
PORTFOLIO.commit()
def do_iemtracker(obs):
"""Iterate over the obs and do IEM Tracker related activities """
threshold = datetime.datetime.utcnow() - datetime.timedelta(hours=3)
threshold = threshold.replace(tzinfo=pytz.timezone("UTC"))
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor())
tracker.process_network(obs, 'iarwis', NT, threshold)
tracker.send_emails()
IEM.commit()
PORTFOLIO.commit()
def main():
"""Go Main Go"""
NT = NetworkTable("ISUSM")
IEM = get_dbconn("iem")
PORTFOLIO = get_dbconn("portfolio")
threshold = datetime.datetime.utcnow() - datetime.timedelta(hours=12)
threshold = threshold.replace(tzinfo=pytz.UTC)
icursor = IEM.cursor()
icursor.execute(
"""
SELECT id, valid from current c JOIN stations t
ON (t.iemid = c.iemid) WHERE t.network = 'ISUSM'
"""
)
obs = {}
for row in icursor:
obs[row[0]] = dict(id=row[0], valid=row[1])
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor(), 7)
tracker.process_network(obs, "isusm", NT, threshold)
tracker.send_emails()
tac = tracker.action_count
if tac > 6:
print("check_isusm_online.py had %s actions, did not email" % (tac,))
IEM.commit()
PORTFOLIO.commit()
def main():
"""Go Main Go"""
NT = NetworkTable("AWOS")
IEM = get_dbconn('iem')
PORTFOLIO = get_dbconn('portfolio')
threshold = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
threshold = threshold.replace(tzinfo=pytz.utc)
icursor = IEM.cursor()
icursor.execute("""
SELECT id, valid from current c JOIN stations t
ON (t.iemid = c.iemid) WHERE t.network = 'AWOS'
""")
obs = {}
for row in icursor:
obs[row[0]] = dict(id=row[0], valid=row[1])
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor(), 10)
tracker.process_network(obs, 'iaawos', NT, threshold)
tracker.send_emails()
tac = tracker.action_count
if tac > 6:
print('check_awos_online.py had %s actions, did not email' % (tac,))
IEM.commit()
PORTFOLIO.commit()
def do(netname, pname):
"""Do something please"""
mcursor = MESOSITE.cursor()
mcursor.execute("""
SELECT id, network, name from webcams where
network = %s
and online ORDER by id ASC
""", (netname, ))
NT = NetworkTable(None)
obs = {}
missing = 0
for row in mcursor:
NT.sts[row[0]] = dict(id=row[0], network=row[1], name=row[2],
tzname='America/Chicago')
fn = "%s/%s.jpg" % (mydir, row[0])
if not os.path.isfile(fn):
missing += 1
if missing > 1:
print 'Missing webcam file: %s' % (fn,)
continue
ticks = os.stat(fn)[stat.ST_MTIME]
valid = (datetime.datetime(1970, 1, 1) +
datetime.timedelta(seconds=ticks))
valid = valid.replace(tzinfo=pytz.timezone("UTC"))
obs[row[0]] = dict(valid=valid)
# Abort out if no obs are found
if len(obs) == 0:
return
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor(), 10)
tracker.process_network(obs, pname, NT, threshold)
tracker.send_emails()
IEM.commit()
PORTFOLIO.commit()
def do_iemtracker(obs):
"""Iterate over the obs and do IEM Tracker related activities """
threshold = datetime.datetime.utcnow() - datetime.timedelta(hours=3)
threshold = threshold.replace(tzinfo=pytz.UTC)
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor())
tracker.process_network(obs, 'iarwis', NT, threshold)
tracker.send_emails()
IEM.commit()
PORTFOLIO.commit()
def workflow(netname, pname):
"""Do something please"""
pgconn_iem = get_dbconn("iem")
pgconn_mesosite = get_dbconn("mesosite")
pgconn_portfolio = get_dbconn("portfolio")
# Now lets check files
mydir = "/home/ldm/data/camera/stills"
threshold = datetime.datetime.utcnow() - datetime.timedelta(hours=2)
threshold = threshold.replace(tzinfo=pytz.UTC)
mcursor = pgconn_mesosite.cursor()
mcursor.execute(
"""
SELECT id, network, name from webcams where
network = %s
and online ORDER by id ASC
""",
(netname, ),
)
nt = NetworkTable(None)
obs = {}
missing = 0
for row in mcursor:
nt.sts[row[0]] = dict(id=row[0],
network=row[1],
name=row[2],
tzname="America/Chicago")
fn = "%s/%s.jpg" % (mydir, row[0])
if not os.path.isfile(fn):
missing += 1
if missing > 1:
print("Missing webcam file: %s" % (fn, ))
continue
ticks = os.stat(fn)[stat.ST_MTIME]
valid = datetime.datetime(1970, 1,
1) + datetime.timedelta(seconds=ticks)
valid = valid.replace(tzinfo=pytz.UTC)
obs[row[0]] = dict(valid=valid)
# Abort out if no obs are found
if not obs:
return
tracker = TrackerEngine(pgconn_iem.cursor(), pgconn_portfolio.cursor(), 10)
tracker.process_network(obs, pname, nt, threshold)
tracker.send_emails()
pgconn_iem.commit()
pgconn_portfolio.commit()
def test_workflow(pcursor, icursor):
""" Test that we can do stuff! """
sid1 = 'XXX'
sid2 = 'YYY'
pnetwork = 'xxxxxx'
nt = NetworkTable(None)
nt.sts[sid1] = dict(name='XXX Site Name', network='IA_XXXX',
tzname='America/Chicago')
nt.sts[sid2] = dict(name='YYY Site Name', network='IA_XXXX',
tzname='America/Chicago')
valid = datetime.datetime.utcnow()
valid = valid.replace(tzinfo=pytz.timezone("UTC"))
threshold = valid - datetime.timedelta(hours=3)
obs = {sid1: {'valid': valid},
sid2: {'valid': valid - datetime.timedelta(hours=6)}}
# Create dummy iem_site_contacts
pcursor.execute("""
INSERT into iem_site_contacts
(portfolio, s_mid, email) VALUES (%s, %s, %s)
""", (pnetwork, sid1, 'akrherz@localhost'))
pcursor.execute("""
INSERT into iem_site_contacts
(portfolio, s_mid, email) VALUES (%s, %s, %s)
""", (pnetwork, sid2, 'root@localhost'))
# Create some dummy tickets
pcursor.execute("""
INSERT into tt_base (portfolio, s_mid, subject,
status, author) VALUES (%s, %s, %s, %s, %s) RETURNING id
""", (pnetwork, sid1, 'FIXME PLEASE OPEN', 'OPEN', 'mesonet'))
pcursor.execute("""
INSERT into tt_base (portfolio, s_mid, subject,
status, author) VALUES (%s, %s, %s, %s, %s) RETURNING id
""", (pnetwork, sid1, 'FIXME PLEASE CLOSED', 'CLOSED', 'mesonet'))
tracker = TrackerEngine(icursor, pcursor)
tracker.process_network(obs, pnetwork, nt, threshold)
tracker.send_emails(really_send=False)
assert len(tracker.emails) == 1
tracker.emails = {}
obs[sid1]['valid'] = valid - datetime.timedelta(hours=6)
obs[sid2]['valid'] = valid
tracker.process_network(obs, pnetwork, nt, threshold)
tracker.send_emails(really_send=False)
assert len(tracker.emails) == 2
tracker.emails = {}
obs[sid1]['valid'] = valid - datetime.timedelta(hours=6)
obs[sid2]['valid'] = valid
tracker.process_network(obs, pnetwork, nt, threshold)
tracker.send_emails(really_send=False)
assert not tracker.emails
def test_workflow(pcursor, icursor):
""" Test that we can do stuff! """
sid1 = 'XXX'
sid2 = 'YYY'
pnetwork = 'xxxxxx'
nt = NetworkTable(None)
nt.sts[sid1] = dict(name='XXX Site Name',
network='IA_XXXX',
tzname='America/Chicago')
nt.sts[sid2] = dict(name='YYY Site Name',
network='IA_XXXX',
tzname='America/Chicago')
valid = datetime.datetime.utcnow()
valid = valid.replace(tzinfo=pytz.timezone("UTC"))
threshold = valid - datetime.timedelta(hours=3)
obs = {
sid1: {
'valid': valid
},
sid2: {
'valid': valid - datetime.timedelta(hours=6)
}
}
# Create dummy iem_site_contacts
pcursor.execute(
"""
INSERT into iem_site_contacts
(portfolio, s_mid, email) VALUES (%s, %s, %s)
""", (pnetwork, sid1, 'akrherz@localhost'))
pcursor.execute(
"""
INSERT into iem_site_contacts
(portfolio, s_mid, email) VALUES (%s, %s, %s)
""", (pnetwork, sid2, 'root@localhost'))
# Create some dummy tickets
pcursor.execute(
"""
INSERT into tt_base (portfolio, s_mid, subject,
status, author) VALUES (%s, %s, %s, %s, %s) RETURNING id
""", (pnetwork, sid1, 'FIXME PLEASE OPEN', 'OPEN', 'mesonet'))
pcursor.execute(
"""
INSERT into tt_base (portfolio, s_mid, subject,
status, author) VALUES (%s, %s, %s, %s, %s) RETURNING id
""", (pnetwork, sid1, 'FIXME PLEASE CLOSED', 'CLOSED', 'mesonet'))
tracker = TrackerEngine(icursor, pcursor)
tracker.process_network(obs, pnetwork, nt, threshold)
tracker.send_emails(really_send=False)
assert len(tracker.emails) == 1
tracker.emails = {}
obs[sid1]['valid'] = valid - datetime.timedelta(hours=6)
obs[sid2]['valid'] = valid
tracker.process_network(obs, pnetwork, nt, threshold)
tracker.send_emails(really_send=False)
assert len(tracker.emails) == 2
tracker.emails = {}
obs[sid1]['valid'] = valid - datetime.timedelta(hours=6)
obs[sid2]['valid'] = valid
tracker.process_network(obs, pnetwork, nt, threshold)
tracker.send_emails(really_send=False)
assert not tracker.emails
def doNetwork(_network, shef_fp, thres, qdict):
"""
Process a schoolnet network and do various things
@param network string network name
@param shef_fp file pointer to shef product file
@param thres time threshold we care about for alerting
@param qdict dictionary of sites we don't care about
"""
now = datetime.datetime.now()
# Get Obs
obs = get_network(_network)
tophour_obs = get_network_recent(_network, now.replace(minute=0))
hr3_obs = get_network_recent(_network, now + datetime.timedelta(hours=-3))
hr6_obs = get_network_recent(_network, now + datetime.timedelta(hours=-6))
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor(), 10)
mynet = NetworkTable(_network)
tracker.process_network(obs, '%ssnet' % (_network.lower(), ), mynet, thres)
tracker.send_emails()
keys = list(obs.keys())
keys.sort()
for nwsli in keys:
ob = obs[nwsli]
if ob['valid'] < thres:
continue
current_pmonth = fetch_pmonth(obs, nwsli)
tophour_pmonth = fetch_pmonth(tophour_obs, nwsli)
hr3_pmonth = fetch_pmonth(hr3_obs, nwsli)
hr6_pmonth = fetch_pmonth(hr6_obs, nwsli)
if nwsli not in moCntr:
moCntr[nwsli] = 0.
yrCntr[nwsli] = 0.
# reset
if moCntr[nwsli] > current_pmonth:
yrCntr[nwsli] += current_pmonth
moCntr[nwsli] = current_pmonth
pcounter = yrCntr[nwsli] + current_pmonth
# Compute offsets
phour = precip_diff(current_pmonth, tophour_pmonth)
p03i = precip_diff(current_pmonth, hr3_pmonth)
p06i = precip_diff(current_pmonth, hr6_pmonth)
pday = ob['pday']
if qdict.get(nwsli, {}).get('precip', False) or _network == 'KIMT':
phour = None
p03i = None
p06i = None
pcounter = None
pday = None
# Finally ready to write SHEF!
shef_fp.write(
("%s %s / %4s / %4s / %4s / %5s : %s\n"
"") % (nwsli, pretty_tmpf(ob), pretty_precip(phour),
pretty_precip(p03i), pretty_precip(p06i),
pretty_precip(pcounter), ob.get('sname', 'Unknown')))
if _network == 'KCCI':
LOCDSMRR5.write(("%s %3s / %5s\n") %
(nwsli, pretty_tmpf(ob), pretty_precip(pcounter)))
# CSV FILE!
utc = ob['valid'].astimezone(pytz.UTC)
CSVFILE.write(
("%s,%s,%s,%s,%i,%s,%s,%s,%s\n") %
(nwsli, utc.strftime("%Y/%m/%d %H:%M:%S"), getm_c(
ob, 'tmpf'), getm_c(ob, 'dwpf'), ob['sknt'], ob['drct'], phour
or 'M', pday or 'M', ob['pres']))
if _network != 'KIMT':
SAOFILE.write(metar(obs[nwsli]))
import pytz
from pyiem.tracker import TrackerEngine
from pyiem.network import Table as NetworkTable
NT = NetworkTable("AWOS")
IEM = psycopg2.connect(database="iem", host="iemdb")
PORTFOLIO = psycopg2.connect(database="portfolio", host="iemdb")
threshold = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
threshold = threshold.replace(tzinfo=pytz.timezone("UTC"))
icursor = IEM.cursor()
icursor.execute(
"""SELECT id, valid from current c JOIN stations t
ON (t.iemid = c.iemid) WHERE t.network = 'AWOS'
"""
)
obs = {}
for row in icursor:
obs[row[0]] = dict(id=row[0], valid=row[1])
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor(), 10)
tracker.process_network(obs, "iaawos", NT, threshold)
tracker.send_emails()
tac = tracker.action_count
if tac > 6:
print "check_awos_online.py had %s actions, did not email" % (tac,)
IEM.commit()
PORTFOLIO.commit()
import datetime
import psycopg2
import pytz
from pyiem.tracker import TrackerEngine
from pyiem.network import Table as NetworkTable
NT = NetworkTable("ISUSM")
IEM = psycopg2.connect(database='iem', host='iemdb')
PORTFOLIO = psycopg2.connect(database='portfolio', host='iemdb')
threshold = datetime.datetime.utcnow() - datetime.timedelta(hours=12)
threshold = threshold.replace(tzinfo=pytz.timezone("UTC"))
icursor = IEM.cursor()
icursor.execute("""SELECT id, valid from current c JOIN stations t
ON (t.iemid = c.iemid) WHERE t.network = 'ISUSM'
""")
obs = {}
for row in icursor:
obs[row[0]] = dict(id=row[0], valid=row[1])
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor(), 7)
tracker.process_network(obs, 'isusm', NT, threshold)
tracker.send_emails()
tac = tracker.action_count
if tac > 6:
print 'check_isusm_online.py had %s actions, did not email' % (tac, )
IEM.commit()
PORTFOLIO.commit()
def doNetwork(_network, shef_fp, thres, qdict):
"""
Process a schoolnet network and do various things
@param network string network name
@param shef_fp file pointer to shef product file
@param thres time threshold we care about for alerting
@param qdict dictionary of sites we don't care about
"""
now = datetime.datetime.now()
# Get Obs
obs = get_network(_network)
tophour_obs = get_network_recent(_network, now.replace(minute=0))
hr3_obs = get_network_recent(_network, now + datetime.timedelta(hours=-3))
hr6_obs = get_network_recent(_network, now + datetime.timedelta(hours=-6))
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor(), 10)
mynet = NetworkTable(_network)
tracker.process_network(obs, '%ssnet' % (_network.lower(), ), mynet, thres)
tracker.send_emails()
keys = obs.keys()
keys.sort()
for nwsli in keys:
ob = obs[nwsli]
if ob['valid'] < thres:
continue
current_pmonth = fetch_pmonth(obs, nwsli)
tophour_pmonth = fetch_pmonth(tophour_obs, nwsli)
hr3_pmonth = fetch_pmonth(hr3_obs, nwsli)
hr6_pmonth = fetch_pmonth(hr6_obs, nwsli)
if nwsli not in moCntr:
moCntr[nwsli] = 0.
yrCntr[nwsli] = 0.
# reset
if moCntr[nwsli] > current_pmonth:
yrCntr[nwsli] += current_pmonth
moCntr[nwsli] = current_pmonth
pcounter = yrCntr[nwsli] + current_pmonth
# Compute offsets
phour = precip_diff(current_pmonth, tophour_pmonth)
p03i = precip_diff(current_pmonth, hr3_pmonth)
p06i = precip_diff(current_pmonth, hr6_pmonth)
pday = ob['pday']
if qdict.get(nwsli, {}).get('precip', False) or _network == 'KIMT':
phour = None
p03i = None
p06i = None
pcounter = None
pday = None
# Finally ready to write SHEF!
shef_fp.write(("%s %s / %4s / %4s / %4s / %5s : %s\n"
"") % (nwsli, pretty_tmpf(ob), pretty_precip(phour),
pretty_precip(p03i), pretty_precip(p06i),
pretty_precip(pcounter),
ob.get('sname', 'Unknown')))
if _network == 'KCCI':
LOCDSMRR5.write("%s %3s / %5s\n" % (nwsli,
pretty_tmpf(ob), pretty_precip(pcounter)))
# CSV FILE!
for fp in [CSVFILE, KELOCSVFILE]:
if _network != 'KELO' and fp == KELOCSVFILE:
continue
utc = ob['valid'].astimezone(pytz.timezone("UTC"))
fp.write(("%s,%s,%s,%s,%i,%s,%s,%s,%s\n"
"") % (nwsli, utc.strftime("%Y/%m/%d %H:%M:%S"),
getm_c(ob, 'tmpf'), getm_c(ob, 'dwpf'),
ob['sknt'], ob['drct'], phour or 'M',
pday or 'M', ob['pres']))
if _network != 'KIMT':
SAOFILE.write(metar(obs[nwsli]))
import datetime
import psycopg2
import pytz
from pyiem.tracker import TrackerEngine
from pyiem.network import Table as NetworkTable
NT = NetworkTable("ISUSM")
IEM = psycopg2.connect(database='iem', host='iemdb')
PORTFOLIO = psycopg2.connect(database='portfolio', host='iemdb')
threshold = datetime.datetime.utcnow() - datetime.timedelta(hours=12)
threshold = threshold.replace(tzinfo=pytz.timezone("UTC"))
icursor = IEM.cursor()
icursor.execute("""SELECT id, valid from current c JOIN stations t
ON (t.iemid = c.iemid) WHERE t.network = 'ISUSM'
""")
obs = {}
for row in icursor:
obs[row[0]] = dict(id=row[0], valid=row[1])
tracker = TrackerEngine(IEM.cursor(), PORTFOLIO.cursor(), 7)
tracker.process_network(obs, 'isusm', NT, threshold)
tracker.send_emails()
tac = tracker.action_count
if tac > 6:
print 'check_isusm_online.py had %s actions, did not email' % (tac,)
IEM.commit()
PORTFOLIO.commit()