""" PIREP parser! """
import datetime
import os
from twisted.internet import reactor
from twisted.python import log
from pyldm import ldmbridge
from pyiem.nws.products.pirep import parser as pirepparser
import common
TABLESDIR = os.path.join(os.path.dirname(__file__), "../tables")
PIREPS = {}
DBPOOL = common.get_database("postgis")
JABBER = common.make_jabber_client("pirep")
# Load LOCS table
LOCS = {}
def cleandb():
"""To keep LSRDB from growing too big, we clean it out
Lets hold 1 days of data!
"""
thres = datetime.datetime.utcnow() - datetime.timedelta(hours=24 * 1)
init_size = len(PIREPS.keys())
for key in PIREPS:
if PIREPS[key] < thres:
del PIREPS[key]
fin_size = len(PIREPS.keys())
log.msg("cleandb() init_size: %s final_size: %s" % (init_size, fin_size))
nwsli_dict[row['nwsli']] = nwsli.NWSLI(row['nwsli'], name=nm)
log.msg("nwsli_dict loaded %s entries" % (len(nwsli_dict),))
return None
def ready(dummy):
''' cb when our database work is done '''
ldmbridge.LDMProductFactory(MyProductIngestor())
def dbload():
''' Load up database stuff '''
df = PGCONN.runInteraction(load_ugc)
df.addCallback(ready)
if __name__ == '__main__':
MANUAL = False
if len(sys.argv) == 2 and sys.argv[1] == 'manual':
log.msg("Manual runtime (no jabber, 1 database connection) requested")
MANUAL = True
# Fire up!
PGCONN = common.get_database("postgis", cp_max=(5 if not MANUAL else 1))
dbload()
jabber = common.make_jabber_client('generic_parser')
reactor.run()
if data['data'].get('precip_today'):
val = data['data']['precip_today']
if val != row['pday']:
updatesql.append(' pday = %s' % (val,))
logmsg.append('PDay O:%s N:%s' % (row['pday'], val))
if data['data'].get('snow_today'):
val = data['data']['snow_today']
if row['snow'] is None or val != row['snow']:
updatesql.append(' snow = %s' % (val,))
logmsg.append('Snow O:%s N:%s' % (row['snow'], val))
if len(updatesql) > 0:
txn.execute("""UPDATE """+table+""" d SET
""" + ','.join(updatesql) + """
FROM stations t WHERE t.iemid = d.iemid and d.day = %s and t.id = %s
and t.network ~* 'ASOS' """, (data['cli_valid'], station))
log.msg(("%s rows for %s (%s) %s"
) % (txn.rowcount, station,
data['cli_valid'].strftime("%y%m%d"), ','.join(logmsg)))
save_data(txn, prod, station, data)
if __name__ == '__main__':
# Do Stuff
jabber = common.make_jabber_client('cli_parser')
ldmbridge.LDMProductFactory(MyProductIngestor())
reactor.run()
hours=reference.offsets.get(prod.z,0) )).strftime("%-I:%M %p"),
prod.z)
mess = "%s: %s issues %s for %s %s %s?pid=%s" % (prod.source[1:],
prod.source[1:], headline, counties, expire,
config.get('urls', 'product'), product_id)
htmlmess = "%s issues <a href='%s?pid=%s'>%s</a> for %s %s" % (
prod.source[1:], config.get('urls', 'product'),
product_id, headline, counties, expire)
jabber.sendMessage(mess, htmlmess, xtra)
twt = "%s for %s %s" % (headline, counties, expire)
url = "%s?pid=%s" % (config.get('urls', 'product'), product_id)
common.tweet([prod.source[1:],], twt, url)
jabber = common.make_jabber_client('sps_parser')
def ready(bogus):
ldmbridge.LDMProductFactory( myProductIngestor() )
def killer(err):
log.err( err )
reactor.stop()
df = POSTGIS.runInteraction(load_ugc)
df.addCallback(ready)
df.addErrback( killer )
reactor.run()
def really_process_data(txn, data):
''' We are called with a hard coded AFOS PIL '''
tp = TextProduct(data)
if tp.afos is None:
compute_afos(tp)
utc = tp.valid
table = "products_%s_0106" % (utc.year,)
if utc.month > 6:
table = "products_%s_0712" % (utc.year,)
sql = """INSERT into """ + table + """
(pil, data, source, wmo, entered) values(%s,%s,%s,%s,%s)"""
sqlargs = (tp.afos, tp.text,
tp.source, tp.wmo, utc.strftime("%Y-%m-%d %H:%M+00"))
txn.execute(sql, sqlargs)
if tp.afos[:3] == 'FRH':
return
jmsgs = tp.get_jabbers(
common.SETTINGS.get('pywwa_product_url', 'pywwa_product_url'))
for jmsg in jmsgs:
JABBER.send_message(*jmsg)
if __name__ == '__main__':
PGCONN = common.get_database("afos", cp_max=1)
JABBER = common.make_jabber_client('fake_afos_dump')
ldmbridge.LDMProductFactory(MyProductIngestor())
reactor.run()
""" load ugc"""
sql = "SELECT name, ugc, wfo from nws_ugc WHERE name IS NOT Null"
txn.execute(sql)
for row in txn:
ugc_dict[ row['ugc'] ] = (row["name"]).replace("\x92"," ").replace("\xc2"," ")
ugc2wfo[ row['ugc'] ] = re.findall(r'([A-Z][A-Z][A-Z])',row['wfo'])
""" Load up H-VTEC NWSLI reference """
nwsli_dict = {}
def load_nwsli(txn):
""" load_nwsli"""
sql = """SELECT nwsli,
river_name || ' ' || proximity || ' ' || name || ' ['||state||']' as rname
from hvtec_nwsli"""
txn.execute( sql )
for row in txn:
nwsli_dict[ row['nwsli'] ] = (row['rname']).replace("&"," and ")
return None
def ready(res):
ldmbridge.LDMProductFactory( MyProductIngestor() )
df = POSTGIS.runInteraction(load_ugc)
df = POSTGIS.runInteraction(load_nwsli)
df.addCallback( ready )
jabber = common.make_jabber_client('vtec_parser')
reactor.run()
"""SPC Geo Products Parser!"""
from twisted.python import log
from twisted.internet import reactor
from pyldm import ldmbridge
from pyiem.nws.products.spcpts import parser
import common # @UnresolvedImport
DBPOOL = common.get_database("postgis")
WAITFOR = 20
JABBER = common.make_jabber_client("spc_parser")
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
"""shutdown"""
log.msg("connectionLost")
log.err(reason)
reactor.callLater(5, self.shutdown) # @UndefinedVariable
def shutdown(self):
"""shutdown"""
log.msg("shutdown() is called")
reactor.callWhenRunning(reactor.stop) # @UndefinedVariable
def process_data(self, data):
""" Process the product """
df = DBPOOL.runInteraction(real_parser, data)
df.addErrback(common.email_error, data)
tokens = re.findall("ATTN (WFOS|RFCS)(.*)", raw)
channels = []
for tpair in tokens:
for center in re.findall(r"([A-Z]+)\.\.\.", tpair[1]):
channels.append("SPENES.%s" % (center, ))
xtra = {"product_id": product_id}
xtra["channels"] = ",".join(channels)
xtra["twitter"] = ("NESDIS issues Satellite Precipitation "
"Estimates %s?pid=%s") % (PYWWA_PRODUCT_URL, product_id)
body = ("NESDIS issues Satellite Precipitation Estimates %s?pid=%s") % (
PYWWA_PRODUCT_URL,
product_id,
)
htmlbody = ("<p>NESDIS issues "
"<a href='%s?pid=%s'>Satellite Precipitation Estimates</a>"
"</p>") % (PYWWA_PRODUCT_URL, product_id)
jabber.send_message(body, htmlbody, xtra)
def killer():
"""Stop Right Now!"""
reactor.stop()
if __name__ == "__main__":
jabber = common.make_jabber_client("spe_parser")
ldmbridge.LDMProductFactory(MyProductIngestor(dedup=True))
reactor.run()
""" % (
wkt,
wkt,
)
txn.execute(sql)
cwsus = []
for row in txn.fetchall():
cwsus.append(row["id"])
return cwsus
def real_process(txn, raw):
""" "
Actually process a single MCD
"""
prod = mcdparser(raw)
prod.cwsus = find_cwsus(txn, prod)
j = prod.get_jabbers(
common.SETTINGS.get("pywwa_product_url", "pywwa_product_url"))
if len(j) == 1:
JABBER.send_message(j[0][0], j[0][1], j[0][2])
prod.database_save(txn)
ldmbridge.LDMProductFactory(MyProductIngestor())
JABBER = common.make_jabber_client("mcd_parser")
reactor.run()
common.email_error(myexp, buf)
return
defer = DBPOOL.runInteraction(prod.sql)
defer.addCallback(final_step, prod)
defer.addErrback(common.email_error, buf)
def final_step(_, prod):
"""
"""
for j in prod.get_jabbers(
common.settings.get('pywwa_product_url', 'pywwa_product_url'), ''):
jabber.sendMessage(j[0], j[1], j[2])
MESOSITE = common.get_database('mesosite')
def onready(res):
log.msg("onready() called...")
ldmbridge.LDMProductFactory(MyProductIngestor())
MESOSITE.close()
df = MESOSITE.runInteraction(load_database)
df.addCallback(onready)
df.addErrback(common.email_error, 'ERROR on load_database')
df.addErrback(log.err)
jabber = common.make_jabber_client("aviation")
reactor.run()
""" PIREP parser! """
import datetime
import os
from twisted.internet import reactor
from twisted.python import log
from pyldm import ldmbridge
from pyiem.nws.products.pirep import parser as pirepparser
import common
TABLESDIR = os.path.join(os.path.dirname(__file__), "../tables")
PIREPS = {}
DBPOOL = common.get_database("postgis")
JABBER = common.make_jabber_client('pirep')
# Load LOCS table
LOCS = {}
def cleandb():
""" To keep LSRDB from growing too big, we clean it out
Lets hold 1 days of data!
"""
thres = datetime.datetime.utcnow() - datetime.timedelta(hours=24*1)
init_size = len(PIREPS.keys())
for key in PIREPS:
if PIREPS[key] < thres:
del PIREPS[key]
fin_size = len(PIREPS.keys())
log.msg("cleandb() init_size: %s final_size: %s" % (init_size, fin_size))
prod = lsrparser(text)
for lsr in prod.lsrs:
if lsr.typetext.upper() not in reference.lsr_events:
errmsg = "Unknown LSR typecode '%s'" % (lsr.typetext, )
common.email_error(errmsg, text)
uniquekey = hash(lsr.text)
if uniquekey in LSRDB:
prod.duplicates += 1
lsr.duplicate = True
continue
LSRDB[uniquekey] = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
lsr.sql(txn)
j = prod.get_jabbers(common.SETTINGS.get("pywwa_lsr_url", "pywwa_lsr_url"))
for i, (p, h, x) in enumerate(j):
# delay some to perhaps stop triggering SPAM lock outs at twitter
reactor.callLater(i, JABBER.send_message, p, h, x)
if prod.warnings:
common.email_error("\n\n".join(prod.warnings), text)
elif not prod.lsrs:
raise Exception("No LSRs parsed!", text)
reactor.callLater(0, loaddb)
JABBER = common.make_jabber_client("lsr_parser")
LDM = ldmbridge.LDMProductFactory(MyProductIngestor())
reactor.callLater(20, cleandb)
reactor.run()
mess = "%s: %s issues %s #%s for %s %s" % (wfo2, centertext.get(wfo, wfo), what, tnum, tt, myurl)
htmlmess = "%s issues <a href=\"%s\">%s #%s</a> for %s" % ( centertext.get(wfo, wfo), myurl, what, tnum, tt)
#print htmlmess, mess
jabber.sendMessage(mess, htmlmess, xtra)
channels.append( wfo2 )
twt = "%s issues %s %s for %s" % (centertext.get(wfo, wfo), what, tnum, tt)
common.tweet(channels, twt, myurl)
""" Load up H-VTEC NWSLI reference """
nwsli_dict = {}
def load_nwsli(txn):
""" Load up NWSLI Dict """
sql = """SELECT nwsli, river_name as r,
proximity || ' ' || name || ' ['||state||']' as rname
from hvtec_nwsli"""
txn.execute(sql)
for row in txn:
nwsli_dict[ row['nwsli'] ] = {
'rname': (row['r']).replace("&"," and "),
'river': (row['rname']).replace("&"," and ") }
log.msg("nwsli_dict is loaded...")
POSTGIS.runInteraction(load_nwsli)
jabber = common.make_jabber_client("gp")
ldm = ldmbridge.LDMProductFactory( myProductIngestor() )
reactor.run()
def ready(_dummy):
""" cb when our database work is done """
ldmbridge.LDMProductFactory(MyProductIngestor(dedup=True))
def bootstrap():
"""Things to do at startup"""
df = PGCONN.runInteraction(load_ugc)
df.addCallback(ready)
df.addErrback(common.email_error, "load_ugc failure!")
if __name__ == "__main__":
SMTPSenderFactory.noisy = False
ugc_dict = {}
nwsli_dict = {}
MANUAL = False
if len(sys.argv) == 2 and sys.argv[1].lower() == "manual":
log.msg("Manual runtime (no jabber, 1 database connection) requested")
MANUAL = True
# Fire up!
PGCONN = common.get_database(common.CONFIG["databaserw"]["postgis"],
cp_max=(5 if not MANUAL else 1))
bootstrap()
jabber = common.make_jabber_client("vtec_parser")
reactor.run()
log.msg("nwsli_dict loaded %s entries" % (len(nwsli_dict), ))
return None
def ready(_):
""" cb when our database work is done """
ldmbridge.LDMProductFactory(MyProductIngestor())
def dbload():
""" Load up database stuff """
df = PGCONN.runInteraction(load_ugc)
df.addCallback(ready)
if __name__ == "__main__":
MANUAL = False
if len(sys.argv) == 2 and sys.argv[1] == "manual":
log.msg("Manual runtime (no jabber, 1 database connection) requested")
MANUAL = True
# Fire up!
PGCONN = common.get_database("postgis", cp_max=(5 if not MANUAL else 1))
dbload()
jabber = common.make_jabber_client("generic_parser")
reactor.run()
'''
wkt = 'SRID=4326;%s' % (prod.geometry.wkt,)
sql = """select distinct id from cwsu WHERE
st_overlaps('%s', geom) or
st_covers(geom, '%s') ORDER by id ASC""" % (wkt, wkt)
txn.execute(sql)
cwsus = []
for row in txn:
cwsus.append(row['id'])
return cwsus
def real_process(txn, raw):
""""
Actually process a single MCD
"""
prod = mcdparser(raw)
prod.cwsus = find_cwsus(txn, prod)
j = prod.get_jabbers(common.settings.get('pywwa_product_url',
'pywwa_product_url'))
if len(j) == 1:
JABBER.sendMessage(j[0][0], j[0][1], j[0][2])
prod.database_save(txn)
ldmbridge.LDMProductFactory(MyProductIngestor())
JABBER = common.make_jabber_client('mcd_parser')
reactor.run()
myargs = (sqlraw, product_id)
txn.execute(sql, myargs)
tokens = re.findall("ATTN (WFOS|RFCS)(.*)", raw)
channels = []
for tpair in tokens:
for center in re.findall("([A-Z]+)\.\.\.", tpair[1]):
channels.append("SPENES.%s" % (center, ))
xtra = {'product_id': product_id}
xtra['channels'] = ','.join(channels)
xtra['twitter'] = ("NESDIS issues Satellite Precipitation "
"Estimates %s?pid=%s") % (PYWWA_PRODUCT_URL,
product_id)
body = ("NESDIS issues Satellite Precipitation Estimates %s?pid=%s"
) % (PYWWA_PRODUCT_URL, product_id)
htmlbody = ("<p>NESDIS issues "
"<a href='%s?pid=%s'>Satellite Precipitation Estimates</a>"
"</p>"
) % (PYWWA_PRODUCT_URL, product_id)
jabber.sendMessage(body, htmlbody, xtra)
def killer():
reactor.stop()
if __name__ == '__main__':
jabber = common.make_jabber_client("spe_parser")
ldmbridge.LDMProductFactory(MyProductIngestor(dedup=True))
reactor.run()
""" SPC Watch Ingestor """
from twisted.python import log
from twisted.internet import reactor
from pyiem.nws.products.saw import parser as sawparser
from pyldm import ldmbridge
import common # @UnresolvedImport
DBPOOL = common.get_database("postgis", cp_max=1)
IEM_URL = common.SETTINGS.get("pywwa_watch_url", "pywwa_watch_url")
JABBER = common.make_jabber_client("new_watch")
def shutdown():
"""Shut things down, please"""
reactor.callWhenRunning(reactor.stop) # @UndefinedVariable
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
"""STDIN is shut, so lets shutdown"""
log.msg("connectionLost")
log.err(reason)
reactor.callLater(7, shutdown) # @UndefinedVariable
def process_data(self, data):
"""Process the product!"""
df = DBPOOL.runInteraction(real_process, data)
df.addErrback(common.email_error, data)
def real_processor(txn, text):
""" Lets actually process! """
prod = lsrparser(text)
if len(prod.lsrs) == 0:
raise Exception("No LSRs parsed!", text)
for lsr in prod.lsrs:
if lsr.typetext not in reference.lsr_events:
errmsg = "Unknown LSR typecode '%s'" % (lsr.typetext,)
common.email_error(errmsg, text)
uniquekey = hash(lsr.text)
if uniquekey in LSRDB:
prod.duplicates += 1
lsr.duplicate = True
continue
LSRDB[uniquekey] = datetime.datetime.utcnow().replace(
tzinfo=pytz.timezone("UTC"))
lsr.sql(txn)
j = prod.get_jabbers(common.settings.get('pywwa_lsr_url', 'pywwa_lsr_url'))
for (p, h, x) in j:
JABBER.sendMessage(p, h, x)
reactor.callLater(0, loaddb)
JABBER = common.make_jabber_client("lsr_parser")
LDM = ldmbridge.LDMProductFactory(MyProductIngestor())
reactor.callLater(20, cleandb)
reactor.run()
log.msg("Unknown WFO for id: %s, skipping WindAlert" % (iemid,))
return
st = row['state']
nm = row['name']
network = row['network']
extra = ""
if (clean_metar.find("$") > 0):
extra = "(Caution: Maintenance Check Indicator)"
url = "http://mesonet.agron.iastate.edu/ASOS/current.phtml?network=%s" % (network,)
jtxt = "%s: %s,%s (%s) ASOS %s reports gust of %s knots from %s @ %s\n%s"\
% (wfo, nm, st, iemid, extra, v, drct2dirTxt(d), \
t.strftime("%H%MZ"), clean_metar )
jabber.sendMessage(jtxt, jtxt)
twt = "%s,%s (%s) ASOS reports gust of %s knots from %s @ %s" % (nm,
st, iemid, v, drct2dirTxt(d), t.strftime("%H%MZ"))
common.tweet([wfo], twt, url, {'lat': str(row['lat']), 'long': str(row['lon'])})
def ready(bogus):
ingest = myProductIngestor()
ldmbridge.LDMProductFactory( ingest )
jabber = common.make_jabber_client("metar_parser")
df = IEMDB.runInteraction(load_stations)
df.addCallback(ready)
reactor.run()
''' Process a chunk of data '''
df = DBPOOL.runInteraction(real_process, raw)
df.addErrback(common.email_error, raw)
def real_process(txn, raw):
""""
Actually process a single MCD
"""
prod = mcdparser(raw)
product_id = prod.get_product_id()
channels = [prod.afos,]
channels.extend( prod.attn_wfo )
channels.extend( prod.attn_rfc )
channels.extend( prod.find_cwsus(txn) )
body, htmlbody = prod.get_jabbers(common.settings.get('pywwa_product_url',
'pywwa_product_url'))
JABBER.sendMessage(body, htmlbody, {
'channels': ",".join( channels ),
'product_id': product_id,
'twitter': prod.tweet()} )
prod.database_save(txn)
ldmbridge.LDMProductFactory( MyProductIngestor() )
JABBER = common.make_jabber_client('mcdparse')
reactor.run()
# Figure out who should get notification of the watch...
sql = """SELECT distinct wfo from ugcs WHERE
ST_Contains('SRID=4326;MULTIPOLYGON(((%s)))', geom)
and end_ts is null""" % (wkt,)
txn.execute(sql)
rs = txn.fetchall()
channels = ['SPC', ]
for i in range(len(rs)):
wfo = rs[i]['wfo']
channels.append(wfo)
xtra = {'channels': ','.join(channels)}
jabber.sendMessage(jabberTxt, jabberTxtHTML, xtra)
# Special message for SPC
lines = raw.split("\n")
twt = lines[5].replace("\r\r", "")
twt += (" http://www.spc.noaa.gov/products/watch/ww%04i.html"
) % (int(ww_num), )
xtra['channels'] = 'SPC'
jabber.sendMessage(twt, twt, xtra)
df = DBPOOL.runInteraction(runner2)
df.addErrback(common.email_error, raw)
if __name__ == '__main__':
jabber = common.make_jabber_client("new_watch")
ldm = ldmbridge.LDMProductFactory(MyProductIngestor())
reactor.run()
""" We are called with a hard coded AFOS PIL """
tp = TextProduct(data)
if tp.afos is None:
compute_afos(tp)
sql = ("INSERT into products "
"(pil, data, source, wmo, entered) values(%s,%s,%s,%s,%s)")
sqlargs = (
tp.afos,
tp.text,
tp.source,
tp.wmo,
tp.valid.strftime("%Y-%m-%d %H:%M+00"),
)
txn.execute(sql, sqlargs)
if tp.afos[:3] == "FRH":
return
jmsgs = tp.get_jabbers(
common.SETTINGS.get("pywwa_product_url", "pywwa_product_url"))
for jmsg in jmsgs:
JABBER.send_message(*jmsg)
if __name__ == "__main__":
PGCONN = common.get_database("afos", cp_max=1)
JABBER = common.make_jabber_client("fake_afos_dump")
ldmbridge.LDMProductFactory(MyProductIngestor())
reactor.run()
"""SPC Geo Products Parser!"""
from twisted.python import log
from twisted.internet import reactor
from pyldm import ldmbridge
from pyiem.nws.products.spcpts import parser
import common # @UnresolvedImport
DBPOOL = common.get_database('postgis')
WAITFOR = 20
JABBER = common.make_jabber_client('spc_parser')
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
"""shutdown"""
log.msg('connectionLost')
log.err(reason)
reactor.callLater(5, self.shutdown) # @UndefinedVariable
def shutdown(self):
"""shutdown"""
log.msg("shutdown() is called")
reactor.callWhenRunning(reactor.stop) # @UndefinedVariable
def process_data(self, data):
""" Process the product """
df = DBPOOL.runInteraction(real_parser, data)