def main():
"""Go Main Go"""
mesosite = common.get_database("mesosite", cp_max=1)
df = mesosite.runInteraction(load_station_table)
df.addCallback(on_ready, mesosite)
df.addErrback(errback)
reactor.run()
def save_if_required(self, wx, force=False):
"""!
Save data if not in database already, unless forced
@param wx Data to save into database if not already present
@param force Whether or not to load data if it is already in database
@return Timestamp for loaded run
"""
# HACK: somehow still getting duplicates
index = wx.index.names
# NOTE: need to reset_index() because otherwise only compares wx values
wx = wx.reset_index().drop_duplicates().set_index(index)
# HACK: Timestamp format is nicer than datetime's
for_run = wx.reset_index()['Generated'].max()
# want to put the data into the database for the start date but check if it exists based on for_run
start_date = wx.reset_index()['ForTime'].min()
dbname = common.get_database(start_date)
# if at any point for_run is already in the database then we're done
if not force:
logging.debug(
'Checking if data is already present for {} model generated at {}'
.format(self.name, for_run))
exists = self.check_exists(for_run, dbname)
if exists:
logging.debug('Data already loaded - aborting')
return pandas.Timestamp(for_run)
self.save_data(wx, dbname)
# return the run that we ended up loading data for
return for_run
def dbtest():
webdb = common.get_database(
"/home/vikrant/programming/work/publicgit/gridpath/mh.db")
scenario1 = "rpo30_pass1"
scenario2 = 'rpo30_pass2'
project = 'Koyna_Stage_3'
adjusted_mean_results(webdb, scenario1, scenario2, project)
def load_specific_records(self, for_run, force=False):
"""!
Load records for specific period
@param self Pointer to self
@param for_run Which run of model to use
@param force Whether or not to force loading if records already exist
@return Timestamp of run that records were loaded for
"""
# save into database that corresponds to the start of this run
# want to put the data into the database for the start date but check if it exists based on for_run
start_date = for_run + datetime.timedelta(hours=self.lead_time)
dbname = common.get_database(start_date)
# if at any point for_run is already in the database then we're done
if not force:
logging.debug(
'Checking if data is already present for {} model generated at {}'
.format(self.name, for_run))
exists = self.check_exists(for_run, dbname)
if exists:
logging.debug('Data already loaded - aborting')
return pandas.Timestamp(for_run)
results = []
for d in self.for_days:
results.append(self.get_records(for_run, d))
# don't save data until everything is loaded
wx = pandas.concat(results)
self.save_data(wx, dbname)
# return the run that we ended up loading data for
# HACK: Timestamp format is nicer than datetime's
return pandas.Timestamp(for_run)
def test_compare_with_db():
def get_db_results():
subscenario_id = get_hydro_ops_chars_scenario_id(
webdb, compare_scenario, project)
rows = webdb.where(
"inputs_project_hydro_operational_chars",
project=project,
hydro_operational_chars_scenario_id=subscenario_id).list()
return pd.DataFrame(rows)
gridpath = "/home/vikrant/programming/work/publicgit/gridpath-0.8.1/gridpath"
database = os.path.join(gridpath, "mh.db")
csv_location = os.path.join(gridpath, "db", "csvs_mh")
scenario1 = "rpo50S3_pass1_2"
scenario2 = "rpo50S3_pass2"
compare_scenario = "rpo50S3_pass2"
project = "Sardar_Sarovar_CHPH"
webdb = common.get_database(database)
subscenario = "hydro_operational_chars_scenario_id"
print(f"Computing data for {project}")
subscenario_id = get_hydro_ops_chars_scenario_id(webdb, scenario2, project)
results = adjusted_mean_results(webdb, scenario1, scenario2, project)
write_results_csv(results, project, subscenario, subscenario_id,
csv_location, "rpo50S3_all")
csvpath = common.get_subscenario_csvpath(project, subscenario,
subscenario_id, csv_location,
"rpo50S3_all")
filedata = pd.read_csv(csvpath)
filedata.set_index("horizon", inplace=True)
dailyfile = filedata[filedata.balancing_type_project == "month"]
b1 = dailyfile['average_power_fraction']
dbdata = get_db_results()
dbdata.set_index('horizon', inplace=True)
dailydb = dbdata[dbdata.balancing_type_project == "month"]
b2 = dailydb['average_power_fraction']
printcols(b1, b2, b1.index, b2.index)
diff = abs(b1 - b2) >= 0.001
printcols(b1[diff], b2[diff], b1.index[diff], b2.index[diff])
assert np.all(abs(b1 - b2) <= 0.001)
def hydro_op_chars(database, csv_location, gridpath_repo, scenario1, scenario2,
description, project, update_database):
webdb = common.get_database(database)
projects = get_projects(webdb, scenario1)
if project:
projects = [project] #projects[:1]
subscenario = "hydro_operational_chars_scenario_id"
for project_ in projects:
print(f"Computing data for {project_}")
subscenario_id = get_hydro_ops_chars_scenario_id(
webdb, scenario2, project_)
results = adjusted_mean_results(webdb, scenario1, scenario2, project_)
write_results_csv(results, project_, subscenario, subscenario_id,
csv_location, description)
if update_database:
common.update_subscenario_via_gridpath(subscenario, subscenario_id,
project_, csv_location,
database, gridpath_repo)
""" We are called with a hard coded AFOS PIL """
tp = TextProduct(data)
if tp.afos is None:
compute_afos(tp)
sql = ("INSERT into products "
"(pil, data, source, wmo, entered) values(%s,%s,%s,%s,%s)")
sqlargs = (
tp.afos,
tp.text,
tp.source,
tp.wmo,
tp.valid.strftime("%Y-%m-%d %H:%M+00"),
)
txn.execute(sql, sqlargs)
if tp.afos[:3] == "FRH":
return
jmsgs = tp.get_jabbers(
common.SETTINGS.get("pywwa_product_url", "pywwa_product_url"))
for jmsg in jmsgs:
JABBER.send_message(*jmsg)
if __name__ == "__main__":
PGCONN = common.get_database("afos", cp_max=1)
JABBER = common.make_jabber_client("fake_afos_dump")
ldmbridge.LDMProductFactory(MyProductIngestor())
reactor.run()
from pyiem.reference import TRACE_VALUE
from pyiem.observation import Observation
from pyiem.nws import product
from pyiem import reference
from pyldm import ldmbridge
import common # @UnresolvedImport
# from pympler import tracker, summary, muppy
# TR = tracker.SummaryTracker()
# Setup Database Links
# the current_shef table is not very safe when two processes attempt to update
# it at the same time, use a single process for this connection
ACCESSDB = common.get_database("iem", module_name="psycopg2", cp_max=20)
HADSDB = common.get_database("hads", module_name="psycopg2", cp_max=20)
# a form for IDs we will log as unknown
NWSLIRE = re.compile("[A-Z]{4}[0-9]")
# stations we don't know about
UNKNOWN = dict()
# station metadata
LOCS = dict()
# database timezones to pytz cache
TIMEZONES = dict()
# a queue for saving database IO
CURRENT_QUEUE = {}
U1980 = datetime.datetime.utcnow()
U1980 = U1980.replace(day=1, year=1980, tzinfo=pytz.utc)
def init_db_cayley():
client = init_client(MongoDBConfig.MONGODB_HOST,
MongoDBConfig.MONGODB_PORT)
db = get_database(client, MongoDBConfig.DATABASE_NAME)
return client, db
syslog.startLogging(prefix="pyWWA/mos_parser", facility=LOG_LOCAL2)
# Twisted Python imports
from twisted.internet import reactor
# Standard Python modules
import re
import datetime
import pytz
# pyWWA stuff
from pyldm import ldmbridge
import common
DBPOOL = common.get_database("mos")
class myProductIngestor(ldmbridge.LDMProductReceiver):
def process_data(self, buf):
"""
Actual ingestor
"""
try:
real_process(buf)
except Exception, myexp:
common.email_error(myexp, buf)
def connectionLost(self, reason):
"""
Called when ldm closes the pipe
""" SPC Geo Products Parser! """
# Twisted Python imports
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/spc_parser', facility=LOG_LOCAL2)
from twisted.python import log
from twisted.internet import reactor
# pyWWA stuff
from pyldm import ldmbridge
from pyiem.nws import product, spcpts
import common
DBPOOL = common.get_database('postgis')
WAITFOR = 20
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
log.msg('connectionLost')
log.err(reason)
reactor.callLater(5, self.shutdown)
def shutdown(self):
reactor.callWhenRunning(reactor.stop)
def process_data(self, buf):
NOTE: It is difficult to keep track of where I am getting the `Metar` library.
So let us document it here for my own sanity.
18 Jul 2017: `snowdepth` branch of my python-metar fork installed with pip
"""
import sys
# Twisted Python imports
from twisted.python import log
from twisted.internet import reactor
from pyiem.nws.products import metarcollect
from pyiem.util import get_properties
from pyldm import ldmbridge
import common # @UnresolvedImport
IEMDB = common.get_database("iem")
ASOSDB = common.get_database("asos")
MANUAL = "manual" in [x.lower() for x in sys.argv]
NWSLI_PROVIDER = {}
# Manual list of sites that are sent to jabber :/
metarcollect.JABBER_SITES = {
"KJFK": None,
"KLGA": None,
"KEWR": None,
"KTEB": None,
"KIAD": None,
"KDCA": None,
"KBWI": None,
"KRIC": None,
"KPHL": None,
"""SPENES product ingestor"""
import re
from twisted.python import log
from twisted.internet import reactor
from pyldm import ldmbridge
from pyiem.nws import product
import common
POSTGIS = common.get_database("postgis", cp_max=1)
PYWWA_PRODUCT_URL = common.SETTINGS.get("pywwa_product_url",
"pywwa_product_url")
DB_ON = bool(common.SETTINGS.get("pywwa_save_text_products", False))
def shutdown():
""" Stop this app """
log.msg("Shutting down...")
reactor.callWhenRunning(reactor.stop)
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
""" callback when the stdin reader connection is closed """
log.msg("connectionLost() called...")
log.err(reason)
reactor.callLater(7, shutdown)
def process_data(self, data):
""" Process the product """
log.msg("nwsli_dict loaded %s entries" % (len(nwsli_dict), ))
return None
def ready(_):
""" cb when our database work is done """
ldmbridge.LDMProductFactory(MyProductIngestor())
def dbload():
""" Load up database stuff """
df = PGCONN.runInteraction(load_ugc)
df.addCallback(ready)
if __name__ == "__main__":
MANUAL = False
if len(sys.argv) == 2 and sys.argv[1] == "manual":
log.msg("Manual runtime (no jabber, 1 database connection) requested")
MANUAL = True
# Fire up!
PGCONN = common.get_database("postgis", cp_max=(5 if not MANUAL else 1))
dbload()
jabber = common.make_jabber_client("generic_parser")
reactor.run()
""" PIREP parser! """
import datetime
import os
from twisted.internet import reactor
from twisted.python import log
from pyldm import ldmbridge
from pyiem.nws.products.pirep import parser as pirepparser
import common
TABLESDIR = os.path.join(os.path.dirname(__file__), "../tables")
PIREPS = {}
DBPOOL = common.get_database("postgis")
JABBER = common.make_jabber_client('pirep')
# Load LOCS table
LOCS = {}
def cleandb():
""" To keep LSRDB from growing too big, we clean it out
Lets hold 1 days of data!
"""
thres = datetime.datetime.utcnow() - datetime.timedelta(hours=24*1)
init_size = len(PIREPS.keys())
for key in PIREPS:
if PIREPS[key] < thres:
del PIREPS[key]
fin_size = len(PIREPS.keys())
log.msg("cleandb() init_size: %s final_size: %s" % (init_size, fin_size))
""" NESDIS SCP Ingestor """
from twisted.python import log
from twisted.internet import reactor
from pyiem.nws.products.scp import parser
from pyldm import ldmbridge
import common # @UnresolvedImport
DBPOOL = common.get_database("asos", cp_max=1)
def shutdown():
"""Shut things down, please"""
reactor.callWhenRunning(reactor.stop) # @UndefinedVariable
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
"""STDIN is shut, so lets shutdown"""
log.msg("connectionLost")
log.err(reason)
reactor.callLater(7, shutdown) # @UndefinedVariable
def process_data(self, data):
"""Process the product!"""
df = DBPOOL.runInteraction(real_process, data)
df.addErrback(common.email_error, data)
def real_process(txn, raw):
""" Aviation Product Parser! """
import common
import os
# Twisted Python imports
from twisted.internet import reactor
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/aviation', facility=LOG_LOCAL2)
from twisted.python import log
# pyWWA stuff
from pyldm import ldmbridge
from pyiem.nws.products.sigmet import parser
DBPOOL = common.get_database('postgis')
# Load LOCS table
LOCS = {}
_MYDIR = os.path.dirname(os.path.abspath(__file__))
TABLE_PATH = os.path.normpath(os.path.join(_MYDIR, "..", "tables"))
def load_database(txn):
txn.execute("""
SELECT id, name, ST_x(geom) as lon, ST_y(geom) as lat from stations
WHERE network ~* 'ASOS' or network ~* 'AWOS'
""")
for row in txn:
from pyiem import wellknowntext
import re
import pytz
import dbflib
import shapelib
import zipfile
import os
import shutil
import subprocess
import common
syslog.startLogging(prefix="pyWWA/stoia_parser", facility=LOG_LOCAL2)
EPSG26915 = """PROJCS["NAD_1983_UTM_Zone_15N",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-93.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]"""
DBPOOL = common.get_database("postgis", cp_max=1)
CONDITIONS = {}
ROADS = {}
# Changedir to /tmp
os.chdir("/tmp")
def shutdown():
""" Down we go! """
log.msg("Stopping...")
reactor.callWhenRunning(reactor.stop)
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
""" NLDN """
from io import BytesIO
from twisted.internet import reactor
from pyldm import ldmbridge
from pyiem.nws.products.nldn import parser
import common
DBPOOL = common.get_database("nldn")
class myProductIngestor(ldmbridge.LDMProductReceiver):
"""My hacky ingest"""
product_end = b"NLDN"
def process_data(self, data):
"""Actual ingestor"""
if data == b"":
return
real_process(data)
try:
real_process(data)
except Exception as myexp:
common.email_error(myexp, data)
def connectionLost(self, reason):
"""
Called when ldm closes the pipe
"""
print("connectionLost")
"""Process and Archive the NEXRAD Level III NCR Attribute Table"""
from io import BytesIO
import math
import pytz
from twisted.python import log
from twisted.internet import reactor
from metpy.io.nexrad import Level3File
from pyldm import ldmbridge
import common
# Setup Database Links
PGCONN = common.get_database("radar")
ST = {}
def load_station_table(txn):
""" Load the station table of NEXRAD sites """
log.msg("load_station_table called() ...")
txn.execute(
"SELECT id, ST_x(geom) as lon, ST_y(geom) as lat from stations "
"where network in ('NEXRAD','TWDR')")
for row in txn.fetchall():
ST[row["id"]] = {"lat": row["lat"], "lon": row["lon"]}
log.msg("Station Table size %s" % (len(ST.keys())))
class MyProductIngestor(ldmbridge.LDMProductReceiver):
"""My ingest protocol"""
def connectionLost(self, reason):
""" SPC Watch Ingestor """
from twisted.python import log
from twisted.internet import reactor
from pyiem.nws.products.saw import parser as sawparser
from pyldm import ldmbridge
import common # @UnresolvedImport
DBPOOL = common.get_database('postgis', cp_max=1)
IEM_URL = common.SETTINGS.get('pywwa_watch_url', 'pywwa_watch_url')
JABBER = common.make_jabber_client("new_watch")
def shutdown():
"""Shut things down, please"""
reactor.callWhenRunning(reactor.stop) # @UndefinedVariable
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
"""STDIN is shut, so lets shutdown"""
log.msg('connectionLost')
log.err(reason)
reactor.callLater(7, shutdown) # @UndefinedVariable
def process_data(self, data):
"""Process the product!"""
df = DBPOOL.runInteraction(real_process, data)
df.addErrback(common.email_error, data)
The CLI report has lots of good data that is hard to find in other products,
so we take what data we find in this product and overwrite the database
storage of what we got from the automated observations
"""
from pyiem.nws.products import parser
from pyldm import ldmbridge
from pyiem.network import Table as NetworkTable
import common
from twisted.internet import reactor
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/cli_parser', facility=LOG_LOCAL2)
from twisted.python import log
DBPOOL = common.get_database('iem', cp_max=1)
NT = NetworkTable("NWSCLI")
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
""" Connection was lost! """
log.err(reason)
reactor.callLater(7, reactor.callWhenRunning, reactor.stop)
def process_data(self, text):
""" Process the product """
deffer = DBPOOL.runInteraction(preprocessor, text)
"""SPENES product ingestor"""
# Twisted Python imports
from syslog import LOG_LOCAL2
from twisted.python import syslog
from twisted.python import log
syslog.startLogging(prefix='pyWWA/spe_parser', facility=LOG_LOCAL2)
from twisted.internet import reactor
import re
import common
# pyLDM https://github.com/akrherz/pyLDM
from pyldm import ldmbridge
from pyiem.nws import product
POSTGIS = common.get_database('postgis', cp_max=1)
PYWWA_PRODUCT_URL = common.settings.get('pywwa_product_url',
'pywwa_product_url')
DB_ON = bool(common.settings.get('pywwa_save_text_products', False))
def shutdown():
''' Stop this app '''
log.msg("Shutting down...")
reactor.callWhenRunning(reactor.stop)
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
""" MOS Data Ingestor, why not? """
from __future__ import print_function
from twisted.internet import reactor
from pyldm import ldmbridge
from pyiem.nws.products.mos import parser
import common # @UnresolvedImport
DBPOOL = common.get_database('mos')
MEMORY = {'ingested': 0}
class MyProductIngestor(ldmbridge.LDMProductReceiver):
"""Do what we do!"""
def process_data(self, data):
"""
Actual ingestor
"""
try:
real_process(data)
except Exception as myexp:
common.email_error(myexp, data)
def connectionLost(self, reason):
"""
Called when ldm closes the pipe
"""
reactor.callLater(5, self.shutdown)
def shutdown(self):
""" HML parser! """
from twisted.internet import reactor
from twisted.python import log
from pyldm import ldmbridge
from pyiem.nws.products.hml import parser as hmlparser
import common # @UnresolvedImport
DBPOOL = common.get_database("hml")
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
"""Connection was lost"""
log.msg("connectionLost")
log.err(reason)
reactor.callLater(5, reactor.stop) # @UndefinedVariable
def process_data(self, data):
""" Process the product """
defer = DBPOOL.runInteraction(real_parser, data)
defer.addErrback(common.email_error, data)
defer.addErrback(log.err)
def real_parser(txn, buf):
"""I'm gonna do the heavy lifting here"""
prod = hmlparser(buf)
log.msg("nwsli_dict loaded %s entries" % (len(nwsli_dict),))
return None
def ready(dummy):
''' cb when our database work is done '''
ldmbridge.LDMProductFactory(MyProductIngestor(dedup=True))
if __name__ == '__main__':
syslog.startLogging(prefix='pyWWA/vtec_parser', facility=LOG_LOCAL2)
HTTPClientFactory.noisy = False
SMTPSenderFactory.noisy = False
ugc_dict = {}
nwsli_dict = {}
MANUAL = False
if len(sys.argv) == 2 and sys.argv[1] == 'manual':
log.msg("Manual runtime (no jabber, 1 database connection) requested")
MANUAL = True
# Fire up!
PGCONN = common.get_database(common.config['databaserw']['postgis'],
cp_max=(5 if not MANUAL else 1))
df = PGCONN.runInteraction(load_ugc)
df.addCallback(ready)
df.addErrback(common.email_error, "load_ugc failure!")
jabber = common.make_jabber_client('vtec_parser')
reactor.run()
def ready(_dummy):
""" cb when our database work is done """
ldmbridge.LDMProductFactory(MyProductIngestor(dedup=True))
def bootstrap():
"""Things to do at startup"""
df = PGCONN.runInteraction(load_ugc)
df.addCallback(ready)
df.addErrback(common.email_error, "load_ugc failure!")
if __name__ == "__main__":
SMTPSenderFactory.noisy = False
ugc_dict = {}
nwsli_dict = {}
MANUAL = False
if len(sys.argv) == 2 and sys.argv[1].lower() == "manual":
log.msg("Manual runtime (no jabber, 1 database connection) requested")
MANUAL = True
# Fire up!
PGCONN = common.get_database(common.CONFIG["databaserw"]["postgis"],
cp_max=(5 if not MANUAL else 1))
bootstrap()
jabber = common.make_jabber_client("vtec_parser")
reactor.run()
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/lsr_parser', facility=LOG_LOCAL2)
from twisted.python import log
from twisted.internet import reactor
# Third party python stuff
import pytz
from pyiem import reference
from pyiem.nws.products.lsr import parser as lsrparser
from pyldm import ldmbridge
# IEM python Stuff
import common
import datetime
DBPOOL = common.get_database(common.config['databaserw']['postgis'])
# Cheap datastore for LSRs to avoid Dups!
LSRDB = {}
def loaddb():
''' load memory '''
if os.path.isfile('lsrdb.p'):
mydict = pickle.load(open('lsrdb.p'))
for key in mydict:
LSRDB[key] = mydict[key]
def cleandb():
""" To keep LSRDB from growing too big, we clean it out
""" ASOS Daily Summary Message Parser ingestor """
import pytz
from twisted.internet import reactor
from twisted.python import log
from pyldm import ldmbridge
from pyiem.nws.products.dsm import parser
from pyiem.util import get_dbconn
import common
DBPOOL = common.get_database("iem", cp_max=1)
# database timezones to pytz cache
TIMEZONES = dict()
STATIONS = dict()
def load_stations(txn):
"""load station metadata to build a xref."""
txn.execute("""
SELECT id, tzname from stations
where network ~* 'ASOS' or network = 'AWOS'
""")
for row in txn:
# we need four char station IDs
station = row[0] if len(row[0]) == 4 else "K" + row[0]
tzname = row[1]
if tzname not in TIMEZONES:
try:
TIMEZONES[tzname] = pytz.timezone(tzname)
except Exception as exp:
log.msg("pytz does not like tzname: %s %s" % (tzname, exp))
def init_db_cayley():
client = init_client(MongoDBConfig.MONGODB_HOST,
MongoDBConfig.MONGODB_PORT)
db = get_database(client, MongoDBConfig.DATABASE_NAME)
return client, db
def really_process_data(txn, data):
''' We are called with a hard coded AFOS PIL '''
tp = TextProduct(data)
if tp.afos is None:
compute_afos(tp)
utc = tp.valid
table = "products_%s_0106" % (utc.year,)
if utc.month > 6:
table = "products_%s_0712" % (utc.year,)
sql = """INSERT into """ + table + """
(pil, data, source, wmo, entered) values(%s,%s,%s,%s,%s)"""
sqlargs = (tp.afos, tp.text,
tp.source, tp.wmo, utc.strftime("%Y-%m-%d %H:%M+00"))
txn.execute(sql, sqlargs)
if tp.afos[:3] == 'FRH':
return
jmsgs = tp.get_jabbers(
common.SETTINGS.get('pywwa_product_url', 'pywwa_product_url'))
for jmsg in jmsgs:
JABBER.send_message(*jmsg)
if __name__ == '__main__':
PGCONN = common.get_database("afos", cp_max=1)
JABBER = common.make_jabber_client('fake_afos_dump')
ldmbridge.LDMProductFactory(MyProductIngestor())
reactor.run()
NOTE: It is difficult to keep track of where I am getting the `Metar` library.
So let us document it here for my own sanity.
18 Jul 2017: `snowdepth` branch of my python-metar fork installed with pip
"""
from __future__ import print_function
import sys
# Twisted Python imports
from twisted.python import log
from twisted.internet import reactor
from pyiem.nws.products import metarcollect
from pyldm import ldmbridge
import common # @UnresolvedImport
IEMDB = common.get_database('iem')
ASOSDB = common.get_database('asos')
MANUAL = ('MANUAL' in sys.argv)
NWSLI_PROVIDER = {}
# Manual list of sites that are sent to jabber :/
metarcollect.JABBER_SITES = {
'KJFK': None, 'KLGA': None, 'KEWR': None, 'KTEB': None,
'KIAD': None, 'KDCA': None, 'KBWI': None, 'KRIC': None,
'KPHL': None
}
def load_stations(txn):
"""load station metadata to build a xref of stations to networks"""
txn.execute("""
"""
Support SPC's MCD product
Support WPC's FFG product
"""
from twisted.python import log
from twisted.internet import reactor
from pyiem.nws.products.mcd import parser as mcdparser
from pyldm import ldmbridge
import common
DBPOOL = common.get_database(common.CONFIG["databaserw"]["postgis"], cp_max=2)
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
""" Called when the STDIN connection is lost """
log.msg("connectionLost")
log.err(reason)
reactor.callLater(15, reactor.callWhenRunning, reactor.stop)
def process_data(self, data):
""" Process a chunk of data """
# BUG
data = data.upper()
df = DBPOOL.runInteraction(real_process, data)
df.addErrback(common.email_error, data)
"""SPC Geo Products Parser!"""
from twisted.python import log
from twisted.internet import reactor
from pyldm import ldmbridge
from pyiem.nws.products.spcpts import parser
import common # @UnresolvedImport
DBPOOL = common.get_database("postgis")
WAITFOR = 20
JABBER = common.make_jabber_client("spc_parser")
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
"""shutdown"""
log.msg("connectionLost")
log.err(reason)
reactor.callLater(5, self.shutdown) # @UndefinedVariable
def shutdown(self):
"""shutdown"""
log.msg("shutdown() is called")
reactor.callWhenRunning(reactor.stop) # @UndefinedVariable
def process_data(self, data):
""" Process the product """
df = DBPOOL.runInteraction(real_parser, data)
df.addErrback(common.email_error, data)
"""Process and Archive the NEXRAD Level III NCR Attribute Table"""
from io import BytesIO
import math
import pytz
from twisted.python import log
from twisted.internet import reactor
from metpy.io.nexrad import Level3File
from pyldm import ldmbridge
import common
# Setup Database Links
POSTGISDB = common.get_database('postgis')
ST = {}
def load_station_table(txn):
""" Load the station table of NEXRAD sites """
log.msg("load_station_table called() ...")
txn.execute("""
SELECT id, ST_x(geom) as lon, ST_y(geom) as lat from stations
where network in ('NEXRAD','TWDR')
""")
for row in txn.fetchall():
ST[row['id']] = {'lat': row['lat'],
'lon': row['lon']}
log.msg("Station Table size %s" % (len(ST.keys(),)))
class MyProductIngestor(ldmbridge.LDMProductReceiver):
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/hml_parser', facility=LOG_LOCAL2)
# Twisted Python imports
from twisted.internet import reactor
from twisted.python import log
# Standard Python modules
import datetime
import common
import os
from pyldm import ldmbridge
from pyiem.nws.products.hml import parser as hmlparser
DBPOOL = common.get_database("hads")
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
log.msg('connectionLost')
log.err(reason)
reactor.callLater(5, self.shutdown)
def shutdown(self):
reactor.callWhenRunning(reactor.stop)
def process_data(self, buf):
nwsli_dict[row['nwsli']] = nwsli.NWSLI(row['nwsli'], name=nm)
log.msg("nwsli_dict loaded %s entries" % (len(nwsli_dict),))
return None
def ready(dummy):
''' cb when our database work is done '''
ldmbridge.LDMProductFactory(MyProductIngestor())
def dbload():
''' Load up database stuff '''
df = PGCONN.runInteraction(load_ugc)
df.addCallback(ready)
if __name__ == '__main__':
MANUAL = False
if len(sys.argv) == 2 and sys.argv[1] == 'manual':
log.msg("Manual runtime (no jabber, 1 database connection) requested")
MANUAL = True
# Fire up!
PGCONN = common.get_database("postgis", cp_max=(5 if not MANUAL else 1))
dbload()
jabber = common.make_jabber_client('generic_parser')
reactor.run()
""" FFG """
from twisted.python import log
from twisted.internet import reactor
from pyldm import ldmbridge
from pyiem.nws.products.ffg import parser
import common # @UnresolvedImport
DBPOOL = common.get_database("postgis", cp_max=1)
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
"""Connection was lost"""
log.msg("connectionLost")
log.err(reason)
reactor.callLater(5, shutdown) # @UndefinedVariable
def process_data(self, data):
"""Process the product"""
df = DBPOOL.runInteraction(real_parser, data)
df.addErrback(common.email_error, data)
def shutdown():
"""Go shutdown"""
reactor.callWhenRunning(reactor.stop) # @UndefinedVariable
""" ASOS Daily Summary Message Parser ingestor """
import pytz
from twisted.internet import reactor
from twisted.python import log
from pyldm import ldmbridge
from pyiem.nws.products.dsm import parser
from pyiem.util import get_dbconn
import common
DBPOOL = common.get_database("iem", cp_max=1)
# database timezones to pytz cache
TIMEZONES = dict()
STATIONS = dict()
def load_stations(txn):
"""load station metadata to build a xref."""
txn.execute("""
SELECT id, tzname from stations
where network ~* 'ASOS' or network = 'AWOS'
""")
for row in txn:
# we need four char station IDs
station = row[0] if len(row[0]) == 4 else "K" + row[0]
tzname = row[1]
if tzname not in TIMEZONES:
try:
TIMEZONES[tzname] = pytz.timezone(tzname)
except Exception as exp:
log.msg("pytz does not like tzname: %s %s" % (tzname, exp))
""" Twisted Way to dump data to the database """
import sys
import datetime
import pytz
from twisted.python import log
from twisted.internet import reactor
from txyam.client import YamClient
from pyldm import ldmbridge
from pyiem.nws import product
from pyiem.util import utc
import common # @UnresolvedImport
DBPOOL = common.get_database("afos")
MEMCACHE_EXCLUDE = [
"RR1",
"RR2",
"RR3",
"RR4",
"RR5",
"RR6",
"RR7",
"RR8",
"RR9",
"ROB",
"HML",
]
MEMCACHE_CLIENT = YamClient(reactor, ["tcp:iem-memcached3:11211"])
MEMCACHE_CLIENT.connect()
""" LSR product ingestor """
import pickle
import os
import datetime
import pytz
from twisted.python import log
from twisted.internet import reactor
from pyiem import reference
from pyiem.nws.products.lsr import parser as lsrparser
from pyldm import ldmbridge
import common
DBPOOL = common.get_database(common.CONFIG['databaserw']['postgis'])
# Cheap datastore for LSRs to avoid Dups!
LSRDB = {}
def loaddb():
''' load memory '''
if os.path.isfile('lsrdb.p'):
mydict = pickle.load(open('lsrdb.p', 'rb'))
for key in mydict:
LSRDB[key] = mydict[key]
def cleandb():
""" To keep LSRDB from growing too big, we clean it out
Lets hold 7 days of data!
"""
from pyiem.reference import TRACE_VALUE
from pyiem.observation import Observation
from pyiem.nws import product
from pyiem import reference
from pyldm import ldmbridge
import common # @UnresolvedImport
# from pympler import tracker, summary, muppy
# TR = tracker.SummaryTracker()
# Setup Database Links
# the current_shef table is not very safe when two processes attempt to update
# it at the same time, use a single process for this connection
ACCESSDB = common.get_database('iem', module_name='psycopg2', cp_max=20)
HADSDB = common.get_database('hads', module_name='psycopg2', cp_max=20)
# a form for IDs we will log as unknown
NWSLIRE = re.compile("[A-Z]{4}[0-9]")
# stations we don't know about
UNKNOWN = dict()
# station metadata
LOCS = dict()
# database timezones to pytz cache
TIMEZONES = dict()
# a queue for saving database IO
CURRENT_QUEUE = {}
U1980 = datetime.datetime.utcnow()
U1980 = U1980.replace(day=1, year=1980, tzinfo=pytz.utc)
"""SPS product ingestor"""
import re
from twisted.python import log
from twisted.internet import reactor
from pyiem.nws.products.sps import parser
from pyiem.nws.ugc import UGC
from pyldm import ldmbridge
import common
POSTGIS = common.get_database("postgis")
PYWWA_PRODUCT_URL = common.SETTINGS.get(
"pywwa_product_url", "pywwa_product_url"
)
ugc_provider = {}
def load_ugc(txn):
""" load ugc dict """
# Careful here not to load things from the future
txn.execute(
"""
SELECT name, ugc, wfo from ugcs WHERE
name IS NOT Null and begin_ts < now() and
(end_ts is null or end_ts > now())
"""
)
for row in txn.fetchall():
nm = (row["name"]).replace("\x92", " ").replace("\xc2", " ")
wfos = re.findall(r"([A-Z][A-Z][A-Z])", row["wfo"])
