def main(): """Go Main Go""" mesosite = common.get_database("mesosite", cp_max=1) df = mesosite.runInteraction(load_station_table) df.addCallback(on_ready, mesosite) df.addErrback(errback) reactor.run()
def save_if_required(self, wx, force=False): """! Save data if not in database already, unless forced @param wx Data to save into database if not already present @param force Whether or not to load data if it is already in database @return Timestamp for loaded run """ # HACK: somehow still getting duplicates index = wx.index.names # NOTE: need to reset_index() because otherwise only compares wx values wx = wx.reset_index().drop_duplicates().set_index(index) # HACK: Timestamp format is nicer than datetime's for_run = wx.reset_index()['Generated'].max() # want to put the data into the database for the start date but check if it exists based on for_run start_date = wx.reset_index()['ForTime'].min() dbname = common.get_database(start_date) # if at any point for_run is already in the database then we're done if not force: logging.debug( 'Checking if data is already present for {} model generated at {}' .format(self.name, for_run)) exists = self.check_exists(for_run, dbname) if exists: logging.debug('Data already loaded - aborting') return pandas.Timestamp(for_run) self.save_data(wx, dbname) # return the run that we ended up loading data for return for_run
def dbtest(): webdb = common.get_database( "/home/vikrant/programming/work/publicgit/gridpath/mh.db") scenario1 = "rpo30_pass1" scenario2 = 'rpo30_pass2' project = 'Koyna_Stage_3' adjusted_mean_results(webdb, scenario1, scenario2, project)
def load_specific_records(self, for_run, force=False): """! Load records for specific period @param self Pointer to self @param for_run Which run of model to use @param force Whether or not to force loading if records already exist @return Timestamp of run that records were loaded for """ # save into database that corresponds to the start of this run # want to put the data into the database for the start date but check if it exists based on for_run start_date = for_run + datetime.timedelta(hours=self.lead_time) dbname = common.get_database(start_date) # if at any point for_run is already in the database then we're done if not force: logging.debug( 'Checking if data is already present for {} model generated at {}' .format(self.name, for_run)) exists = self.check_exists(for_run, dbname) if exists: logging.debug('Data already loaded - aborting') return pandas.Timestamp(for_run) results = [] for d in self.for_days: results.append(self.get_records(for_run, d)) # don't save data until everything is loaded wx = pandas.concat(results) self.save_data(wx, dbname) # return the run that we ended up loading data for # HACK: Timestamp format is nicer than datetime's return pandas.Timestamp(for_run)
def test_compare_with_db(): def get_db_results(): subscenario_id = get_hydro_ops_chars_scenario_id( webdb, compare_scenario, project) rows = webdb.where( "inputs_project_hydro_operational_chars", project=project, hydro_operational_chars_scenario_id=subscenario_id).list() return pd.DataFrame(rows) gridpath = "/home/vikrant/programming/work/publicgit/gridpath-0.8.1/gridpath" database = os.path.join(gridpath, "mh.db") csv_location = os.path.join(gridpath, "db", "csvs_mh") scenario1 = "rpo50S3_pass1_2" scenario2 = "rpo50S3_pass2" compare_scenario = "rpo50S3_pass2" project = "Sardar_Sarovar_CHPH" webdb = common.get_database(database) subscenario = "hydro_operational_chars_scenario_id" print(f"Computing data for {project}") subscenario_id = get_hydro_ops_chars_scenario_id(webdb, scenario2, project) results = adjusted_mean_results(webdb, scenario1, scenario2, project) write_results_csv(results, project, subscenario, subscenario_id, csv_location, "rpo50S3_all") csvpath = common.get_subscenario_csvpath(project, subscenario, subscenario_id, csv_location, "rpo50S3_all") filedata = pd.read_csv(csvpath) filedata.set_index("horizon", inplace=True) dailyfile = filedata[filedata.balancing_type_project == "month"] b1 = dailyfile['average_power_fraction'] dbdata = get_db_results() dbdata.set_index('horizon', inplace=True) dailydb = dbdata[dbdata.balancing_type_project == "month"] b2 = dailydb['average_power_fraction'] printcols(b1, b2, b1.index, b2.index) diff = abs(b1 - b2) >= 0.001 printcols(b1[diff], b2[diff], b1.index[diff], b2.index[diff]) assert np.all(abs(b1 - b2) <= 0.001)
def hydro_op_chars(database, csv_location, gridpath_repo, scenario1, scenario2, description, project, update_database): webdb = common.get_database(database) projects = get_projects(webdb, scenario1) if project: projects = [project] #projects[:1] subscenario = "hydro_operational_chars_scenario_id" for project_ in projects: print(f"Computing data for {project_}") subscenario_id = get_hydro_ops_chars_scenario_id( webdb, scenario2, project_) results = adjusted_mean_results(webdb, scenario1, scenario2, project_) write_results_csv(results, project_, subscenario, subscenario_id, csv_location, description) if update_database: common.update_subscenario_via_gridpath(subscenario, subscenario_id, project_, csv_location, database, gridpath_repo)
""" We are called with a hard coded AFOS PIL """ tp = TextProduct(data) if tp.afos is None: compute_afos(tp) sql = ("INSERT into products " "(pil, data, source, wmo, entered) values(%s,%s,%s,%s,%s)") sqlargs = ( tp.afos, tp.text, tp.source, tp.wmo, tp.valid.strftime("%Y-%m-%d %H:%M+00"), ) txn.execute(sql, sqlargs) if tp.afos[:3] == "FRH": return jmsgs = tp.get_jabbers( common.SETTINGS.get("pywwa_product_url", "pywwa_product_url")) for jmsg in jmsgs: JABBER.send_message(*jmsg) if __name__ == "__main__": PGCONN = common.get_database("afos", cp_max=1) JABBER = common.make_jabber_client("fake_afos_dump") ldmbridge.LDMProductFactory(MyProductIngestor()) reactor.run()
from pyiem.reference import TRACE_VALUE from pyiem.observation import Observation from pyiem.nws import product from pyiem import reference from pyldm import ldmbridge import common # @UnresolvedImport # from pympler import tracker, summary, muppy # TR = tracker.SummaryTracker() # Setup Database Links # the current_shef table is not very safe when two processes attempt to update # it at the same time, use a single process for this connection ACCESSDB = common.get_database("iem", module_name="psycopg2", cp_max=20) HADSDB = common.get_database("hads", module_name="psycopg2", cp_max=20) # a form for IDs we will log as unknown NWSLIRE = re.compile("[A-Z]{4}[0-9]") # stations we don't know about UNKNOWN = dict() # station metadata LOCS = dict() # database timezones to pytz cache TIMEZONES = dict() # a queue for saving database IO CURRENT_QUEUE = {} U1980 = datetime.datetime.utcnow() U1980 = U1980.replace(day=1, year=1980, tzinfo=pytz.utc)
def init_db_cayley(): client = init_client(MongoDBConfig.MONGODB_HOST, MongoDBConfig.MONGODB_PORT) db = get_database(client, MongoDBConfig.DATABASE_NAME) return client, db
syslog.startLogging(prefix="pyWWA/mos_parser", facility=LOG_LOCAL2) # Twisted Python imports from twisted.internet import reactor # Standard Python modules import re import datetime import pytz # pyWWA stuff from pyldm import ldmbridge import common DBPOOL = common.get_database("mos") class myProductIngestor(ldmbridge.LDMProductReceiver): def process_data(self, buf): """ Actual ingestor """ try: real_process(buf) except Exception, myexp: common.email_error(myexp, buf) def connectionLost(self, reason): """ Called when ldm closes the pipe
""" SPC Geo Products Parser! """ # Twisted Python imports from syslog import LOG_LOCAL2 from twisted.python import syslog syslog.startLogging(prefix='pyWWA/spc_parser', facility=LOG_LOCAL2) from twisted.python import log from twisted.internet import reactor # pyWWA stuff from pyldm import ldmbridge from pyiem.nws import product, spcpts import common DBPOOL = common.get_database('postgis') WAITFOR = 20 # LDM Ingestor class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): log.msg('connectionLost') log.err(reason) reactor.callLater(5, self.shutdown) def shutdown(self): reactor.callWhenRunning(reactor.stop) def process_data(self, buf):
NOTE: It is difficult to keep track of where I am getting the `Metar` library. So let us document it here for my own sanity. 18 Jul 2017: `snowdepth` branch of my python-metar fork installed with pip """ import sys # Twisted Python imports from twisted.python import log from twisted.internet import reactor from pyiem.nws.products import metarcollect from pyiem.util import get_properties from pyldm import ldmbridge import common # @UnresolvedImport IEMDB = common.get_database("iem") ASOSDB = common.get_database("asos") MANUAL = "manual" in [x.lower() for x in sys.argv] NWSLI_PROVIDER = {} # Manual list of sites that are sent to jabber :/ metarcollect.JABBER_SITES = { "KJFK": None, "KLGA": None, "KEWR": None, "KTEB": None, "KIAD": None, "KDCA": None, "KBWI": None, "KRIC": None, "KPHL": None,
"""SPENES product ingestor""" import re from twisted.python import log from twisted.internet import reactor from pyldm import ldmbridge from pyiem.nws import product import common POSTGIS = common.get_database("postgis", cp_max=1) PYWWA_PRODUCT_URL = common.SETTINGS.get("pywwa_product_url", "pywwa_product_url") DB_ON = bool(common.SETTINGS.get("pywwa_save_text_products", False)) def shutdown(): """ Stop this app """ log.msg("Shutting down...") reactor.callWhenRunning(reactor.stop) class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): """ callback when the stdin reader connection is closed """ log.msg("connectionLost() called...") log.err(reason) reactor.callLater(7, shutdown) def process_data(self, data): """ Process the product """
log.msg("nwsli_dict loaded %s entries" % (len(nwsli_dict), )) return None def ready(_): """ cb when our database work is done """ ldmbridge.LDMProductFactory(MyProductIngestor()) def dbload(): """ Load up database stuff """ df = PGCONN.runInteraction(load_ugc) df.addCallback(ready) if __name__ == "__main__": MANUAL = False if len(sys.argv) == 2 and sys.argv[1] == "manual": log.msg("Manual runtime (no jabber, 1 database connection) requested") MANUAL = True # Fire up! PGCONN = common.get_database("postgis", cp_max=(5 if not MANUAL else 1)) dbload() jabber = common.make_jabber_client("generic_parser") reactor.run()
""" PIREP parser! """ import datetime import os from twisted.internet import reactor from twisted.python import log from pyldm import ldmbridge from pyiem.nws.products.pirep import parser as pirepparser import common TABLESDIR = os.path.join(os.path.dirname(__file__), "../tables") PIREPS = {} DBPOOL = common.get_database("postgis") JABBER = common.make_jabber_client('pirep') # Load LOCS table LOCS = {} def cleandb(): """ To keep LSRDB from growing too big, we clean it out Lets hold 1 days of data! """ thres = datetime.datetime.utcnow() - datetime.timedelta(hours=24*1) init_size = len(PIREPS.keys()) for key in PIREPS: if PIREPS[key] < thres: del PIREPS[key] fin_size = len(PIREPS.keys()) log.msg("cleandb() init_size: %s final_size: %s" % (init_size, fin_size))
""" NESDIS SCP Ingestor """ from twisted.python import log from twisted.internet import reactor from pyiem.nws.products.scp import parser from pyldm import ldmbridge import common # @UnresolvedImport DBPOOL = common.get_database("asos", cp_max=1) def shutdown(): """Shut things down, please""" reactor.callWhenRunning(reactor.stop) # @UndefinedVariable class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): """STDIN is shut, so lets shutdown""" log.msg("connectionLost") log.err(reason) reactor.callLater(7, shutdown) # @UndefinedVariable def process_data(self, data): """Process the product!""" df = DBPOOL.runInteraction(real_process, data) df.addErrback(common.email_error, data) def real_process(txn, raw):
""" Aviation Product Parser! """ import common import os # Twisted Python imports from twisted.internet import reactor from syslog import LOG_LOCAL2 from twisted.python import syslog syslog.startLogging(prefix='pyWWA/aviation', facility=LOG_LOCAL2) from twisted.python import log # pyWWA stuff from pyldm import ldmbridge from pyiem.nws.products.sigmet import parser DBPOOL = common.get_database('postgis') # Load LOCS table LOCS = {} _MYDIR = os.path.dirname(os.path.abspath(__file__)) TABLE_PATH = os.path.normpath(os.path.join(_MYDIR, "..", "tables")) def load_database(txn): txn.execute(""" SELECT id, name, ST_x(geom) as lon, ST_y(geom) as lat from stations WHERE network ~* 'ASOS' or network ~* 'AWOS' """) for row in txn:
from pyiem import wellknowntext import re import pytz import dbflib import shapelib import zipfile import os import shutil import subprocess import common syslog.startLogging(prefix="pyWWA/stoia_parser", facility=LOG_LOCAL2) EPSG26915 = """PROJCS["NAD_1983_UTM_Zone_15N",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-93.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]""" DBPOOL = common.get_database("postgis", cp_max=1) CONDITIONS = {} ROADS = {} # Changedir to /tmp os.chdir("/tmp") def shutdown(): """ Down we go! """ log.msg("Stopping...") reactor.callWhenRunning(reactor.stop) class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """
""" NLDN """ from io import BytesIO from twisted.internet import reactor from pyldm import ldmbridge from pyiem.nws.products.nldn import parser import common DBPOOL = common.get_database("nldn") class myProductIngestor(ldmbridge.LDMProductReceiver): """My hacky ingest""" product_end = b"NLDN" def process_data(self, data): """Actual ingestor""" if data == b"": return real_process(data) try: real_process(data) except Exception as myexp: common.email_error(myexp, data) def connectionLost(self, reason): """ Called when ldm closes the pipe """ print("connectionLost")
"""Process and Archive the NEXRAD Level III NCR Attribute Table""" from io import BytesIO import math import pytz from twisted.python import log from twisted.internet import reactor from metpy.io.nexrad import Level3File from pyldm import ldmbridge import common # Setup Database Links PGCONN = common.get_database("radar") ST = {} def load_station_table(txn): """ Load the station table of NEXRAD sites """ log.msg("load_station_table called() ...") txn.execute( "SELECT id, ST_x(geom) as lon, ST_y(geom) as lat from stations " "where network in ('NEXRAD','TWDR')") for row in txn.fetchall(): ST[row["id"]] = {"lat": row["lat"], "lon": row["lon"]} log.msg("Station Table size %s" % (len(ST.keys()))) class MyProductIngestor(ldmbridge.LDMProductReceiver): """My ingest protocol""" def connectionLost(self, reason):
""" SPC Watch Ingestor """ from twisted.python import log from twisted.internet import reactor from pyiem.nws.products.saw import parser as sawparser from pyldm import ldmbridge import common # @UnresolvedImport DBPOOL = common.get_database('postgis', cp_max=1) IEM_URL = common.SETTINGS.get('pywwa_watch_url', 'pywwa_watch_url') JABBER = common.make_jabber_client("new_watch") def shutdown(): """Shut things down, please""" reactor.callWhenRunning(reactor.stop) # @UndefinedVariable class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): """STDIN is shut, so lets shutdown""" log.msg('connectionLost') log.err(reason) reactor.callLater(7, shutdown) # @UndefinedVariable def process_data(self, data): """Process the product!""" df = DBPOOL.runInteraction(real_process, data) df.addErrback(common.email_error, data)
The CLI report has lots of good data that is hard to find in other products, so we take what data we find in this product and overwrite the database storage of what we got from the automated observations """ from pyiem.nws.products import parser from pyldm import ldmbridge from pyiem.network import Table as NetworkTable import common from twisted.internet import reactor from syslog import LOG_LOCAL2 from twisted.python import syslog syslog.startLogging(prefix='pyWWA/cli_parser', facility=LOG_LOCAL2) from twisted.python import log DBPOOL = common.get_database('iem', cp_max=1) NT = NetworkTable("NWSCLI") # LDM Ingestor class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): """ Connection was lost! """ log.err(reason) reactor.callLater(7, reactor.callWhenRunning, reactor.stop) def process_data(self, text): """ Process the product """ deffer = DBPOOL.runInteraction(preprocessor, text)
"""SPENES product ingestor""" # Twisted Python imports from syslog import LOG_LOCAL2 from twisted.python import syslog from twisted.python import log syslog.startLogging(prefix='pyWWA/spe_parser', facility=LOG_LOCAL2) from twisted.internet import reactor import re import common # pyLDM https://github.com/akrherz/pyLDM from pyldm import ldmbridge from pyiem.nws import product POSTGIS = common.get_database('postgis', cp_max=1) PYWWA_PRODUCT_URL = common.settings.get('pywwa_product_url', 'pywwa_product_url') DB_ON = bool(common.settings.get('pywwa_save_text_products', False)) def shutdown(): ''' Stop this app ''' log.msg("Shutting down...") reactor.callWhenRunning(reactor.stop) class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason):
""" MOS Data Ingestor, why not? """ from __future__ import print_function from twisted.internet import reactor from pyldm import ldmbridge from pyiem.nws.products.mos import parser import common # @UnresolvedImport DBPOOL = common.get_database('mos') MEMORY = {'ingested': 0} class MyProductIngestor(ldmbridge.LDMProductReceiver): """Do what we do!""" def process_data(self, data): """ Actual ingestor """ try: real_process(data) except Exception as myexp: common.email_error(myexp, data) def connectionLost(self, reason): """ Called when ldm closes the pipe """ reactor.callLater(5, self.shutdown) def shutdown(self):
""" HML parser! """ from twisted.internet import reactor from twisted.python import log from pyldm import ldmbridge from pyiem.nws.products.hml import parser as hmlparser import common # @UnresolvedImport DBPOOL = common.get_database("hml") # LDM Ingestor class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): """Connection was lost""" log.msg("connectionLost") log.err(reason) reactor.callLater(5, reactor.stop) # @UndefinedVariable def process_data(self, data): """ Process the product """ defer = DBPOOL.runInteraction(real_parser, data) defer.addErrback(common.email_error, data) defer.addErrback(log.err) def real_parser(txn, buf): """I'm gonna do the heavy lifting here""" prod = hmlparser(buf)
log.msg("nwsli_dict loaded %s entries" % (len(nwsli_dict),)) return None def ready(dummy): ''' cb when our database work is done ''' ldmbridge.LDMProductFactory(MyProductIngestor(dedup=True)) if __name__ == '__main__': syslog.startLogging(prefix='pyWWA/vtec_parser', facility=LOG_LOCAL2) HTTPClientFactory.noisy = False SMTPSenderFactory.noisy = False ugc_dict = {} nwsli_dict = {} MANUAL = False if len(sys.argv) == 2 and sys.argv[1] == 'manual': log.msg("Manual runtime (no jabber, 1 database connection) requested") MANUAL = True # Fire up! PGCONN = common.get_database(common.config['databaserw']['postgis'], cp_max=(5 if not MANUAL else 1)) df = PGCONN.runInteraction(load_ugc) df.addCallback(ready) df.addErrback(common.email_error, "load_ugc failure!") jabber = common.make_jabber_client('vtec_parser') reactor.run()
def ready(_dummy): """ cb when our database work is done """ ldmbridge.LDMProductFactory(MyProductIngestor(dedup=True)) def bootstrap(): """Things to do at startup""" df = PGCONN.runInteraction(load_ugc) df.addCallback(ready) df.addErrback(common.email_error, "load_ugc failure!") if __name__ == "__main__": SMTPSenderFactory.noisy = False ugc_dict = {} nwsli_dict = {} MANUAL = False if len(sys.argv) == 2 and sys.argv[1].lower() == "manual": log.msg("Manual runtime (no jabber, 1 database connection) requested") MANUAL = True # Fire up! PGCONN = common.get_database(common.CONFIG["databaserw"]["postgis"], cp_max=(5 if not MANUAL else 1)) bootstrap() jabber = common.make_jabber_client("vtec_parser") reactor.run()
from syslog import LOG_LOCAL2 from twisted.python import syslog syslog.startLogging(prefix='pyWWA/lsr_parser', facility=LOG_LOCAL2) from twisted.python import log from twisted.internet import reactor # Third party python stuff import pytz from pyiem import reference from pyiem.nws.products.lsr import parser as lsrparser from pyldm import ldmbridge # IEM python Stuff import common import datetime DBPOOL = common.get_database(common.config['databaserw']['postgis']) # Cheap datastore for LSRs to avoid Dups! LSRDB = {} def loaddb(): ''' load memory ''' if os.path.isfile('lsrdb.p'): mydict = pickle.load(open('lsrdb.p')) for key in mydict: LSRDB[key] = mydict[key] def cleandb(): """ To keep LSRDB from growing too big, we clean it out
""" ASOS Daily Summary Message Parser ingestor """ import pytz from twisted.internet import reactor from twisted.python import log from pyldm import ldmbridge from pyiem.nws.products.dsm import parser from pyiem.util import get_dbconn import common DBPOOL = common.get_database("iem", cp_max=1) # database timezones to pytz cache TIMEZONES = dict() STATIONS = dict() def load_stations(txn): """load station metadata to build a xref.""" txn.execute(""" SELECT id, tzname from stations where network ~* 'ASOS' or network = 'AWOS' """) for row in txn: # we need four char station IDs station = row[0] if len(row[0]) == 4 else "K" + row[0] tzname = row[1] if tzname not in TIMEZONES: try: TIMEZONES[tzname] = pytz.timezone(tzname) except Exception as exp: log.msg("pytz does not like tzname: %s %s" % (tzname, exp))
def really_process_data(txn, data): ''' We are called with a hard coded AFOS PIL ''' tp = TextProduct(data) if tp.afos is None: compute_afos(tp) utc = tp.valid table = "products_%s_0106" % (utc.year,) if utc.month > 6: table = "products_%s_0712" % (utc.year,) sql = """INSERT into """ + table + """ (pil, data, source, wmo, entered) values(%s,%s,%s,%s,%s)""" sqlargs = (tp.afos, tp.text, tp.source, tp.wmo, utc.strftime("%Y-%m-%d %H:%M+00")) txn.execute(sql, sqlargs) if tp.afos[:3] == 'FRH': return jmsgs = tp.get_jabbers( common.SETTINGS.get('pywwa_product_url', 'pywwa_product_url')) for jmsg in jmsgs: JABBER.send_message(*jmsg) if __name__ == '__main__': PGCONN = common.get_database("afos", cp_max=1) JABBER = common.make_jabber_client('fake_afos_dump') ldmbridge.LDMProductFactory(MyProductIngestor()) reactor.run()
NOTE: It is difficult to keep track of where I am getting the `Metar` library. So let us document it here for my own sanity. 18 Jul 2017: `snowdepth` branch of my python-metar fork installed with pip """ from __future__ import print_function import sys # Twisted Python imports from twisted.python import log from twisted.internet import reactor from pyiem.nws.products import metarcollect from pyldm import ldmbridge import common # @UnresolvedImport IEMDB = common.get_database('iem') ASOSDB = common.get_database('asos') MANUAL = ('MANUAL' in sys.argv) NWSLI_PROVIDER = {} # Manual list of sites that are sent to jabber :/ metarcollect.JABBER_SITES = { 'KJFK': None, 'KLGA': None, 'KEWR': None, 'KTEB': None, 'KIAD': None, 'KDCA': None, 'KBWI': None, 'KRIC': None, 'KPHL': None } def load_stations(txn): """load station metadata to build a xref of stations to networks""" txn.execute("""
""" Support SPC's MCD product Support WPC's FFG product """ from twisted.python import log from twisted.internet import reactor from pyiem.nws.products.mcd import parser as mcdparser from pyldm import ldmbridge import common DBPOOL = common.get_database(common.CONFIG["databaserw"]["postgis"], cp_max=2) # LDM Ingestor class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): """ Called when the STDIN connection is lost """ log.msg("connectionLost") log.err(reason) reactor.callLater(15, reactor.callWhenRunning, reactor.stop) def process_data(self, data): """ Process a chunk of data """ # BUG data = data.upper() df = DBPOOL.runInteraction(real_process, data) df.addErrback(common.email_error, data)
"""SPC Geo Products Parser!""" from twisted.python import log from twisted.internet import reactor from pyldm import ldmbridge from pyiem.nws.products.spcpts import parser import common # @UnresolvedImport DBPOOL = common.get_database("postgis") WAITFOR = 20 JABBER = common.make_jabber_client("spc_parser") # LDM Ingestor class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): """shutdown""" log.msg("connectionLost") log.err(reason) reactor.callLater(5, self.shutdown) # @UndefinedVariable def shutdown(self): """shutdown""" log.msg("shutdown() is called") reactor.callWhenRunning(reactor.stop) # @UndefinedVariable def process_data(self, data): """ Process the product """ df = DBPOOL.runInteraction(real_parser, data) df.addErrback(common.email_error, data)
"""Process and Archive the NEXRAD Level III NCR Attribute Table""" from io import BytesIO import math import pytz from twisted.python import log from twisted.internet import reactor from metpy.io.nexrad import Level3File from pyldm import ldmbridge import common # Setup Database Links POSTGISDB = common.get_database('postgis') ST = {} def load_station_table(txn): """ Load the station table of NEXRAD sites """ log.msg("load_station_table called() ...") txn.execute(""" SELECT id, ST_x(geom) as lon, ST_y(geom) as lat from stations where network in ('NEXRAD','TWDR') """) for row in txn.fetchall(): ST[row['id']] = {'lat': row['lat'], 'lon': row['lon']} log.msg("Station Table size %s" % (len(ST.keys(),))) class MyProductIngestor(ldmbridge.LDMProductReceiver):
from twisted.python import syslog syslog.startLogging(prefix='pyWWA/hml_parser', facility=LOG_LOCAL2) # Twisted Python imports from twisted.internet import reactor from twisted.python import log # Standard Python modules import datetime import common import os from pyldm import ldmbridge from pyiem.nws.products.hml import parser as hmlparser DBPOOL = common.get_database("hads") # LDM Ingestor class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): log.msg('connectionLost') log.err(reason) reactor.callLater(5, self.shutdown) def shutdown(self): reactor.callWhenRunning(reactor.stop) def process_data(self, buf):
nwsli_dict[row['nwsli']] = nwsli.NWSLI(row['nwsli'], name=nm) log.msg("nwsli_dict loaded %s entries" % (len(nwsli_dict),)) return None def ready(dummy): ''' cb when our database work is done ''' ldmbridge.LDMProductFactory(MyProductIngestor()) def dbload(): ''' Load up database stuff ''' df = PGCONN.runInteraction(load_ugc) df.addCallback(ready) if __name__ == '__main__': MANUAL = False if len(sys.argv) == 2 and sys.argv[1] == 'manual': log.msg("Manual runtime (no jabber, 1 database connection) requested") MANUAL = True # Fire up! PGCONN = common.get_database("postgis", cp_max=(5 if not MANUAL else 1)) dbload() jabber = common.make_jabber_client('generic_parser') reactor.run()
""" FFG """ from twisted.python import log from twisted.internet import reactor from pyldm import ldmbridge from pyiem.nws.products.ffg import parser import common # @UnresolvedImport DBPOOL = common.get_database("postgis", cp_max=1) # LDM Ingestor class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): """Connection was lost""" log.msg("connectionLost") log.err(reason) reactor.callLater(5, shutdown) # @UndefinedVariable def process_data(self, data): """Process the product""" df = DBPOOL.runInteraction(real_parser, data) df.addErrback(common.email_error, data) def shutdown(): """Go shutdown""" reactor.callWhenRunning(reactor.stop) # @UndefinedVariable
""" Twisted Way to dump data to the database """ import sys import datetime import pytz from twisted.python import log from twisted.internet import reactor from txyam.client import YamClient from pyldm import ldmbridge from pyiem.nws import product from pyiem.util import utc import common # @UnresolvedImport DBPOOL = common.get_database("afos") MEMCACHE_EXCLUDE = [ "RR1", "RR2", "RR3", "RR4", "RR5", "RR6", "RR7", "RR8", "RR9", "ROB", "HML", ] MEMCACHE_CLIENT = YamClient(reactor, ["tcp:iem-memcached3:11211"]) MEMCACHE_CLIENT.connect()
""" LSR product ingestor """ import pickle import os import datetime import pytz from twisted.python import log from twisted.internet import reactor from pyiem import reference from pyiem.nws.products.lsr import parser as lsrparser from pyldm import ldmbridge import common DBPOOL = common.get_database(common.CONFIG['databaserw']['postgis']) # Cheap datastore for LSRs to avoid Dups! LSRDB = {} def loaddb(): ''' load memory ''' if os.path.isfile('lsrdb.p'): mydict = pickle.load(open('lsrdb.p', 'rb')) for key in mydict: LSRDB[key] = mydict[key] def cleandb(): """ To keep LSRDB from growing too big, we clean it out Lets hold 7 days of data! """
from pyiem.reference import TRACE_VALUE from pyiem.observation import Observation from pyiem.nws import product from pyiem import reference from pyldm import ldmbridge import common # @UnresolvedImport # from pympler import tracker, summary, muppy # TR = tracker.SummaryTracker() # Setup Database Links # the current_shef table is not very safe when two processes attempt to update # it at the same time, use a single process for this connection ACCESSDB = common.get_database('iem', module_name='psycopg2', cp_max=20) HADSDB = common.get_database('hads', module_name='psycopg2', cp_max=20) # a form for IDs we will log as unknown NWSLIRE = re.compile("[A-Z]{4}[0-9]") # stations we don't know about UNKNOWN = dict() # station metadata LOCS = dict() # database timezones to pytz cache TIMEZONES = dict() # a queue for saving database IO CURRENT_QUEUE = {} U1980 = datetime.datetime.utcnow() U1980 = U1980.replace(day=1, year=1980, tzinfo=pytz.utc)
"""SPS product ingestor""" import re from twisted.python import log from twisted.internet import reactor from pyiem.nws.products.sps import parser from pyiem.nws.ugc import UGC from pyldm import ldmbridge import common POSTGIS = common.get_database("postgis") PYWWA_PRODUCT_URL = common.SETTINGS.get( "pywwa_product_url", "pywwa_product_url" ) ugc_provider = {} def load_ugc(txn): """ load ugc dict """ # Careful here not to load things from the future txn.execute( """ SELECT name, ugc, wfo from ugcs WHERE name IS NOT Null and begin_ts < now() and (end_ts is null or end_ts > now()) """ ) for row in txn.fetchall(): nm = (row["name"]).replace("\x92", " ").replace("\xc2", " ") wfos = re.findall(r"([A-Z][A-Z][A-Z])", row["wfo"])