def init_db(): import plenario.models from plenario.utils.weather import WeatherETL, WeatherStationsETL from plenario.utils.shapefile_helpers import PlenarioShapeETL import datetime print 'creating master, meta and user tables' Base.metadata.create_all(bind=app_engine) if plenario.settings.DEFAULT_USER: print 'creating default user %s' % plenario.settings.DEFAULT_USER[ 'name'] user = plenario.models.User(**plenario.settings.DEFAULT_USER) session.add(user) session.commit() print 'initializing NOAA weather stations' s = WeatherStationsETL() s.initialize() print 'initializing NOAA daily and hourly weather observations for %s/%s' % ( datetime.datetime.now().month, datetime.datetime.now().year) print 'this will take a few minutes ...' e = WeatherETL() e.initialize_month(datetime.datetime.now().year, datetime.datetime.now().month) print 'initializing and populating US Census blocks' print 'this will *also* take a few minutes ...' shp = PlenarioShapeETL(plenario.settings.CENSUS_BLOCKS) shp.add()
def init_db(): import plenario.models from plenario.utils.weather import WeatherETL, WeatherStationsETL from plenario.utils.shapefile_helpers import PlenarioShapeETL import datetime print 'creating master, meta and user tables' Base.metadata.create_all(bind=app_engine) if plenario.settings.DEFAULT_USER: print 'creating default user %s' % plenario.settings.DEFAULT_USER['name'] user = plenario.models.User(**plenario.settings.DEFAULT_USER) session.add(user) session.commit() print 'initializing NOAA weather stations' s = WeatherStationsETL() s.initialize() print 'initializing NOAA daily and hourly weather observations for %s/%s' % (datetime.datetime.now().month, datetime.datetime.now().year) print 'this will take a few minutes ...' e = WeatherETL() e.initialize_month(datetime.datetime.now().year, datetime.datetime.now().month) print 'initializing and populating US Census blocks' print 'this will *also* take a few minutes ...' shp = PlenarioShapeETL(plenario.settings.CENSUS_BLOCKS) shp.add()
def update_metar(): print "update_metar()" celery_metar_illinois_area_wbans = [u'14855', u'54808', u'14834', u'04838', u'04876', u'03887', u'04871', u'04873', u'04831', u'04879', u'04996', u'14880', u'04899', u'94892', u'94891', u'04890', u'54831', u'94870', u'04894', u'94854', u'14842', u'93822', u'04807', u'04808', u'54811', u'94822', u'94846', u'04868', u'04845', u'04896', u'04867', u'04866', u'04889', u'14816', u'04862', u'94866', u'04880', u'14819'] ohare_mdw= ['94846', '14819'] w = WeatherETL() w.metar_initialize_current(weather_stations_list = celery_metar_illinois_area_wbans) #w.metar_initialize_current(weather_stations_list = ohare_mdw) return 'Added current metars'
def update_metar() -> bool: """Run a METAR update. """ logger.info('Begin.') w = WeatherETL() logger.debug('Call metar initialization method.') w.metar_initialize_current() logger.info('End.') return True
def init_weather(): print 'initializing NOAA weather stations' s = WeatherStationsETL() s.initialize() print 'initializing NOAA daily and hourly weather observations for %s/%s' % (datetime.datetime.now().month, datetime.datetime.now().year) print 'this will take a few minutes ...' e = WeatherETL() e.initialize_month(datetime.datetime.now().year, datetime.datetime.now().month)
def init_weather(): print 'initializing NOAA weather stations' s = WeatherStationsETL() s.initialize() print 'initializing NOAA daily and hourly weather observations for %s/%s' % (datetime.datetime.now().month, datetime.datetime.now().year) print 'this will take a few minutes ...' e = WeatherETL() try: e.initialize_month(datetime.datetime.now().year, datetime.datetime.now().month) except Exception as e: session.rollback() raise e
def update_metar(): print "update_metar()" celery_metar_illinois_area_wbans = [ u'14855', u'54808', u'14834', u'04838', u'04876', u'03887', u'04871', u'04873', u'04831', u'04879', u'04996', u'14880', u'04899', u'94892', u'94891', u'04890', u'54831', u'94870', u'04894', u'94854', u'14842', u'93822', u'04807', u'04808', u'54811', u'94822', u'94846', u'04868', u'04845', u'04896', u'04867', u'04866', u'04889', u'14816', u'04862', u'94866', u'04880', u'14819' ] ohare_mdw = ['94846', '14819'] w = WeatherETL() w.metar_initialize_current( weather_stations_list=celery_metar_illinois_area_wbans) #w.metar_initialize_current(weather_stations_list = ohare_mdw) return 'Added current metars'
def update_metar(): print "update_metar()" celery_metar_illinois_area_wbans = [ u"14855", u"54808", u"14834", u"04838", u"04876", u"03887", u"04871", u"04873", u"04831", u"04879", u"04996", u"14880", u"04899", u"94892", u"94891", u"04890", u"54831", u"94870", u"04894", u"94854", u"14842", u"93822", u"04807", u"04808", u"54811", u"94822", u"94846", u"04868", u"04845", u"04896", u"04867", u"04866", u"04889", u"14816", u"04862", u"94866", u"04880", u"14819", ] ohare_mdw = ["94846", "14819"] w = WeatherETL() w.metar_initialize_current(weather_stations_list=celery_metar_illinois_area_wbans) # w.metar_initialize_current(weather_stations_list = ohare_mdw) return "Added current metars"
def init(): """Initialize the database. """ # TODO(heyzoos) # Check for dependencies to fail fast and helpfully before running: # - postgresql-client base_uri = DATABASE_CONN.rsplit('/', 1)[0] base_engine = create_engine(base_uri) connection_attempts = 6 interval = 10 for connection_attempt in range(0, connection_attempts): try: create_database(base_engine, DB_NAME) break except ProgrammingError: logger.debug('[plenario] It already exists!') break except sqlalchemy.exc.OperationalError: logger.debug('[plenario] Database has not started yet.') sleep(interval) try: create_extension(plenario_engine, 'postgis') create_extension(plenario_engine, 'plv8') except ProgrammingError: logger.debug('[plenario] It already exists!') logger.debug('[plenario] Creating metadata tables') postgres_base.metadata.create_all() logger.debug('[plenario] Creating weather tables') WeatherStationsETL().make_station_table() WeatherETL().make_tables() # Set up custom functions, triggers and views in postgres psql('./plenario/dbscripts/sensor_tree.sql') psql('./plenario/dbscripts/point_from_location.sql') # Set up the default user if we are running in anything but production if os.environ.get('CONFIG') != 'prod': logger.debug('[plenario] Create default user') user = User(**DEFAULT_USER) try: postgres_session.add(user) postgres_session.commit() except IntegrityError: logger.debug('[plenario] Already exists!') postgres_session.rollback() # This will get celery to set up its meta tables try: health.delay() except OperationalError: logger.debug('[plenario] Redis is not running!')
def update_weather(month=None, year=None, wbans=None) -> bool: """Run a weather update. """ # This should do the current month AND the previous month, just in case. last_month_dt = datetime.now() - timedelta(days=7) last_month = last_month_dt.month last_year = last_month_dt.year if not month: month = datetime.now().month if not year: year = datetime.now().year w = WeatherETL() if last_month != month: w.initialize_month(last_year, last_month, weather_stations_list=wbans) w.initialize_month(year, month, weather_stations_list=wbans) return True
def clean_metar() -> bool: """Given the latest datetime available in hourly observations table, delete all metar records older than that datetime. Records which exist in the hourly table are the quality-controlled versions of records that existed in the metar table. """ logger.info('Begin.') WeatherETL().clear_metars() logger.info('End.') return True
def update_weather(): # This should do the current month AND the previous month, just in case. lastMonth_dt = datetime.now() - timedelta(days=1) lastMonth = lastMonth_dt.month lastYear = lastMonth_dt.year month, year = datetime.now().month, datetime.now().year w = WeatherETL() if lastMonth != month: w.initialize_month(lastYear, lastMonth) w.initialize_month(year, month) # Given that this was the most recent month, year, call this function, # which will figure out the most recent hourly weather observation and # delete all metars before that datetime. w.clear_metars() return 'Added weather for %s %s' % (month, year)
def update_weather(): # This should do the current month AND the previous month, just in case. lastMonth_dt = datetime.now() - timedelta(days=1) lastMonth = lastMonth_dt.month lastYear = lastMonth_dt.year month, year = datetime.now().month, datetime.now().year #stations = ['12921', '94740', '14855', '03981', '93894', '94746', '93816', '93904', '03887', '14752', '93810', '53981', '04953', '14776', '14777', '53983', '54723', '14770', '14771', '94765', '94761', '93986', '14778', '03968', '53952', '53950', '94892', '94891', '54757', '13802', '23293', '54734', '54735', '54733', '04789', '64775', '04787', '64776', '04783', '54738', '54739', '93823', '93822', '14740', '23272', '53802', '54811', '13922', '03985', '14742', '03972', '04868', '14747', '53822', '93227', '93817', '93228', '14748', '04867', '04866', '03879', '23289', '53891', '04862', '53944', '53947', '64705', '53942', '64707', '23203', '53935', '54704', '63853', '94982', '14834', '64761', '04751', '04876', '04871', '04873', '04739', '04879', '93232', '93231', '14758', '13902', '14754', '23240', '94741', '14757', '14750', '94979', '94745', '14753', '53939', '12910', '03868', '14923', '03902', '53933', '54831', '94870', '13904', '14775', '64758', '13911', '04996', '64753', '12911', '63840', '63841', '64757', '64756', '13909', '04903', '23250', '23257', '04807', '23254', '04724', '04808', '04726', '23258', '04720', '03999', '13984', '94908', '13986', '23285', '04889', '14931', '54789', '54788', '64706', '94868', '14733', '94866', '54782', '54781', '03948', '54787', '54786', '54785', '03919', '14719', '54768', '54760', '04925', '93997', '54767', '94721', '94723', '94725', '03969', '53999', '04725', '23244', '94728', '12961', '23907', '14739', '53913', '54792', '54793', '14732', '93211', '04950', '14736', '14737', '14734', '14735', '04899', '53938', '04890', '03928', '04896', '04894', '94854', '53997', '94822', '54778', '54779', '13966', '13967', '04949', '14786', '54770', '94959', '54777', '23239', '94733', '93943', '04947', '14703', '14702', '12979', '04742', '23230', '04741', '14990', '23234', '53986', '12971', '94846', '23237', '03932', '04930', '53909', '53897', '54808', '13919', '14816', '03838', '14819', '03933', '63814', '63817', '63810', '53886', '13958', '14790', '14792', '54756', '14794', '93942', '54790', '04838', '93978', '14712', '13961', '14714', '14715', '14717', '63878', '04831', '94705', '94704', '94702', '54773', '13999', '14880', '93947', '03958', '94789', '93984', '93985', '53977', '53976', '64774', '53979', '93989', '54742', '53150', '23211', '13940', '14937', '13945', '04921', '14842', '53887', '04880', '53889', '54771', '04781', '94794', '94790', '14707', '54780', '14761', '14760', '14763', '14762', '04845', '14768', '54746', '53964', '53965', '54743', '63901', '54740', '13809', '03950', '53969', '03957', '03954', '94737'] stations = ['94846', '14855', '04807', '14819', '94866', '04831', '04838'] w = WeatherETL() if (lastMonth != month): w.initialize_month(lastYear, lastMonth, weather_stations_list=stations) w.initialize_month(year, month, weather_stations_list=stations) # Given that this was the most recent month, year, call this function, # which will figure out the most recent hourly weather observation and # delete all metars before that datetime. w.clear_metars(year, month, weather_stations_list=stations) return 'Added weather for %s %s' % (month, year)
def update_weather(): # This should do the current month AND the previous month, just in case. lastMonth_dt = datetime.now() - timedelta(days=1) lastMonth = lastMonth_dt.month lastYear = lastMonth_dt.year month, year = datetime.now().month, datetime.now().year #stations = ['12921', '94740', '14855', '03981', '93894', '94746', '93816', '93904', '03887', '14752', '93810', '53981', '04953', '14776', '14777', '53983', '54723', '14770', '14771', '94765', '94761', '93986', '14778', '03968', '53952', '53950', '94892', '94891', '54757', '13802', '23293', '54734', '54735', '54733', '04789', '64775', '04787', '64776', '04783', '54738', '54739', '93823', '93822', '14740', '23272', '53802', '54811', '13922', '03985', '14742', '03972', '04868', '14747', '53822', '93227', '93817', '93228', '14748', '04867', '04866', '03879', '23289', '53891', '04862', '53944', '53947', '64705', '53942', '64707', '23203', '53935', '54704', '63853', '94982', '14834', '64761', '04751', '04876', '04871', '04873', '04739', '04879', '93232', '93231', '14758', '13902', '14754', '23240', '94741', '14757', '14750', '94979', '94745', '14753', '53939', '12910', '03868', '14923', '03902', '53933', '54831', '94870', '13904', '14775', '64758', '13911', '04996', '64753', '12911', '63840', '63841', '64757', '64756', '13909', '04903', '23250', '23257', '04807', '23254', '04724', '04808', '04726', '23258', '04720', '03999', '13984', '94908', '13986', '23285', '04889', '14931', '54789', '54788', '64706', '94868', '14733', '94866', '54782', '54781', '03948', '54787', '54786', '54785', '03919', '14719', '54768', '54760', '04925', '93997', '54767', '94721', '94723', '94725', '03969', '53999', '04725', '23244', '94728', '12961', '23907', '14739', '53913', '54792', '54793', '14732', '93211', '04950', '14736', '14737', '14734', '14735', '04899', '53938', '04890', '03928', '04896', '04894', '94854', '53997', '94822', '54778', '54779', '13966', '13967', '04949', '14786', '54770', '94959', '54777', '23239', '94733', '93943', '04947', '14703', '14702', '12979', '04742', '23230', '04741', '14990', '23234', '53986', '12971', '94846', '23237', '03932', '04930', '53909', '53897', '54808', '13919', '14816', '03838', '14819', '03933', '63814', '63817', '63810', '53886', '13958', '14790', '14792', '54756', '14794', '93942', '54790', '04838', '93978', '14712', '13961', '14714', '14715', '14717', '63878', '04831', '94705', '94704', '94702', '54773', '13999', '14880', '93947', '03958', '94789', '93984', '93985', '53977', '53976', '64774', '53979', '93989', '54742', '53150', '23211', '13940', '14937', '13945', '04921', '14842', '53887', '04880', '53889', '54771', '04781', '94794', '94790', '14707', '54780', '14761', '14760', '14763', '14762', '04845', '14768', '54746', '53964', '53965', '54743', '63901', '54740', '13809', '03950', '53969', '03957', '03954', '94737'] #stations = ['94846', '14855', '04807', '14819', '94866', '04831', '04838'] w = WeatherETL() if (lastMonth != month): w.initialize_month(lastYear, lastMonth) w.initialize_month(year, month) # Given that this was the most recent month, year, call this function, # which will figure out the most recent hourly weather observation and # delete all metars before that datetime. w.clear_metars() return 'Added weather for %s %s' % (month, year)
def update_metar(): print "update_metar()" w = WeatherETL() w.metar_initialize_current() return 'Added current metars'
def update_weather(): month, year = datetime.now().month, datetime.now().year stations = ['94846', '14855', '04807', '14819', '94866', '04831', '04838'] w = WeatherETL() w.initialize_month(year, month, weather_stations_list=stations) return 'Added weather for %s %s' % (month, year)