def workflow(ts, irealtime, justprecip): """Do Work""" # load up our current data ds = iemre.get_grids(ts) LOG.debug("loaded %s variables from IEMRE database", len(ds)) # For this date, the 12 UTC COOP obs will match the date if not justprecip: LOG.debug("doing 12z logic for %s", ts) grid_day12(ts, ds) do_precip12(ts, ds) # This is actually yesterday! if irealtime: iemre.set_grids(ts, ds) subprocess.call( "python db_to_netcdf.py %s" % (ts.strftime("%Y %m %d"),), shell=True, ) ts -= datetime.timedelta(days=1) ds = iemre.get_grids(ts) if not justprecip: LOG.debug("doing calendar day logic for %s", ts) grid_day(ts, ds) do_precip(ts, ds) LOG.debug("calling iemre.set_grids()") iemre.set_grids(ts, ds) subprocess.call( "python db_to_netcdf.py %s" % (ts.strftime("%Y %m %d"),), shell=True )
def test_get_grids(): """Can we get grids?""" pgconn = get_dbconn('iemre') cursor = pgconn.cursor() valid = utc(2019, 12, 1, 1) cursor.execute( """ DELETE from iemre_hourly_201912 WHERE valid = %s """, (valid, )) cursor.execute( """ DELETE from iemre_hourly_201912 WHERE valid = %s """, (valid + datetime.timedelta(days=1), )) cursor.execute( """ INSERT into iemre_hourly_201912 (gid, valid, tmpk, dwpk, uwnd, vwnd, p01m) select gid, %s, random(), null, random(), random(), random() from iemre_grid """, (valid, )) ds = iemre.get_grids(valid, varnames='tmpk', cursor=cursor) assert 'tmpk' in ds assert 'bogus' not in ds ds = iemre.get_grids(valid, cursor=cursor) assert np.isnan(ds['dwpk'].values.max()) iemre.set_grids(valid, ds, cursor=cursor) iemre.set_grids(valid + datetime.timedelta(days=1), ds, cursor=cursor)
def try_merra(ts): """Attempt to use MERRA data.""" # Our files are UTC date based :/ ncfn1 = ts.strftime("/mesonet/merra2/%Y/%Y%m%d.nc") ncfn2 = ( ts + datetime.timedelta(days=1) ).strftime("/mesonet/merra2/%Y/%Y%m%d.nc") if not os.path.isfile(ncfn1) or not os.path.isfile(ncfn2): return False with ncopen(ncfn1) as nc: # Total up from 6z to end of file for today total = np.sum(nc.variables['SWGDN'][5:, :, :], axis=0) with ncopen(ncfn2) as nc: lat1d = nc.variables['lat'][:] lon1d = nc.variables['lon'][:] # Total up to 6z total += np.sum(nc.variables['SWGDN'][:6, :, :], axis=0) # We wanna store as W m-2, so we just average out the data by hour total = total / 24.0 lons, lats = np.meshgrid(lon1d, lat1d) nn = NearestNDInterpolator( (lons.flatten(), lats.flatten()), total.flatten() ) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) ds = iemre.get_grids(ts.date(), varnames='rsds') ds['rsds'].values = nn(xi, yi) iemre.set_grids(ts.date(), ds) subprocess.call( "python db_to_netcdf.py %s" % (ts.strftime("%Y %m %d"), ), shell=True) return True
def do_coop(ts): """Use COOP solar radiation data""" pgconn = get_dbconn('coop', user='******') cursor = pgconn.cursor() cursor.execute("""SELECT ST_x(geom), ST_y(geom), coalesce(narr_srad, merra_srad) from alldata a JOIN stations t ON (a.station = t.id) WHERE day = %s and t.network ~* 'CLIMATE' and substr(id, 3, 1) != 'C' and substr(id, 3, 4) != '0000' """, (ts.strftime("%Y-%m-%d"), )) lons = [] lats = [] vals = [] for row in cursor: if row[2] is None or row[2] < 0: continue lons.append(row[0]) lats.append(row[1]) vals.append(row[2]) nn = NearestNDInterpolator((np.array(lons), np.array(lats)), np.array(vals)) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) ds = iemre.get_grids(ts.date(), varnames='rsds') # Convert MJ/d to Wm2 ds['rsds'].values = nn(xi, yi) * 1000000. / 86400. iemre.set_grids(ts.date(), ds) subprocess.call( "python db_to_netcdf.py %s" % (ts.strftime("%Y %m %d"), ), shell=True)
def test_forecast_grids(): """Test getting and setting grids from the future.""" pgconn = get_dbconn("iemre") cursor = pgconn.cursor() valid = datetime.date(2029, 12, 1) cursor.execute( """ DELETE from iemre_daily_forecast WHERE valid = %s """, (valid, ), ) cursor.execute( """ INSERT into iemre_daily_forecast (gid, valid, high_tmpk, low_tmpk, p01d, rsds) select gid, %s, random(), random(), random(), random() from iemre_grid LIMIT 100 """, (valid, ), ) ds = iemre.get_grids(valid, cursor=cursor, table="iemre_daily_forecast") assert "high_tmpk" in ds assert "bogus" not in ds iemre.set_grids(valid, ds, cursor=cursor, table="iemre_daily_forecast")
def main(argv): """Do work please""" day = datetime.date(int(argv[1]), int(argv[2]), int(argv[3])) pgconn = get_dbconn('coop') df = read_sql(""" SELECT a.precip, st_x(t.geom) as lon, st_y(t.geom) as lat from alldata a JOIN stations t ON (a.station = t.id) WHERE a.day = %s and t.network ~* 'CLIMATE' and substr(a.station,3,4) != '0000' and substr(station,3,1) != 'C' and precip >= 0 and precip < 50 """, pgconn, params=(day, )) res = generic_gridder(day, df, 'precip') if res is not None: ds = get_grids(day, varnames='p01d_12z') ds['p01d_12z'].values = res.to(mpunits('mm')).magnitude[0, :, :] subprocess.call("python db_to_netcdf.py %s" % ( day.strftime("%Y %m %d"), ), shell=True)
def main(argv): """Go Main Go.""" log = logger() if len(argv) == 6: valid = utc(int(argv[1]), int(argv[2]), int(argv[3]), int(argv[4])) ncfn = iemre.get_hourly_ncname(valid.year) idx = iemre.hourly_offset(valid) else: valid = datetime.date(int(argv[1]), int(argv[2]), int(argv[3])) ncfn = iemre.get_daily_ncname(valid.year) idx = iemre.daily_offset(valid) ds = iemre.get_grids(valid) with ncopen(ncfn, 'a', timeout=600) as nc: for vname in ds: if vname not in nc.variables: continue log.debug("copying database var %s to netcdf", vname) nc.variables[vname][idx, :, :] = ds[vname].values
def main(argv): """Go Main Go.""" date = datetime.date(int(argv[1]), int(argv[2]), int(argv[3])) ds = iemre.get_grids(date) pgconn = get_dbconn("coop") for state in state_names: if state in ["AK", "HI", "DC"]: continue table = "alldata_%s" % (state, ) cursor = pgconn.cursor() df = load_table(state, date) df = merge_network_obs(df, "%s_COOP" % (state, ), date) df = merge_network_obs(df, "%s_ASOS" % (state, ), date) estimate_hilo(df, ds) estimate_precip(df, ds) estimate_snow(df, ds) if not commit(cursor, table, df, date): return cursor.close() pgconn.commit()
def do_hrrr(ts): """Convert the hourly HRRR data to IEMRE grid""" LCC = pyproj.Proj( ("+lon_0=-97.5 +y_0=0.0 +R=6367470. +proj=lcc +x_0=0.0" " +units=m +lat_2=38.5 +lat_1=38.5 +lat_0=38.5") ) total = None xaxis = None yaxis = None # So IEMRE is storing data from coast to coast, so we should be # aggressive about running for an entire calendar date now = ts.replace(hour=1) for _ in range(24): now += datetime.timedelta(hours=1) utcnow = now.astimezone(pytz.UTC) fn = utcnow.strftime( ("/mesonet/ARCHIVE/data/%Y/%m/%d/model/hrrr/%H/" "hrrr.t%Hz.3kmf00.grib2") ) if not os.path.isfile(fn): continue grbs = pygrib.open(fn) try: if utcnow >= SWITCH_DATE: grb = grbs.select(name='Downward short-wave radiation flux') else: grb = grbs.select(parameterNumber=192) except ValueError: # don't complain about late evening no-solar if utcnow.hour > 10 and utcnow.hour < 24: print('iemre/grid_rsds.py %s had no solar rad' % (fn,)) continue if not grb: print('Could not find SWDOWN in HRR %s' % (fn,)) continue g = grb[0] if total is None: total = g.values lat1 = g['latitudeOfFirstGridPointInDegrees'] lon1 = g['longitudeOfFirstGridPointInDegrees'] llcrnrx, llcrnry = LCC(lon1, lat1) nx = g['Nx'] ny = g['Ny'] dx = g['DxInMetres'] dy = g['DyInMetres'] xaxis = llcrnrx + dx * np.arange(nx) yaxis = llcrnry + dy * np.arange(ny) else: total += g.values if total is None: print(('iemre/grid_rsds.py found no HRRR data for %s' ) % (ts.strftime("%d %b %Y"), )) return # We wanna store as W m-2, so we just average out the data by hour total = total / 24.0 ds = iemre.get_grids(ts.date(), varnames='rsds') for i, lon in enumerate(iemre.XAXIS): for j, lat in enumerate(iemre.YAXIS): (x, y) = LCC(lon, lat) i2 = np.digitize([x], xaxis)[0] j2 = np.digitize([y], yaxis)[0] try: ds['rsds'].values[j, i] = total[j2, i2] except IndexError: continue iemre.set_grids(ts.date(), ds) subprocess.call( "python db_to_netcdf.py %s" % (ts.strftime("%Y %m %d"), ), shell=True)
def main(argv): """Go Main Go.""" year = int(argv[1]) sts = datetime.date(year, 1, 1) ets = min([datetime.date(year, 12, 31), datetime.date.today()]) current = {} now = ets while now >= sts: ds = iemre.get_grids(now, varnames="power_swdn") maxval = ds["power_swdn"].values.max() if np.isnan(maxval) or maxval < 0: LOG.debug("adding %s as currently empty", now) current[now] = {"data": ds, "dirty": False} now -= datetime.timedelta(days=1) sts = min(list(current.keys())) ets = max(list(current.keys())) LOG.debug("running between %s and %s", sts, ets) queue = [] for x0 in np.arange(iemre.WEST, iemre.EAST, 5.0): for y0 in np.arange(iemre.SOUTH, iemre.NORTH, 5.0): queue.append([x0, y0]) for x0, y0 in tqdm(queue, disable=not sys.stdout.isatty()): url = ( "https://power.larc.nasa.gov/cgi-bin/v1/DataAccess.py?" "request=execute&identifier=Regional&" "parameters=ALLSKY_SFC_SW_DWN&" "startDate=%s&endDate=%s&userCommunity=SSE&" "tempAverage=DAILY&bbox=%s,%s,%s,%s&user=anonymous&" "outputList=NETCDF" ) % ( sts.strftime("%Y%m%d"), ets.strftime("%Y%m%d"), y0, x0, min([y0 + 5.0, iemre.NORTH]) - 0.1, min([x0 + 5.0, iemre.EAST]) - 0.1, ) req = exponential_backoff(requests.get, url, timeout=60) js = req.json() if "outputs" not in js: LOG.debug(url) LOG.debug(str(js)) continue fn = js["outputs"]["netcdf"] req = requests.get(fn, timeout=60, stream=True) ncfn = "/tmp/power%s.nc" % (year,) with open(ncfn, "wb") as fh: for chunk in req.iter_content(chunk_size=1024): if chunk: fh.write(chunk) fh.close() with ncopen(ncfn) as nc: for day, _ in enumerate(nc.variables["time"][:]): date = sts + datetime.timedelta(days=day) if date not in current: continue # kwh to MJ/d 3600 * 1000 / 1e6 data = nc.variables["ALLSKY_SFC_SW_DWN"][day, :, :] * 3.6 # Sometimes there are missing values? if np.ma.is_masked(data): data[data.mask] = np.mean(data) i, j = iemre.find_ij(x0, y0) # resample data is 0.5, iemre is 0.125 data = np.repeat(np.repeat(data, 4, axis=0), 4, axis=1) data = np.where(data < 0, np.nan, data) shp = np.shape(data) jslice = slice(j, j + shp[0]) islice = slice(i, i + shp[1]) # get currentdata present = current[date]["data"]["power_swdn"].values[ jslice, islice ] if present.mean() == data.mean(): continue current[date]["data"]["power_swdn"].values[ jslice, islice ] = data current[date]["dirty"] = True for date in current: if not current[date]["dirty"]: continue LOG.debug("saving %s", date) iemre.set_grids(date, current[date]["data"]) subprocess.call( "python ../iemre/db_to_netcdf.py %s" % (date.strftime("%Y %m %d"),), shell=True, )