def try_merra(ts): """Attempt to use MERRA data.""" # Our files are UTC date based :/ ncfn1 = ts.strftime("/mesonet/merra2/%Y/%Y%m%d.nc") ncfn2 = ( ts + datetime.timedelta(days=1) ).strftime("/mesonet/merra2/%Y/%Y%m%d.nc") if not os.path.isfile(ncfn1) or not os.path.isfile(ncfn2): return False with ncopen(ncfn1) as nc: # Total up from 6z to end of file for today total = np.sum(nc.variables['SWGDN'][5:, :, :], axis=0) with ncopen(ncfn2) as nc: lat1d = nc.variables['lat'][:] lon1d = nc.variables['lon'][:] # Total up to 6z total += np.sum(nc.variables['SWGDN'][:6, :, :], axis=0) # We wanna store as W m-2, so we just average out the data by hour total = total / 24.0 lons, lats = np.meshgrid(lon1d, lat1d) nn = NearestNDInterpolator( (lons.flatten(), lats.flatten()), total.flatten() ) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) ds = iemre.get_grids(ts.date(), varnames='rsds') ds['rsds'].values = nn(xi, yi) iemre.set_grids(ts.date(), ds) subprocess.call( "python db_to_netcdf.py %s" % (ts.strftime("%Y %m %d"), ), shell=True) return True
def workflow(ts, irealtime, justprecip): """Do Work""" # load up our current data ds = iemre.get_grids(ts) LOG.debug("loaded %s variables from IEMRE database", len(ds)) # For this date, the 12 UTC COOP obs will match the date if not justprecip: LOG.debug("doing 12z logic for %s", ts) grid_day12(ts, ds) do_precip12(ts, ds) # This is actually yesterday! if irealtime: iemre.set_grids(ts, ds) subprocess.call( "python db_to_netcdf.py %s" % (ts.strftime("%Y %m %d"),), shell=True, ) ts -= datetime.timedelta(days=1) ds = iemre.get_grids(ts) if not justprecip: LOG.debug("doing calendar day logic for %s", ts) grid_day(ts, ds) do_precip(ts, ds) LOG.debug("calling iemre.set_grids()") iemre.set_grids(ts, ds) subprocess.call( "python db_to_netcdf.py %s" % (ts.strftime("%Y %m %d"),), shell=True )
def do_coop(ts): """Use COOP solar radiation data""" pgconn = get_dbconn('coop', user='******') cursor = pgconn.cursor() cursor.execute("""SELECT ST_x(geom), ST_y(geom), coalesce(narr_srad, merra_srad) from alldata a JOIN stations t ON (a.station = t.id) WHERE day = %s and t.network ~* 'CLIMATE' and substr(id, 3, 1) != 'C' and substr(id, 3, 4) != '0000' """, (ts.strftime("%Y-%m-%d"), )) lons = [] lats = [] vals = [] for row in cursor: if row[2] is None or row[2] < 0: continue lons.append(row[0]) lats.append(row[1]) vals.append(row[2]) nn = NearestNDInterpolator((np.array(lons), np.array(lats)), np.array(vals)) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) ds = iemre.get_grids(ts.date(), varnames='rsds') # Convert MJ/d to Wm2 ds['rsds'].values = nn(xi, yi) * 1000000. / 86400. iemre.set_grids(ts.date(), ds) subprocess.call( "python db_to_netcdf.py %s" % (ts.strftime("%Y %m %d"), ), shell=True)
def test_get_grids(): """Can we get grids?""" pgconn = get_dbconn('iemre') cursor = pgconn.cursor() valid = utc(2019, 12, 1, 1) cursor.execute( """ DELETE from iemre_hourly_201912 WHERE valid = %s """, (valid, )) cursor.execute( """ DELETE from iemre_hourly_201912 WHERE valid = %s """, (valid + datetime.timedelta(days=1), )) cursor.execute( """ INSERT into iemre_hourly_201912 (gid, valid, tmpk, dwpk, uwnd, vwnd, p01m) select gid, %s, random(), null, random(), random(), random() from iemre_grid """, (valid, )) ds = iemre.get_grids(valid, varnames='tmpk', cursor=cursor) assert 'tmpk' in ds assert 'bogus' not in ds ds = iemre.get_grids(valid, cursor=cursor) assert np.isnan(ds['dwpk'].values.max()) iemre.set_grids(valid, ds, cursor=cursor) iemre.set_grids(valid + datetime.timedelta(days=1), ds, cursor=cursor)
def test_forecast_grids(): """Test getting and setting grids from the future.""" pgconn = get_dbconn("iemre") cursor = pgconn.cursor() valid = datetime.date(2029, 12, 1) cursor.execute( """ DELETE from iemre_daily_forecast WHERE valid = %s """, (valid, ), ) cursor.execute( """ INSERT into iemre_daily_forecast (gid, valid, high_tmpk, low_tmpk, p01d, rsds) select gid, %s, random(), random(), random(), random() from iemre_grid LIMIT 100 """, (valid, ), ) ds = iemre.get_grids(valid, cursor=cursor, table="iemre_daily_forecast") assert "high_tmpk" in ds assert "bogus" not in ds iemre.set_grids(valid, ds, cursor=cursor, table="iemre_daily_forecast")
def do_hrrr(ts): """Convert the hourly HRRR data to IEMRE grid""" LCC = pyproj.Proj( ("+lon_0=-97.5 +y_0=0.0 +R=6367470. +proj=lcc +x_0=0.0" " +units=m +lat_2=38.5 +lat_1=38.5 +lat_0=38.5") ) total = None xaxis = None yaxis = None # So IEMRE is storing data from coast to coast, so we should be # aggressive about running for an entire calendar date now = ts.replace(hour=1) for _ in range(24): now += datetime.timedelta(hours=1) utcnow = now.astimezone(pytz.UTC) fn = utcnow.strftime( ("/mesonet/ARCHIVE/data/%Y/%m/%d/model/hrrr/%H/" "hrrr.t%Hz.3kmf00.grib2") ) if not os.path.isfile(fn): continue grbs = pygrib.open(fn) try: if utcnow >= SWITCH_DATE: grb = grbs.select(name='Downward short-wave radiation flux') else: grb = grbs.select(parameterNumber=192) except ValueError: # don't complain about late evening no-solar if utcnow.hour > 10 and utcnow.hour < 24: print('iemre/grid_rsds.py %s had no solar rad' % (fn,)) continue if not grb: print('Could not find SWDOWN in HRR %s' % (fn,)) continue g = grb[0] if total is None: total = g.values lat1 = g['latitudeOfFirstGridPointInDegrees'] lon1 = g['longitudeOfFirstGridPointInDegrees'] llcrnrx, llcrnry = LCC(lon1, lat1) nx = g['Nx'] ny = g['Ny'] dx = g['DxInMetres'] dy = g['DyInMetres'] xaxis = llcrnrx + dx * np.arange(nx) yaxis = llcrnry + dy * np.arange(ny) else: total += g.values if total is None: print(('iemre/grid_rsds.py found no HRRR data for %s' ) % (ts.strftime("%d %b %Y"), )) return # We wanna store as W m-2, so we just average out the data by hour total = total / 24.0 ds = iemre.get_grids(ts.date(), varnames='rsds') for i, lon in enumerate(iemre.XAXIS): for j, lat in enumerate(iemre.YAXIS): (x, y) = LCC(lon, lat) i2 = np.digitize([x], xaxis)[0] j2 = np.digitize([y], yaxis)[0] try: ds['rsds'].values[j, i] = total[j2, i2] except IndexError: continue iemre.set_grids(ts.date(), ds) subprocess.call( "python db_to_netcdf.py %s" % (ts.strftime("%Y %m %d"), ), shell=True)
def main(argv): """Go Main Go.""" year = int(argv[1]) sts = datetime.date(year, 1, 1) ets = min([datetime.date(year, 12, 31), datetime.date.today()]) current = {} now = ets while now >= sts: ds = iemre.get_grids(now, varnames="power_swdn") maxval = ds["power_swdn"].values.max() if np.isnan(maxval) or maxval < 0: LOG.debug("adding %s as currently empty", now) current[now] = {"data": ds, "dirty": False} now -= datetime.timedelta(days=1) sts = min(list(current.keys())) ets = max(list(current.keys())) LOG.debug("running between %s and %s", sts, ets) queue = [] for x0 in np.arange(iemre.WEST, iemre.EAST, 5.0): for y0 in np.arange(iemre.SOUTH, iemre.NORTH, 5.0): queue.append([x0, y0]) for x0, y0 in tqdm(queue, disable=not sys.stdout.isatty()): url = ( "https://power.larc.nasa.gov/cgi-bin/v1/DataAccess.py?" "request=execute&identifier=Regional&" "parameters=ALLSKY_SFC_SW_DWN&" "startDate=%s&endDate=%s&userCommunity=SSE&" "tempAverage=DAILY&bbox=%s,%s,%s,%s&user=anonymous&" "outputList=NETCDF" ) % ( sts.strftime("%Y%m%d"), ets.strftime("%Y%m%d"), y0, x0, min([y0 + 5.0, iemre.NORTH]) - 0.1, min([x0 + 5.0, iemre.EAST]) - 0.1, ) req = exponential_backoff(requests.get, url, timeout=60) js = req.json() if "outputs" not in js: LOG.debug(url) LOG.debug(str(js)) continue fn = js["outputs"]["netcdf"] req = requests.get(fn, timeout=60, stream=True) ncfn = "/tmp/power%s.nc" % (year,) with open(ncfn, "wb") as fh: for chunk in req.iter_content(chunk_size=1024): if chunk: fh.write(chunk) fh.close() with ncopen(ncfn) as nc: for day, _ in enumerate(nc.variables["time"][:]): date = sts + datetime.timedelta(days=day) if date not in current: continue # kwh to MJ/d 3600 * 1000 / 1e6 data = nc.variables["ALLSKY_SFC_SW_DWN"][day, :, :] * 3.6 # Sometimes there are missing values? if np.ma.is_masked(data): data[data.mask] = np.mean(data) i, j = iemre.find_ij(x0, y0) # resample data is 0.5, iemre is 0.125 data = np.repeat(np.repeat(data, 4, axis=0), 4, axis=1) data = np.where(data < 0, np.nan, data) shp = np.shape(data) jslice = slice(j, j + shp[0]) islice = slice(i, i + shp[1]) # get currentdata present = current[date]["data"]["power_swdn"].values[ jslice, islice ] if present.mean() == data.mean(): continue current[date]["data"]["power_swdn"].values[ jslice, islice ] = data current[date]["dirty"] = True for date in current: if not current[date]["dirty"]: continue LOG.debug("saving %s", date) iemre.set_grids(date, current[date]["data"]) subprocess.call( "python ../iemre/db_to_netcdf.py %s" % (date.strftime("%Y %m %d"),), shell=True, )