def workflow(ts, irealtime, justprecip): """Do Work""" # Load up our netcdf file! ncfn = iemre.get_daily_ncname(ts.year) if not os.path.isfile(ncfn): print("will create %s" % (ncfn, )) cmd = "python init_daily.py %s" % (ts.year, ) subprocess.call(cmd, shell=True) nc = ncopen(ncfn, 'a', timeout=600) domain = nc.variables['hasdata'][:, :] nc.close() # For this date, the 12 UTC COOP obs will match the date if not justprecip: grid_day12(ts, domain) do_precip12(ts) # This is actually yesterday! if irealtime: ts -= datetime.timedelta(days=1) ncfn = iemre.get_daily_ncname(ts.year) if not os.path.isfile(ncfn): print("will create %s" % (ncfn, )) cmd = "python init_daily.py %s" % (ts.year, ) subprocess.call(cmd, shell=True) if not justprecip: grid_day(ts, domain) do_precip(ts)
def write_grid(valid, vname, grid): """Write data to backend netcdf""" offset = iemre.daily_offset(valid) nc = ncopen(iemre.get_daily_ncname(valid.year), 'a', timeout=600) if nc is None: print("daily_analysis#write_grid first open attempt failed, try #2") nc = ncopen(iemre.get_daily_ncname(valid.year), 'a', timeout=600) print(("%13s [idx:%s] min: %6.2f max: %6.2f [%s]") % (vname, offset, np.nanmin(grid), np.nanmax(grid), nc.variables[vname].units)) nc.variables[vname][offset] = grid nc.close()
def main(): """go Main go""" for yr in range(1893, 2016): fn = iemre.get_daily_ncname(yr) if not os.path.isfile(fn): print("Miss %s" % (fn, )) continue print(fn) nc = ncopen(fn, 'a', timeout=300) v1 = nc.createVariable('avg_dwpk', np.float, ('time', 'lat', 'lon'), fill_value=1.e20) v1.units = 'K' v1.long_name = '2m Average Dew Point Temperature' v1.standard_name = 'Dewpoint' v1.coordinates = "lon lat" v1.description = ("Dew Point average computed " "by averaging mixing ratios") v2 = nc.createVariable('wind_speed', np.float, ('time', 'lat', 'lon'), fill_value=1.e20) v2.units = 'm s-1' v2.long_name = 'Wind Speed' v2.standard_name = 'Wind Speed' v2.coordinates = "lon lat" v2.description = "Daily averaged wind speed magnitude" nc.sync() nc.close()
def estimate_hilo(df, ts): """Estimate the High and Low Temperature based on gridded data""" idx = iemre.daily_offset(ts) nc = ncopen(iemre.get_daily_ncname(ts.year), 'r', timeout=300) highgrid12 = temperature(nc.variables['high_tmpk_12z'][idx, :, :], 'K').value('F') lowgrid12 = temperature(nc.variables['low_tmpk_12z'][idx, :, :], 'K').value('F') highgrid00 = temperature(nc.variables['high_tmpk'][idx, :, :], 'K').value('F') lowgrid00 = temperature(nc.variables['low_tmpk'][idx, :, :], 'K').value('F') nc.close() for sid, row in df.iterrows(): if pd.isnull(row['high']): if row['temp24_hour'] in [0, 22, 23]: val = highgrid00[row['gridj'], row['gridi']] else: val = highgrid12[row['gridj'], row['gridi']] if sid == 'IA1402': print(row['temp24_hour']) if not np.ma.is_masked(val): df.at[sid, 'high'] = val if pd.isnull(row['low']): if row['temp24_hour'] in [0, 22, 23]: val = lowgrid00[row['gridj'], row['gridi']] else: val = lowgrid12[row['gridj'], row['gridi']] if not np.ma.is_masked(val): df.at[sid, 'low'] = val
def estimate_precip(df, ts): """Estimate precipitation based on IEMRE""" idx = iemre.daily_offset(ts) nc = ncopen(iemre.get_daily_ncname(ts.year), 'r', timeout=300) grid12 = distance(nc.variables['p01d_12z'][idx, :, :], 'MM').value("IN").filled(0) grid00 = distance(nc.variables['p01d'][idx, :, :], "MM").value("IN").filled(0) nc.close() for sid, row in df.iterrows(): if not pd.isnull(row['precip']): continue if row['precip24_hour'] in [0, 22, 23]: precip = grid00[row['gridj'], row['gridi']] else: precip = grid12[row['gridj'], row['gridi']] # denote trace if precip > 0 and precip < 0.01: df.at[sid, 'precip'] = TRACE_VALUE elif precip < 0: df.at[sid, 'precip'] = 0 elif np.isnan(precip) or np.ma.is_masked(precip): df.at[sid, 'precip'] = 0 else: df.at[sid, 'precip'] = "%.2f" % (precip,)
def do_coop(ts): """Use COOP solar radiation data""" pgconn = get_dbconn('coop', user='******') cursor = pgconn.cursor() cursor.execute( """SELECT ST_x(geom), ST_y(geom), coalesce(narr_srad, merra_srad) from alldata a JOIN stations t ON (a.station = t.id) WHERE day = %s and t.network ~* 'CLIMATE' and substr(id, 3, 1) != 'C' and substr(id, 3, 4) != '0000' """, (ts.strftime("%Y-%m-%d"), )) lons = [] lats = [] vals = [] for row in cursor: if row[2] is None or row[2] < 0: continue lons.append(row[0]) lats.append(row[1]) vals.append(row[2]) nn = NearestNDInterpolator((np.array(lons), np.array(lats)), np.array(vals)) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) nc = ncopen(iemre.get_daily_ncname(ts.year), 'a', timeout=300) offset = iemre.daily_offset(ts) # Data above is MJ / d / m-2, we want W / m-2 nc.variables['rsds'][offset, :, :] = nn(xi, yi) * 1000000. / 86400. nc.close()
def main(): """Go Main Go""" ets = datetime.datetime.now() - datetime.timedelta(days=1) sts = datetime.datetime(ets.year, 1, 1) # Get the normal accumm with ncopen(iemre.get_dailyc_ncname()) as cnc: lons = cnc.variables["lon"][:] lats = cnc.variables["lat"][:] index0 = iemre.daily_offset(sts) index1 = iemre.daily_offset(ets) clprecip = np.sum(cnc.variables["p01d"][index0:index1, :, :], 0) with ncopen(iemre.get_daily_ncname(sts.year)) as nc: obprecip = np.sum(nc.variables["p01d"][index0:index1, :, :], 0) lons, lats = np.meshgrid(lons, lats) # Plot departure from normal mp = MapPlot( sector="midwest", title=("Precipitation Departure %s - %s") % (sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")), subtitle="based on IEM Estimates", ) mp.pcolormesh(lons, lats, (obprecip - clprecip) / 25.4, np.arange(-10, 10, 1)) mp.postprocess( pqstr="plot c 000000000000 summary/year/stage4_diff.png bogus png") mp.close() # Plot normals mp = MapPlot( sector="midwest", title=("Normal Precipitation:: %s - %s") % (sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")), subtitle="based on IEM Estimates", ) mp.pcolormesh(lons, lats, (clprecip) / 25.4, np.arange(0, 30, 2)) mp.postprocess( pqstr="plot c 000000000000 summary/year/stage4_normals.png bogus png") mp.close() # Plot Obs mp = MapPlot( sector="midwest", title=("Estimated Precipitation:: %s - %s") % (sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")), subtitle="based on IEM Estimates", ) mp.pcolormesh(lons, lats, (obprecip) / 25.4, np.arange(0, 30, 2)) mp.postprocess( pqstr="plot c 000000000000 summary/year/stage4obs.png bogus png") mp.close()
def copy_iemre(nc, fromyear, ncdate0, ncdate1, islice, jslice): """Copy IEMRE data from a given year to **inclusive** dates.""" rencfn = iemre.get_daily_ncname(fromyear) if not os.path.isfile(rencfn): print("reanalysis fn %s missing" % (rencfn,)) return renc = ncopen(rencfn) tidx0 = (ncdate0 - datetime.date(fromyear, 1, 1)).days tidx1 = (ncdate1 - datetime.date(fromyear, 1, 1)).days tslice = slice(tidx0, tidx1 + 1) # time steps to fill tsteps = (tidx1 - tidx0) + 1 # figure out the slice if ncdate0.strftime("%m%d") == "0101": retslice = slice(0, tsteps) else: retslice = slice(0 - tsteps, None) # print("copy_iemre from %s filling %s steps nc: %s iemre: %s" % ( # fromyear, tsteps, tslice, retslice # )) highc = temperature( renc.variables["high_tmpk"][retslice, jslice, islice], "K" ).value("C") lowc = temperature( renc.variables["low_tmpk"][retslice, jslice, islice], "K" ).value("C") nc.variables["tmax"][tslice, :, :] = highc nc.variables["tmin"][tslice, :, :] = lowc nc.variables["gdd_f"][tslice, :, :] = gdd( temperature(highc, "C"), temperature(lowc, "C") ) nc.variables["prcp"][tslice, :, :] = renc.variables["p01d"][ retslice, jslice, islice ] for rt, nt in zip( list( range( retslice.start, 0 if retslice.stop is None else retslice.stop ) ), list(range(tslice.start, tslice.stop)), ): # IEMRE power_swdn is MJ, test to see if data exists srad = renc.variables["power_swdn"][rt, jslice, islice] if srad.mask.any(): # IEMRE rsds uses W m-2, we want MJ srad = ( renc.variables["rsds"][rt, jslice, islice] * 86400.0 / 1000000.0 ) nc.variables["srad"][nt, :, :] = srad renc.close()
def main(argv): """Go Main Go.""" year = int(argv[1]) ets = min([datetime.date(year, 12, 31), datetime.date.today()]) queue = [] for x0 in np.arange(iemre.WEST, iemre.EAST, 5.): for y0 in np.arange(iemre.SOUTH, iemre.NORTH, 5.): queue.append([x0, y0]) for x0, y0 in tqdm(queue, disable=not sys.stdout.isatty()): url = ( "https://power.larc.nasa.gov/cgi-bin/v1/DataAccess.py?" "request=execute&identifier=Regional&" "parameters=ALLSKY_SFC_SW_DWN&" "startDate=%s0101&endDate=%s&userCommunity=SSE&" "tempAverage=DAILY&bbox=%s,%s,%s,%s&user=anonymous&" "outputList=NETCDF" ) % (year, ets.strftime("%Y%m%d"), y0, x0, min([y0 + 5., iemre.NORTH]) - 0.1, min([x0 + 5., iemre.EAST]) - 0.1) req = requests.get(url, timeout=60) js = req.json() if 'outputs' not in js: print(url) print(js) continue fn = js['outputs']['netcdf'] req = requests.get(fn, timeout=60, stream=True) ncfn = '/tmp/power%s.nc' % (year, ) with open(ncfn, 'wb') as fh: for chunk in req.iter_content(chunk_size=1024): if chunk: fh.write(chunk) fh.close() nc = ncopen(ncfn) for day, _ in enumerate(nc.variables['time'][:]): date = datetime.date(year, 1, 1) + datetime.timedelta(days=day) # kwh to MJ/d 3600 * 1000 / 1e6 data = nc.variables['ALLSKY_SFC_SW_DWN'][day, :, :] * 3.6 # Sometimes there are missing values? if np.ma.is_masked(data): data[data.mask] = np.mean(data) i, j = iemre.find_ij(x0, y0) # resample data is 0.5, iemre is 0.125 data = np.repeat(np.repeat(data, 4, axis=0), 4, axis=1) shp = np.shape(data) # print("i: %s j: %s shp: %s" % (i, j, shp)) renc = ncopen(iemre.get_daily_ncname(year), 'a') renc.variables['power_swdn'][ iemre.daily_offset(date), slice(j, j+shp[0]), slice(i, i+shp[1]) ] = data renc.close() nc.close()
def load_iemre(): """Use IEM Reanalysis for non-precip data 24km product is smoothed down to the 0.01 degree grid """ printt("load_iemre() called") xaxis = np.arange(MYWEST, MYEAST, 0.01) yaxis = np.arange(MYSOUTH, MYNORTH, 0.01) xi, yi = np.meshgrid(xaxis, yaxis) fn = iemre.get_daily_ncname(VALID.year) if not os.path.isfile(fn): printt("Missing %s for load_solar, aborting" % (fn, )) sys.exit() with ncopen(fn) as nc: offset = iemre.daily_offset(VALID) lats = nc.variables["lat"][:] lons = nc.variables["lon"][:] lons, lats = np.meshgrid(lons, lats) # Storage is W m-2, we want langleys per day data = nc.variables["rsds"][offset, :, :] * 86400.0 / 1000000.0 * 23.9 # Default to a value of 300 when this data is missing, for some reason nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) SOLAR[:] = iemre_bounds_check("rsds", nn(xi, yi), 0, 1000) data = temperature(nc.variables["high_tmpk"][offset, :, :], "K").value("C") nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) HIGH_TEMP[:] = iemre_bounds_check("high_tmpk", nn(xi, yi), -60, 60) data = temperature(nc.variables["low_tmpk"][offset, :, :], "K").value("C") nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) LOW_TEMP[:] = iemre_bounds_check("low_tmpk", nn(xi, yi), -60, 60) data = temperature(nc.variables["avg_dwpk"][offset, :, :], "K").value("C") nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) DEWPOINT[:] = iemre_bounds_check("avg_dwpk", nn(xi, yi), -60, 60) data = nc.variables["wind_speed"][offset, :, :] nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) WIND[:] = iemre_bounds_check("wind_speed", nn(xi, yi), 0, 30) printt("load_iemre() finished")
def load_iemre(): """Use IEM Reanalysis for non-precip data 24km product is smoothed down to the 0.01 degree grid """ printt("load_iemre() called") xaxis = np.arange(MYWEST, MYEAST, 0.01) yaxis = np.arange(MYSOUTH, MYNORTH, 0.01) xi, yi = np.meshgrid(xaxis, yaxis) fn = iemre.get_daily_ncname(VALID.year) if not os.path.isfile(fn): printt("Missing %s for load_solar, aborting" % (fn,)) sys.exit() nc = netCDF4.Dataset(fn, 'r') offset = iemre.daily_offset(VALID) lats = nc.variables['lat'][:] lons = nc.variables['lon'][:] lons, lats = np.meshgrid(lons, lats) # Storage is W m-2, we want langleys per day data = nc.variables['rsds'][offset, :, :] * 86400. / 1000000. * 23.9 # Default to a value of 300 when this data is missing, for some reason nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) SOLAR[:] = iemre_bounds_check('rsds', nn(xi, yi), 0, 1000) data = temperature(nc.variables['high_tmpk'][offset, :, :], 'K').value('C') nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) HIGH_TEMP[:] = iemre_bounds_check('high_tmpk', nn(xi, yi), -60, 60) data = temperature(nc.variables['low_tmpk'][offset, :, :], 'K').value('C') nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) LOW_TEMP[:] = iemre_bounds_check('low_tmpk', nn(xi, yi), -60, 60) data = temperature(nc.variables['avg_dwpk'][offset, :, :], 'K').value('C') nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) DEWPOINT[:] = iemre_bounds_check('avg_dwpk', nn(xi, yi), -60, 60) data = nc.variables['wind_speed'][offset, :, :] nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) WIND[:] = iemre_bounds_check('wind_speed', nn(xi, yi), 0, 30) nc.close() printt("load_iemre() finished")
def load_iemre(): """Use IEM Reanalysis for non-precip data 24km product is smoothed down to the 0.01 degree grid """ printt("load_iemre() called") xaxis = np.arange(WEST, EAST, 0.01) yaxis = np.arange(SOUTH, NORTH, 0.01) xi, yi = np.meshgrid(xaxis, yaxis) fn = iemre.get_daily_ncname(VALID.year) if not os.path.isfile(fn): printt("Missing %s for load_solar, aborting" % (fn, )) sys.exit() nc = netCDF4.Dataset(fn, 'r') offset = iemre.daily_offset(VALID) lats = nc.variables['lat'][:] lons = nc.variables['lon'][:] lons, lats = np.meshgrid(lons, lats) # Storage is W m-2, we want langleys per day data = nc.variables['rsds'][offset, :, :] * 86400. / 1000000. * 23.9 # Default to a value of 300 when this data is missing, for some reason nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) SOLAR[:] = iemre_bounds_check('rsds', nn(xi, yi), 0, 1000) data = temperature(nc.variables['high_tmpk'][offset, :, :], 'K').value('C') nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) HIGH_TEMP[:] = iemre_bounds_check('high_tmpk', nn(xi, yi), -60, 60) data = temperature(nc.variables['low_tmpk'][offset, :, :], 'K').value('C') nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) LOW_TEMP[:] = iemre_bounds_check('low_tmpk', nn(xi, yi), -60, 60) data = temperature(nc.variables['avg_dwpk'][offset, :, :], 'K').value('C') nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) DEWPOINT[:] = iemre_bounds_check('avg_dwpk', nn(xi, yi), -60, 60) data = nc.variables['wind_speed'][offset, :, :] nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)), np.ravel(data)) WIND[:] = iemre_bounds_check('wind_speed', nn(xi, yi), 0, 30) nc.close() printt("load_iemre() finished")
def estimate_snow(df, ts): """Estimate the Snow based on COOP reports""" idx = iemre.daily_offset(ts) nc = ncopen(iemre.get_daily_ncname(ts.year), 'r', timeout=300) snowgrid12 = distance(nc.variables['snow_12z'][idx, :, :], 'MM').value('IN').filled(0) snowdgrid12 = distance(nc.variables['snowd_12z'][idx, :, :], 'MM').value('IN').filled(0) nc.close() for sid, row in df.iterrows(): if pd.isnull(row['snow']): df.at[sid, 'snow'] = snowgrid12[row['gridj'], row['gridi']] if pd.isnull(row['snowd']): df.at[sid, 'snowd'] = snowdgrid12[row['gridj'], row['gridi']]
def main(): """Go Main Go""" for year in range(1893, 2018): nc = ncopen(get_daily_ncname(year), 'a', timeout=600) gridsize = nc.dimensions['lat'].size * nc.dimensions['lon'].size for vname in ['p01d', 'high_tmpk', 'low_tmpk']: for i in range(nc.variables[vname].shape[0]): calday = nc.variables[vname][i, :, :] # Get a count of missing values missing = np.sum(calday.mask) / float(gridsize) if missing > 0.5: print("%s_12z->%s %s %.2f" % (vname, vname, i, missing)) nc.variables[vname][i, :, :] = ( nc.variables[vname + '_12z'][i, :, :] ) nc.close()
def get_data(ctx): """Do the processing work, please""" pgconn = get_dbconn("postgis") states = gpd.GeoDataFrame.from_postgis( """ SELECT the_geom, state_abbr from states where state_abbr = %s """, pgconn, params=(ctx["state"], ), index_col="state_abbr", geom_col="the_geom", ) if states.empty: raise NoDataFound("No data was found.") with ncopen(iemre.get_daily_ncname(ctx["year"])) as nc: precip = nc.variables["p01d"] czs = CachingZonalStats(iemre.AFFINE) hasdata = np.zeros( (nc.dimensions["lat"].size, nc.dimensions["lon"].size)) czs.gen_stats(hasdata, states["the_geom"]) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0.0 jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice, islice]) ctx["iowa"] = np.flipud(hasdata) ctx["iowapts"] = float(np.sum(np.where(hasdata > 0, 1, 0))) now = datetime.datetime(ctx["year"], 1, 1) now += datetime.timedelta(days=(ctx["period"] - 1)) ets = datetime.datetime(ctx["year"], 12, 31) today = datetime.datetime.now() if ets > today: ets = today - datetime.timedelta(days=1) ctx["days"] = [] rows = [] trailthres = ((ctx["trailthres"] * units("inch")).to( units("mm")).magnitude) daythres = (ctx["daythres"] * units("inch")).to(units("mm")).magnitude while now < ets: rows.append(do_date(ctx, now, precip, daythres, trailthres)) now += datetime.timedelta(days=1) return pd.DataFrame(rows)
def workflow(fn): """Do the copy work""" oldnc = netCDF4.Dataset(fn) newnc = netCDF4.Dataset(iemre.get_daily_ncname(fn[:4]), 'a') newnc.set_auto_scale(True) i, j = iemre.find_ij(oldnc.variables['lon'][0], oldnc.variables['lat'][0]) jslice = slice(j, j + oldnc.dimensions['lat'].size * 2) islice = slice(i, i + oldnc.dimensions['lon'].size * 2) # print("i:%s j:%s %s %s" % (i, j, islice, jslice)) for vname in tqdm(oldnc.variables): if vname in ['time', 'lat', 'lon']: continue for tstep in oldnc.variables['time'][:]: oldgrid = np.repeat(oldnc.variables[vname][tstep, :, :], 2, 0).repeat(2, 1) newnc.variables[vname][tstep, jslice, islice] = oldgrid newnc.close()
def main(argv): """Go Main Go.""" log = logger() if len(argv) == 6: valid = utc(int(argv[1]), int(argv[2]), int(argv[3]), int(argv[4])) ncfn = iemre.get_hourly_ncname(valid.year) idx = iemre.hourly_offset(valid) else: valid = datetime.date(int(argv[1]), int(argv[2]), int(argv[3])) ncfn = iemre.get_daily_ncname(valid.year) idx = iemre.daily_offset(valid) ds = iemre.get_grids(valid) with ncopen(ncfn, 'a', timeout=600) as nc: for vname in ds: if vname not in nc.variables: continue log.debug("copying database var %s to netcdf", vname) nc.variables[vname][idx, :, :] = ds[vname].values
def tile_extraction(nc, valid, west, south, isnewfile): """Do our tile extraction""" # update model metadata nc.valid = "CFS model: %s" % (valid.strftime("%Y-%m-%dT%H:%M:%SZ"), ) i, j = iemre.find_ij(west, south) islice = slice(i, i + 16) jslice = slice(j, j + 16) for year in range(1980 if isnewfile else valid.year, valid.year + 1): tidx0 = (datetime.date(year, 1, 1) - datetime.date(1980, 1, 1)).days tidx1 = (datetime.date(year + 1, 1, 1) - datetime.date(1980, 1, 1)).days tslice = slice(tidx0, tidx1) ncfn = iemre.get_daily_ncname(year) if not os.path.isfile(ncfn): continue renc = ncopen(ncfn) # print("tslice: %s jslice: %s islice: %s" % (tslice, jslice, islice)) nc.variables['tmax'][tslice, :, :] = temperature( renc.variables['high_tmpk'][:, jslice, islice], 'K').value('C') nc.variables['tmin'][tslice, :, :] = temperature( renc.variables['low_tmpk'][:, jslice, islice], 'K').value('C') nc.variables['prcp'][tslice, :, :] = (renc.variables['p01d'][:, jslice, islice]) # MJ/d back to average W/m2 nc.variables['srad'][tslice, :, :] = (renc.variables['rsds'][:, jslice, islice]) renc.close() if year != valid.year: continue # replace CFS! renc = ncopen(valid.strftime("/mesonet/data/iemre/cfs_%Y%m%d.nc")) tidx = iemre.daily_offset(valid + datetime.timedelta(days=1)) tslice = slice(tidx0 + tidx, tidx1) nc.variables['srad'][tslice, :, :] = ( renc.variables['srad'][tidx:, jslice, islice] * 1000000. / 86400.) nc.variables['tmax'][tslice, :, :] = temperature( renc.variables['high_tmpk'][tidx:, jslice, islice], 'K').value('C') nc.variables['tmin'][tslice, :, :] = temperature( renc.variables['low_tmpk'][tidx:, jslice, islice], 'K').value('C') nc.variables['prcp'][tslice, :, :] = (renc.variables['p01d'][tidx:, jslice, islice]) renc.close()
def main(argv): """Do work please""" day = datetime.date(int(argv[1]), int(argv[2]), int(argv[3])) pgconn = get_dbconn('coop') df = read_sql(""" SELECT a.precip, st_x(t.geom) as lon, st_y(t.geom) as lat from alldata a JOIN stations t ON (a.station = t.id) WHERE a.day = %s and t.network ~* 'CLIMATE' and substr(a.station,3,4) != '0000' and substr(station,3,1) != 'C' and precip >= 0 and precip < 50 """, pgconn, params=(day, )) nc = ncopen(get_daily_ncname(day.year), 'a') res = generic_gridder(day, nc, df, 'precip') if res is not None: offset = daily_offset(day) nc.variables['p01d_12z'][offset] = res.to(mpunits('mm')).magnitude nc.close()
def compute_hasdata(year): """Compute the has_data grid""" nc = ncopen(iemre.get_daily_ncname(year), 'a', timeout=300) czs = CachingZonalStats(iemre.AFFINE) pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr not in ('AK', 'HI') """, pgconn, index_col='state_abbr', geom_col='the_geom') data = np.flipud(nc.variables['hasdata'][:, :]) czs.gen_stats(data, states['the_geom']) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) data[jslice, islice] = np.where(grid > 0, 1, data[jslice, islice]) nc.variables['hasdata'][:, :] = np.flipud(data) nc.close()
def get_data(ctx): """Do the processing work, please""" pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr = %s """, pgconn, params=(ctx['state'], ), index_col='state_abbr', geom_col='the_geom') with ncopen(iemre.get_daily_ncname(ctx['year'])) as nc: precip = nc.variables['p01d'] czs = CachingZonalStats(iemre.AFFINE) hasdata = np.zeros( (nc.dimensions['lat'].size, nc.dimensions['lon'].size)) czs.gen_stats(hasdata, states['the_geom']) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice, islice]) ctx['iowa'] = np.flipud(hasdata) ctx['iowapts'] = float(np.sum(np.where(hasdata > 0, 1, 0))) now = datetime.datetime(ctx['year'], 1, 1) now += datetime.timedelta(days=(ctx['period'] - 1)) ets = datetime.datetime(ctx['year'], 12, 31) today = datetime.datetime.now() if ets > today: ets = today - datetime.timedelta(days=1) ctx['days'] = [] rows = [] trailthres = (ctx['trailthres'] * units('inch')).to( units('mm')).magnitude daythres = (ctx['daythres'] * units('inch')).to(units('mm')).magnitude while now < ets: rows.append(do_date(ctx, now, precip, daythres, trailthres)) now += datetime.timedelta(days=1) return pd.DataFrame(rows)
def copy_iemre(nc, ncdate0, ncdate1, islice, jslice): """Copy IEMRE data from a given year to **inclusive** dates.""" rencfn = iemre.get_daily_ncname(ncdate0.year) if not os.path.isfile(rencfn): LOG.info("reanalysis fn %s missing", rencfn) return with ncopen(rencfn) as renc: # Compute offsets for yieldfx file tidx0 = (ncdate0 - datetime.date(1980, 1, 1)).days tidx1 = (ncdate1 - datetime.date(1980, 1, 1)).days yfx_slice = slice(tidx0, tidx1 + 1) # Compute offsets for the reanalysis file tidx0 = (ncdate0 - datetime.date(ncdate0.year, 1, 1)).days tidx1 = (ncdate1 - datetime.date(ncdate0.year, 1, 1)).days re_slice = slice(tidx0, tidx1 + 1) # LOG.debug("filling nc: %s iemre: %s", yfx_slice, re_slice) highc = temperature( renc.variables["high_tmpk"][re_slice, jslice, islice], "K").value("C") lowc = temperature( renc.variables["low_tmpk"][re_slice, jslice, islice], "K").value("C") nc.variables["tmax"][yfx_slice, :, :] = highc nc.variables["tmin"][yfx_slice, :, :] = lowc nc.variables["gdd_f"][yfx_slice, :, :] = gdd(temperature(highc, "C"), temperature(lowc, "C")) nc.variables["prcp"][yfx_slice, :, :] = renc.variables["p01d"][ re_slice, jslice, islice] # Special care needed for solar radiation filling for rt, nt in zip( list(range(re_slice.start, re_slice.stop)), list(range(yfx_slice.start, yfx_slice.stop)), ): # IEMRE power_swdn is MJ, test to see if data exists srad = renc.variables["power_swdn"][rt, jslice, islice] if srad.mask.any(): # IEMRE rsds uses W m-2, we want MJ srad = (renc.variables["rsds"][rt, jslice, islice] * 86400.0 / 1000000.0) nc.variables["srad"][nt, :, :] = srad
def try_merra(ts): """Attempt to use MERRA data.""" # Our files are UTC date based :/ ncfn1 = ts.strftime("/mesonet/merra2/%Y/%Y%m%d.nc") ncfn2 = ( ts + datetime.timedelta(days=1) ).strftime("/mesonet/merra2/%Y/%Y%m%d.nc") if not os.path.isfile(ncfn1) or not os.path.isfile(ncfn2): return False nc = ncopen(ncfn1) # Total up from 6z to end of file for today total = np.sum(nc.variables['SWGDN'][5:, :, :], axis=0) nc.close() nc = ncopen(ncfn2) lat1d = nc.variables['lat'][:] lon1d = nc.variables['lon'][:] # Total up to 6z total += np.sum(nc.variables['SWGDN'][:6, :, :], axis=0) nc.close() # We wanna store as W m-2, so we just average out the data by hour total = total / 24.0 lons, lats = np.meshgrid(lon1d, lat1d) nn = NearestNDInterpolator( (lons.flatten(), lats.flatten()), total.flatten() ) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) nc = ncopen(iemre.get_daily_ncname(ts.year), 'a', timeout=300) offset = iemre.daily_offset(ts) # Data above is W m-2 nc.variables['rsds'][offset, :, :] = nn(xi, yi) nc.close() return True
def plotter(fdict): """ Go """ ctx = get_autoplot_context(fdict, get_description()) ptype = ctx["ptype"] date = ctx["date"] varname = ctx["var"] csector = ctx["csector"] title = date.strftime("%-d %B %Y") mp = MapPlot( sector=("state" if len(csector) == 2 else csector), state=ctx["csector"], axisbg="white", nocaption=True, title="IEM Reanalysis of %s for %s" % (PDICT.get(varname), title), subtitle="Data derived from various NOAA datasets", ) (west, east, south, north) = mp.ax.get_extent(ccrs.PlateCarree()) i0, j0 = iemre.find_ij(west, south) i1, j1 = iemre.find_ij(east, north) jslice = slice(j0, j1) islice = slice(i0, i1) idx0 = iemre.daily_offset(date) ncfn = iemre.get_daily_ncname(date.year) if not os.path.isfile(ncfn): raise NoDataFound("No Data Found.") with ncopen(ncfn) as nc: lats = nc.variables["lat"][jslice] lons = nc.variables["lon"][islice] cmap = ctx["cmap"] if varname in ["rsds", "power_swdn"]: # Value is in W m**-2, we want MJ multi = (86400.0 / 1000000.0) if varname == "rsds" else 1 data = nc.variables[varname][idx0, jslice, islice] * multi plot_units = "MJ d-1" clevs = np.arange(0, 37, 3.0) clevs[0] = 0.01 clevstride = 1 elif varname in ["wind_speed"]: data = (masked_array( nc.variables[varname][idx0, jslice, islice], units("meter / second"), ).to(units("mile / hour")).m) plot_units = "mph" clevs = np.arange(0, 41, 2) clevs[0] = 0.01 clevstride = 2 elif varname in ["p01d", "p01d_12z", "snow_12z", "snowd_12z"]: # Value is in W m**-2, we want MJ data = (masked_array(nc.variables[varname][idx0, jslice, islice], units("mm")).to(units("inch")).m) plot_units = "inch" clevs = np.arange(0, 0.25, 0.05) clevs = np.append(clevs, np.arange(0.25, 3.0, 0.25)) clevs = np.append(clevs, np.arange(3.0, 10.0, 1)) clevs[0] = 0.01 clevstride = 1 cmap = stretch_cmap(ctx["cmap"], clevs) elif varname in [ "high_tmpk", "low_tmpk", "high_tmpk_12z", "low_tmpk_12z", "avg_dwpk", ]: # Value is in W m**-2, we want MJ data = (masked_array(nc.variables[varname][idx0, jslice, islice], units("degK")).to(units("degF")).m) plot_units = "F" clevs = np.arange(-30, 120, 5) clevstride = 2 elif varname in ["range_tmpk", "range_tmpk_12z"]: vname1 = "high_tmpk%s" % ("_12z" if varname == "range_tmpk_12z" else "", ) vname2 = "low_tmpk%s" % ("_12z" if varname == "range_tmpk_12z" else "", ) d1 = nc.variables[vname1][idx0, jslice, islice] d2 = nc.variables[vname2][idx0, jslice, islice] data = (masked_array(d1, units("degK")).to(units("degF")).m - masked_array(d2, units("degK")).to(units("degF")).m) plot_units = "F" clevs = np.arange(0, 61, 5) clevstride = 2 if np.ma.is_masked(np.max(data)): raise NoDataFound("Data Unavailable") x, y = np.meshgrid(lons, lats) if ptype == "c": # in the case of contour, use the centroids on the grids mp.contourf( x + 0.125, y + 0.125, data, clevs, clevstride=clevstride, units=plot_units, ilabel=True, labelfmt="%.0f", cmap=cmap, ) else: x, y = np.meshgrid(lons, lats) mp.pcolormesh( x, y, data, clevs, clevstride=clevstride, cmap=cmap, units=plot_units, ) return mp.fig
def plotter(fdict): """ Go """ ctx = util.get_autoplot_context(fdict, get_description()) ptype = ctx["ptype"] sdate = ctx["sdate"] edate = ctx["edate"] src = ctx["src"] opt = ctx["opt"] usdm = ctx["usdm"] if sdate.year != edate.year: raise NoDataFound("Sorry, do not support multi-year plots yet!") days = (edate - sdate).days sector = ctx["sector"] x0 = 0 x1 = -1 y0 = 0 y1 = -1 state = None if len(sector) == 2: state = sector sector = "state" title = compute_title(src, sdate, edate) if src == "mrms": ncfn = iemre.get_daily_mrms_ncname(sdate.year) clncfn = iemre.get_dailyc_mrms_ncname() ncvar = "p01d" source = "MRMS Q3" subtitle = "NOAA MRMS Project, GaugeCorr and RadarOnly" elif src == "iemre": ncfn = iemre.get_daily_ncname(sdate.year) clncfn = iemre.get_dailyc_ncname() ncvar = "p01d_12z" source = "IEM Reanalysis" subtitle = "IEM Reanalysis is derived from various NOAA datasets" else: ncfn = "/mesonet/data/prism/%s_daily.nc" % (sdate.year, ) clncfn = "/mesonet/data/prism/prism_dailyc.nc" ncvar = "ppt" source = "OSU PRISM" subtitle = ("PRISM Climate Group, Oregon State Univ., " "http://prism.oregonstate.edu, created 4 Feb 2004.") mp = MapPlot( sector=sector, state=state, axisbg="white", nocaption=True, title="%s:: %s Precip %s" % (source, title, PDICT3[opt]), subtitle="Data from %s" % (subtitle, ), titlefontsize=14, ) idx0 = iemre.daily_offset(sdate) idx1 = iemre.daily_offset(edate) + 1 if not os.path.isfile(ncfn): raise NoDataFound("No data for that year, sorry.") with util.ncopen(ncfn) as nc: if state is not None: x0, y0, x1, y1 = util.grid_bounds( nc.variables["lon"][:], nc.variables["lat"][:], state_bounds[state], ) elif sector in SECTORS: bnds = SECTORS[sector] x0, y0, x1, y1 = util.grid_bounds( nc.variables["lon"][:], nc.variables["lat"][:], [bnds[0], bnds[2], bnds[1], bnds[3]], ) lats = nc.variables["lat"][y0:y1] lons = nc.variables["lon"][x0:x1] if sdate == edate: p01d = mm2inch(nc.variables[ncvar][idx0, y0:y1, x0:x1]) elif (idx1 - idx0) < 32: p01d = mm2inch( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0)) else: # Too much data can overwhelm this app, need to chunk it for i in range(idx0, idx1, 10): i2 = min([i + 10, idx1]) if idx0 == i: p01d = mm2inch( np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0)) else: p01d += mm2inch( np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0)) if np.ma.is_masked(np.max(p01d)): raise NoDataFound("Data Unavailable") plot_units = "inches" cmap = get_cmap(ctx["cmap"]) cmap.set_bad("white") if opt == "dep": # Do departure work now with util.ncopen(clncfn) as nc: climo = mm2inch( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0)) p01d = p01d - climo [maxv] = np.percentile(np.abs(p01d), [99]) clevs = np.around(np.linspace(0 - maxv, maxv, 11), decimals=2) elif opt == "per": with util.ncopen(clncfn) as nc: climo = mm2inch( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0)) p01d = p01d / climo * 100.0 cmap.set_under("white") cmap.set_over("black") clevs = [1, 10, 25, 50, 75, 100, 125, 150, 200, 300, 500] plot_units = "percent" else: p01d = np.where(p01d < 0.001, np.nan, p01d) cmap.set_under("white") clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10] if days > 6: clevs = [0.01, 0.3, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 10, 15, 20] if days > 29: clevs = [0.01, 0.5, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35] if days > 90: clevs = [0.01, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35, 40] x2d, y2d = np.meshgrid(lons, lats) if ptype == "c": mp.contourf(x2d, y2d, p01d, clevs, cmap=cmap, units=plot_units, iline=False) else: res = mp.pcolormesh(x2d, y2d, p01d, clevs, cmap=cmap, units=plot_units) res.set_rasterized(True) if sector != "midwest": mp.drawcounties() mp.drawcities() if usdm == "yes": mp.draw_usdm(edate, filled=False, hatched=True) return mp.fig
def application(environ, start_response): """Go main go""" os.chdir("/tmp") form = parse_formvars(environ) ts0 = datetime.datetime.strptime(form.get("date0"), "%Y-%m-%d") ts1 = datetime.datetime.strptime(form.get("date1"), "%Y-%m-%d") base = int(form.get("base", 50)) ceil = int(form.get("ceil", 86)) # Make sure we aren't in the future tsend = datetime.date.today() if ts1.date() >= tsend: ts1 = tsend - datetime.timedelta(days=1) ts1 = datetime.datetime(ts1.year, ts1.month, ts1.day) fmt = form.get("format") offset0 = iemre.daily_offset(ts0) offset1 = iemre.daily_offset(ts1) with ncopen(iemre.get_daily_ncname(ts0.year)) as nc: # 2-D precipitation, inches precip = np.sum(nc.variables["p01d"][offset0:offset1, :, :] / 25.4, axis=0) # GDD H = datatypes.temperature(nc.variables["high_tmpk"][offset0:offset1], "K").value("F") H = np.where(H < base, base, H) H = np.where(H > ceil, ceil, H) L = datatypes.temperature(nc.variables["low_tmpk"][offset0:offset1], "K").value("F") L = np.where(L < base, base, L) gdd = np.sum((H + L) / 2.0 - base, axis=0) if fmt == "json": # For example: 19013 ugc = "IAC" + form.get("county")[2:] # Go figure out where this is! postgis = get_dbconn("postgis") pcursor = postgis.cursor() pcursor.execute( """ SELECT ST_x(ST_Centroid(geom)), ST_y(ST_Centroid(geom)) from ugcs WHERE ugc = %s and end_ts is null """, (ugc, ), ) row = pcursor.fetchone() lat = row[1] lon = row[0] (i, j) = iemre.find_ij(lon, lat) myGDD = gdd[j, i] myPrecip = precip[j, i] res = {"data": []} res["data"].append({ "gdd": "%.0f" % (myGDD, ), "precip": "%.1f" % (myPrecip, ), "latitude": "%.4f" % (lat, ), "longitude": "%.4f" % (lon, ), }) headers = [("Content-type", "application/json")] start_response("200 OK", headers) return [json.dumps(res).encode("ascii")] # Time to create the shapefiles basefn = "iemre_%s_%s" % (ts0.strftime("%Y%m%d"), ts1.strftime("%Y%m")) w = shapefile.Writer(basefn) w.field("GDD", "F", 10, 2) w.field("PREC_IN", "F", 10, 2) for x in iemre.XAXIS: for y in iemre.YAXIS: w.poly([[ (x, y), (x, y + iemre.DY), (x + iemre.DX, y + iemre.DY), (x + iemre.DX, y), (x, y), ]]) for i in range(len(iemre.XAXIS)): for j in range(len(iemre.YAXIS)): w.record(gdd[j, i], precip[j, i]) w.close() # Create zip file, send it back to the clients shutil.copyfile("/opt/iem/data/gis/meta/4326.prj", "%s.prj" % (basefn, )) z = zipfile.ZipFile("%s.zip" % (basefn, ), "w", zipfile.ZIP_DEFLATED) for suffix in ["shp", "shx", "dbf", "prj"]: z.write("%s.%s" % (basefn, suffix)) z.close() headers = [ ("Content-type", "application/octet-stream"), ("Content-Disposition", "attachment; filename=%s.zip" % (basefn, )), ] start_response("200 OK", headers) content = open(basefn + ".zip", "rb").read() for suffix in ["zip", "shp", "shx", "dbf", "prj"]: os.unlink("%s.%s" % (basefn, suffix)) return [content]
def do_day(valid): """ Process a day please """ idx = iemre.daily_offset(valid) nc = ncopen(iemre.get_daily_ncname(valid.year), 'r', timeout=300) high = temperature(nc.variables['high_tmpk_12z'][idx, :, :], 'K').value('F') low = temperature(nc.variables['low_tmpk_12z'][idx, :, :], 'K').value('F') precip = distance(nc.variables['p01d_12z'][idx, :, :], 'MM').value("IN") snow = distance(nc.variables['snow_12z'][idx, :, :], 'MM').value("IN") snowd = distance(nc.variables['snowd_12z'][idx, :, :], 'MM').value("IN") nc.close() # build out the state mappers pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr not in ('AK', 'HI', 'DC') """, pgconn, index_col='state_abbr', geom_col='the_geom') czs = CachingZonalStats(iemre.AFFINE) sthigh = czs.gen_stats(np.flipud(high), states['the_geom']) stlow = czs.gen_stats(np.flipud(low), states['the_geom']) stprecip = czs.gen_stats(np.flipud(precip), states['the_geom']) stsnow = czs.gen_stats(np.flipud(snow), states['the_geom']) stsnowd = czs.gen_stats(np.flipud(snowd), states['the_geom']) statedata = {} for i, state in enumerate(states.index.values): statedata[state] = dict(high=sthigh[i], low=stlow[i], precip=stprecip[i], snow=stsnow[i], snowd=stsnowd[i]) update_database(state + "0000", valid, statedata[state]) # build out climate division mappers climdiv = gpd.GeoDataFrame.from_postgis(""" SELECT geom, iemid from climdiv where st_abbrv not in ('AK', 'HI', 'DC') """, pgconn, index_col='iemid', geom_col='geom') czs = CachingZonalStats(iemre.AFFINE) sthigh = czs.gen_stats(np.flipud(high), climdiv['geom']) stlow = czs.gen_stats(np.flipud(low), climdiv['geom']) stprecip = czs.gen_stats(np.flipud(precip), climdiv['geom']) stsnow = czs.gen_stats(np.flipud(snow), climdiv['geom']) stsnowd = czs.gen_stats(np.flipud(snowd), climdiv['geom']) for i, iemid in enumerate(climdiv.index.values): row = dict(high=sthigh[i], low=stlow[i], precip=stprecip[i], snow=stsnow[i], snowd=stsnowd[i]) # we must have temperature data if row['high'] is np.ma.masked or row['low'] is np.ma.masked: print( ("compute_0000 %s has missing temperature data, using state") % (iemid, )) row = statedata[iemid[:2]] update_database(iemid, valid, row)
def init_year(ts): """ Create a new NetCDF file for a year of our specification! """ fn = iemre.get_daily_ncname(ts.year) nc = ncopen(fn, 'w') nc.title = "IEM Daily Reanalysis %s" % (ts.year, ) nc.platform = "Grided Observations" nc.description = "IEM daily analysis on a 0.125 degree grid" nc.institution = "Iowa State University, Ames, IA, USA" nc.source = "Iowa Environmental Mesonet" nc.project_id = "IEM" nc.realization = 1 nc.Conventions = 'CF-1.0' nc.contact = "Daryl Herzmann, [email protected], 515-294-5978" nc.history = ("%s Generated") % ( datetime.datetime.now().strftime("%d %B %Y"), ) nc.comment = "No Comment at this time" # Setup Dimensions nc.createDimension('lat', iemre.NY) nc.createDimension('lon', iemre.NX) days = ((ts.replace(year=ts.year + 1)) - ts).days nc.createDimension('time', int(days)) # Setup Coordinate Variables lat = nc.createVariable('lat', np.float, ('lat', )) lat.units = "degrees_north" lat.long_name = "Latitude" lat.standard_name = "latitude" lat.axis = "Y" lat[:] = iemre.YAXIS lon = nc.createVariable('lon', np.float, ('lon', )) lon.units = "degrees_east" lon.long_name = "Longitude" lon.standard_name = "longitude" lon.axis = "X" lon[:] = iemre.XAXIS tm = nc.createVariable('time', np.float, ('time', )) tm.units = "Days since %s-01-01 00:00:0.0" % (ts.year, ) tm.long_name = "Time" tm.standard_name = "time" tm.axis = "T" tm.calendar = "gregorian" tm[:] = np.arange(0, int(days)) # Tracked variables hasdata = nc.createVariable('hasdata', np.int8, ('lat', 'lon')) hasdata.units = '1' hasdata.long_name = 'Analysis Available for Grid Cell' hasdata.coordinates = "lon lat" hasdata[:] = 0 high = nc.createVariable('high_tmpk', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) high.units = "K" high.scale_factor = 0.01 high.long_name = "2m Air Temperature Daily High" high.standard_name = "2m Air Temperature" high.coordinates = "lon lat" low = nc.createVariable('low_tmpk', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) low.units = "K" low.scale_factor = 0.01 low.long_name = "2m Air Temperature Daily Low" low.standard_name = "2m Air Temperature" low.coordinates = "lon lat" high12 = nc.createVariable('high_tmpk_12z', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) high12.units = "K" high12.scale_factor = 0.01 high12.long_name = "2m Air Temperature 24 Hour Max at 12 UTC" high12.standard_name = "2m Air Temperature" high12.coordinates = "lon lat" low12 = nc.createVariable('low_tmpk_12z', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) low12.units = "K" low12.scale_factor = 0.01 low12.long_name = "2m Air Temperature 12 Hour Min at 12 UTC" low12.standard_name = "2m Air Temperature" low12.coordinates = "lon lat" p01d = nc.createVariable('p01d', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) p01d.units = 'mm' p01d.scale_factor = 0.01 p01d.long_name = 'Precipitation' p01d.standard_name = 'Precipitation' p01d.coordinates = "lon lat" p01d.description = "Precipitation accumulation for the day" p01d12 = nc.createVariable('p01d_12z', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) p01d12.units = 'mm' p01d12.scale_factor = 0.01 p01d12.long_name = 'Precipitation' p01d12.standard_name = 'Precipitation' p01d12.coordinates = "lon lat" p01d12.description = "24 Hour Precipitation Ending 12 UTC" # 0 -> 65535 so 0 to 6553.5 rsds = nc.createVariable('rsds', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) rsds.units = "W m-2" rsds.scale_factor = 0.1 rsds.long_name = 'surface_downwelling_shortwave_flux_in_air' rsds.standard_name = 'surface_downwelling_shortwave_flux_in_air' rsds.coordinates = "lon lat" rsds.description = "Global Shortwave Irradiance" snow = nc.createVariable('snow_12z', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) snow.units = 'mm' snow.scale_factor = 0.01 snow.long_name = 'Snowfall' snow.standard_name = 'Snowfall' snow.coordinates = "lon lat" snow.description = "Snowfall accumulation for the day" # 0 to 6553.5 snowd = nc.createVariable('snowd_12z', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) snowd.units = 'mm' snowd.scale_factor = 0.1 snowd.long_name = 'Snow Depth' snowd.standard_name = 'Snow Depth' snowd.coordinates = "lon lat" snowd.description = "Snow depth at time of observation" v1 = nc.createVariable('avg_dwpk', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) v1.units = 'K' v1.scale_factor = 0.01 v1.long_name = '2m Average Dew Point Temperature' v1.standard_name = 'Dewpoint' v1.coordinates = "lon lat" v1.description = "Dew Point average computed by averaging mixing ratios" v2 = nc.createVariable('wind_speed', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) v2.units = 'm s-1' v2.scale_factor = 0.001 v2.long_name = 'Wind Speed' v2.standard_name = 'Wind Speed' v2.coordinates = "lon lat" v2.description = "Daily averaged wind speed magnitude" nc.close()
def main(): """Go Main Go""" form = cgi.FieldStorage() ts1 = datetime.datetime.strptime(form.getfirst("date1"), "%Y-%m-%d") ts2 = datetime.datetime.strptime(form.getfirst("date2"), "%Y-%m-%d") if ts1 > ts2: send_error("date1 larger than date2") if ts1.year != ts2.year: send_error("multi-year query not supported yet...") # Make sure we aren't in the future tsend = datetime.date.today() if ts2.date() > tsend: ts2 = datetime.datetime.now() - datetime.timedelta(days=1) lat = float(form.getfirst("lat")) lon = float(form.getfirst("lon")) if lon < iemre.WEST or lon > iemre.EAST: send_error("lon value outside of bounds: %s to %s" % (iemre.WEST, iemre.EAST)) if lat < iemre.SOUTH or lat > iemre.NORTH: send_error("lat value outside of bounds: %s to %s" % (iemre.SOUTH, iemre.NORTH)) # fmt = form["format"][0] i, j = iemre.find_ij(lon, lat) offset1 = iemre.daily_offset(ts1) offset2 = iemre.daily_offset(ts2) + 1 # Get our netCDF vars with ncopen(iemre.get_daily_ncname(ts1.year)) as nc: hightemp = datatypes.temperature( nc.variables['high_tmpk'][offset1:offset2, j, i], 'K').value("F") high12temp = datatypes.temperature( nc.variables['high_tmpk_12z'][offset1:offset2, j, i], 'K').value("F") lowtemp = datatypes.temperature( nc.variables['low_tmpk'][offset1:offset2, j, i], 'K').value("F") low12temp = datatypes.temperature( nc.variables['low_tmpk_12z'][offset1:offset2, j, i], 'K').value("F") precip = nc.variables['p01d'][offset1:offset2, j, i] / 25.4 precip12 = nc.variables['p01d_12z'][offset1:offset2, j, i] / 25.4 # Get our climatology vars c2000 = ts1.replace(year=2000) coffset1 = iemre.daily_offset(c2000) c2000 = ts2.replace(year=2000) coffset2 = iemre.daily_offset(c2000) + 1 cnc = ncopen(iemre.get_dailyc_ncname()) chigh = datatypes.temperature( cnc.variables['high_tmpk'][coffset1:coffset2, j, i], 'K').value("F") clow = datatypes.temperature( cnc.variables['low_tmpk'][coffset1:coffset2, j, i], 'K').value("F") cprecip = cnc.variables['p01d'][coffset1:coffset2, j, i] / 25.4 cnc.close() if ts1.year > 1980: nc = ncopen("/mesonet/data/prism/%s_daily.nc" % (ts1.year, )) i2, j2 = prismutil.find_ij(lon, lat) prism_precip = nc.variables['ppt'][offset1:offset2, j2, i2] / 25.4 nc.close() else: prism_precip = [None] * (offset2 - offset1) if ts1.year > 2010: nc = ncopen(iemre.get_daily_mrms_ncname(ts1.year)) j2 = int((lat - iemre.SOUTH) * 100.0) i2 = int((lon - iemre.WEST) * 100.0) mrms_precip = nc.variables['p01d'][offset1:offset2, j2, i2] / 25.4 nc.close() else: mrms_precip = [None] * (offset2 - offset1) res = { 'data': [], } for i in range(0, offset2 - offset1): now = ts1 + datetime.timedelta(days=i) res['data'].append({ 'date': now.strftime("%Y-%m-%d"), 'mrms_precip_in': clean(mrms_precip[i]), 'prism_precip_in': clean(prism_precip[i]), 'daily_high_f': clean(hightemp[i]), '12z_high_f': clean(high12temp[i]), 'climate_daily_high_f': clean(chigh[i]), 'daily_low_f': clean(lowtemp[i]), '12z_low_f': clean(low12temp[i]), 'climate_daily_low_f': clean(clow[i]), 'daily_precip_in': clean(precip[i]), '12z_precip_in': clean(precip12[i]), 'climate_daily_precip_in': clean(cprecip[i]) }) ssw('Content-type: application/json\n\n') ssw(json.dumps(res))
def plotter(fdict): """ Go """ ctx = get_autoplot_context(fdict, get_description()) year = ctx['year'] thres = ctx['thres'] metric = distance(thres, 'IN').value('MM') state = ctx['state'][:2] sts = datetime.datetime(year, 10, 1) ets = datetime.datetime(year + 1, 5, 1) rows = [] pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr = %s """, pgconn, params=(state, ), index_col='state_abbr', geom_col='the_geom') sidx = iemre.daily_offset(sts) ncfn = iemre.get_daily_ncname(sts.year) if not os.path.isfile(ncfn): raise ValueError("Data for year %s not found" % (sts.year, )) nc = ncopen(ncfn) czs = CachingZonalStats(iemre.AFFINE) hasdata = np.zeros((nc.dimensions['lat'].size, nc.dimensions['lon'].size)) czs.gen_stats(hasdata, states['the_geom']) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice, islice]) st = np.flipud(hasdata) stpts = np.sum(np.where(hasdata > 0, 1, 0)) snowd = nc.variables['snowd_12z'][sidx:, :, :] nc.close() for i in range(snowd.shape[0]): rows.append({ 'valid': sts + datetime.timedelta(days=i), 'coverage': f(st, snowd[i], metric, stpts), }) eidx = iemre.daily_offset(ets) nc = ncopen(iemre.get_daily_ncname(ets.year)) snowd = nc.variables['snowd_12z'][:eidx, :, :] nc.close() for i in range(snowd.shape[0]): rows.append({ 'valid': datetime.date(ets.year, 1, 1) + datetime.timedelta(days=i), 'coverage': f(st, snowd[i], metric, stpts), }) df = pd.DataFrame(rows) df = df[np.isfinite(df['coverage'])] (fig, ax) = plt.subplots(1, 1, sharex=True, figsize=(8, 6)) ax.bar(df['valid'].values, df['coverage'].values, fc='tan', ec='tan', align='center') ax.set_title(("IEM Estimated Areal Snow Coverage Percent of %s\n" " percentage of state reporting at least %.2fin snow" " cover" ) % (reference.state_names[state], thres)) ax.set_ylabel("Areal Coverage [%]") ax.xaxis.set_major_locator(mdates.DayLocator([1, 15])) ax.xaxis.set_major_formatter(mdates.DateFormatter("%-d %b\n%Y")) ax.set_yticks(range(0, 101, 25)) ax.grid(True) return fig, df
def main(): """Do Something Fun!""" form = cgi.FieldStorage() ts = datetime.datetime.strptime(form.getfirst("date"), "%Y-%m-%d") lat = float(form.getfirst("lat")) lon = float(form.getfirst("lon")) fmt = form.getfirst("format") if fmt != 'json': ssw("Content-type: text/plain\n\n") ssw("ERROR: Service only emits json at this time") return i, j = iemre.find_ij(lon, lat) offset = iemre.daily_offset(ts) res = { 'data': [], } fn = iemre.get_daily_ncname(ts.year) ssw('Content-type: application/json\n\n') if not os.path.isfile(fn): ssw(json.dumps(res)) sys.exit() if i is None or j is None: ssw(json.dumps({'error': 'Coordinates outside of domain'})) return if ts.year > 1980: ncfn = "/mesonet/data/prism/%s_daily.nc" % (ts.year, ) if not os.path.isfile(ncfn): prism_precip = None else: i2, j2 = prismutil.find_ij(lon, lat) with ncopen(ncfn) as nc: prism_precip = nc.variables['ppt'][offset, j2, i2] / 25.4 else: prism_precip = None if ts.year > 2010: ncfn = iemre.get_daily_mrms_ncname(ts.year) if not os.path.isfile(ncfn): mrms_precip = None else: j2 = int((lat - iemre.SOUTH) * 100.0) i2 = int((lon - iemre.WEST) * 100.0) with ncopen(ncfn) as nc: mrms_precip = nc.variables['p01d'][offset, j2, i2] / 25.4 else: mrms_precip = None nc = ncopen(fn) c2000 = ts.replace(year=2000) coffset = iemre.daily_offset(c2000) cnc = ncopen(iemre.get_dailyc_ncname()) res['data'].append({ 'prism_precip_in': myrounder(prism_precip, 2), 'mrms_precip_in': myrounder(mrms_precip, 2), 'daily_high_f': myrounder( datatypes.temperature(nc.variables['high_tmpk'][offset, j, i], 'K').value('F'), 1), '12z_high_f': myrounder( datatypes.temperature(nc.variables['high_tmpk_12z'][offset, j, i], 'K').value('F'), 1), 'climate_daily_high_f': myrounder( datatypes.temperature(cnc.variables['high_tmpk'][coffset, j, i], 'K').value("F"), 1), 'daily_low_f': myrounder( datatypes.temperature(nc.variables['low_tmpk'][offset, j, i], 'K').value("F"), 1), '12z_low_f': myrounder( datatypes.temperature(nc.variables['low_tmpk_12z'][offset, j, i], 'K').value('F'), 1), 'avg_dewpoint_f': myrounder( datatypes.temperature(nc.variables['avg_dwpk'][offset, j, i], 'K').value('F'), 1), 'climate_daily_low_f': myrounder( datatypes.temperature(cnc.variables['low_tmpk'][coffset, j, i], 'K').value("F"), 1), 'daily_precip_in': myrounder(nc.variables['p01d'][offset, j, i] / 25.4, 2), '12z_precip_in': myrounder(nc.variables['p01d_12z'][offset, j, i] / 25.4, 2), 'climate_daily_precip_in': myrounder(cnc.variables['p01d'][coffset, j, i] / 25.4, 2), 'srad_mj': myrounder(nc.variables['rsds'][offset, j, i] * 86400. / 1000000., 2), 'avg_windspeed_mps': myrounder(nc.variables['wind_speed'][offset, j, i], 2), }) nc.close() cnc.close() ssw(json.dumps(res))
def plotter(fdict): """ Go """ ctx = util.get_autoplot_context(fdict, get_description()) ptype = ctx['ptype'] sdate = ctx['sdate'] edate = ctx['edate'] src = ctx['src'] opt = ctx['opt'] usdm = ctx['usdm'] if sdate.year != edate.year: raise NoDataFound('Sorry, do not support multi-year plots yet!') days = (edate - sdate).days sector = ctx['sector'] if sdate == edate: title = sdate.strftime("%-d %B %Y") else: title = "%s to %s (inclusive)" % (sdate.strftime("%-d %b"), edate.strftime("%-d %b %Y")) x0 = 0 x1 = -1 y0 = 0 y1 = -1 state = None if len(sector) == 2: state = sector sector = 'state' if src == 'mrms': ncfn = iemre.get_daily_mrms_ncname(sdate.year) clncfn = iemre.get_dailyc_mrms_ncname() ncvar = 'p01d' source = 'MRMS Q3' subtitle = 'NOAA MRMS Project, GaugeCorr and RadarOnly' elif src == 'iemre': ncfn = iemre.get_daily_ncname(sdate.year) clncfn = iemre.get_dailyc_ncname() ncvar = 'p01d_12z' source = 'IEM Reanalysis' subtitle = 'IEM Reanalysis is derived from various NOAA datasets' else: ncfn = "/mesonet/data/prism/%s_daily.nc" % (sdate.year, ) clncfn = "/mesonet/data/prism/prism_dailyc.nc" ncvar = 'ppt' source = 'OSU PRISM' subtitle = ('PRISM Climate Group, Oregon State Univ., ' 'http://prism.oregonstate.edu, created 4 Feb 2004.') mp = MapPlot(sector=sector, state=state, axisbg='white', nocaption=True, title='%s:: %s Precip %s' % (source, title, PDICT3[opt]), subtitle='Data from %s' % (subtitle, ), titlefontsize=14) idx0 = iemre.daily_offset(sdate) idx1 = iemre.daily_offset(edate) + 1 if not os.path.isfile(ncfn): raise NoDataFound("No data for that year, sorry.") with util.ncopen(ncfn) as nc: if state is not None: x0, y0, x1, y1 = util.grid_bounds(nc.variables['lon'][:], nc.variables['lat'][:], state_bounds[state]) elif sector in SECTORS: bnds = SECTORS[sector] x0, y0, x1, y1 = util.grid_bounds( nc.variables['lon'][:], nc.variables['lat'][:], [bnds[0], bnds[2], bnds[1], bnds[3]]) lats = nc.variables['lat'][y0:y1] lons = nc.variables['lon'][x0:x1] if sdate == edate: p01d = distance(nc.variables[ncvar][idx0, y0:y1, x0:x1], 'MM').value('IN') elif (idx1 - idx0) < 32: p01d = distance( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0), 'MM').value('IN') else: # Too much data can overwhelm this app, need to chunk it for i in range(idx0, idx1, 10): i2 = min([i + 10, idx1]) if idx0 == i: p01d = distance( np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0), 'MM').value('IN') else: p01d += distance( np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0), 'MM').value('IN') if np.ma.is_masked(np.max(p01d)): raise NoDataFound("Data Unavailable") units = 'inches' cmap = plt.get_cmap(ctx['cmap']) cmap.set_bad('white') if opt == 'dep': # Do departure work now with util.ncopen(clncfn) as nc: climo = distance( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0), 'MM').value('IN') p01d = p01d - climo [maxv] = np.percentile(np.abs(p01d), [ 99, ]) clevs = np.around(np.linspace(0 - maxv, maxv, 11), decimals=2) elif opt == 'per': with util.ncopen(clncfn) as nc: climo = distance( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0), 'MM').value('IN') p01d = p01d / climo * 100. cmap.set_under('white') cmap.set_over('black') clevs = [1, 10, 25, 50, 75, 100, 125, 150, 200, 300, 500] units = 'percent' else: p01d = np.where(p01d < 0.001, np.nan, p01d) cmap.set_under('white') clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10] if days > 6: clevs = [0.01, 0.3, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 10, 15, 20] if days > 29: clevs = [0.01, 0.5, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35] if days > 90: clevs = [0.01, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35, 40] x2d, y2d = np.meshgrid(lons, lats) if ptype == 'c': mp.contourf(x2d, y2d, p01d, clevs, cmap=cmap, units=units, iline=False) else: res = mp.pcolormesh(x2d, y2d, p01d, clevs, cmap=cmap, units=units) res.set_rasterized(True) if sector != 'midwest': mp.drawcounties() mp.drawcities() if usdm == 'yes': mp.draw_usdm(edate, filled=False, hatched=True) return mp.fig
def plotter(fdict): """ Go """ ctx = get_autoplot_context(fdict, get_description()) year = ctx['year'] threshold = ctx['threshold'] period = ctx['period'] state = ctx['state'] pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr = %s """, pgconn, params=(state, ), index_col='state_abbr', geom_col='the_geom') nc = ncopen(iemre.get_daily_ncname(year)) precip = nc.variables['p01d'] czs = CachingZonalStats(iemre.AFFINE) hasdata = np.zeros((nc.dimensions['lat'].size, nc.dimensions['lon'].size)) czs.gen_stats(hasdata, states['the_geom']) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice, islice]) hasdata = np.flipud(hasdata) datapts = np.sum(np.where(hasdata > 0, 1, 0)) now = datetime.date(year, 1, 1) now += datetime.timedelta(days=(period - 1)) ets = datetime.date(year, 12, 31) today = datetime.date.today() if ets > today: ets = today days = [] coverage = [] while now <= ets: idx = iemre.daily_offset(now) sevenday = np.sum(precip[(idx - period):idx, :, :], 0) pday = np.where(hasdata > 0, sevenday[:, :], -1) tots = np.sum(np.where(pday >= (threshold * 25.4), 1, 0)) days.append(now) coverage.append(tots / float(datapts) * 100.0) now += datetime.timedelta(days=1) df = pd.DataFrame(dict(day=pd.Series(days), coverage=pd.Series(coverage))) (fig, ax) = plt.subplots(1, 1) ax.bar(days, coverage, fc='g', ec='g') ax.set_title( ("%s IEM Estimated Areal Coverage Percent of %s\n" " receiving %.2f inches of rain over trailing %s day period") % (year, reference.state_names[state], threshold, period)) ax.set_ylabel("Areal Coverage [%]") ax.xaxis.set_major_formatter(mdates.DateFormatter('%b\n%-d')) ax.set_yticks(range(0, 101, 25)) ax.grid(True) return fig, df