def load_stage4(): """ It sucks, but we need to load the stage IV data to give us something to benchmark the MRMS data against, to account for two things: 1) Wind Farms 2) Over-estimates """ printt("load_stage4() called...") # The stage4 files store precip in the rears, so compute 1 AM one_am = datetime.datetime(VALID.year, VALID.month, VALID.day, 12, 0) one_am = one_am.replace(tzinfo=pytz.timezone("UTC")) one_am = one_am.astimezone(pytz.timezone("America/Chicago")) # stage4 data is stored in the rears, so 1am one_am = one_am.replace(hour=1, minute=0, second=0) # clever hack for CST/CDT tomorrow = one_am + datetime.timedelta(hours=36) tomorrow = tomorrow.replace(hour=1) sts_tidx = iemre.hourly_offset(one_am) ets_tidx = iemre.hourly_offset(tomorrow) printt(("stage4 sts_tidx:%s[%s] ets_tidx:%s[%s]" ) % (sts_tidx, one_am, ets_tidx, tomorrow)) nc = ncopen("/mesonet/data/stage4/%s_stage4_hourly.nc" % (VALID.year, )) p01m = nc.variables['p01m'] lats = nc.variables['lat'][:] lons = nc.variables['lon'][:] # crossing jan 1 if ets_tidx < sts_tidx: printt("Exercise special stageIV logic for jan1!") totals = np.sum(p01m[sts_tidx:, :, :], axis=0) nc.close() nc = ncopen( "/mesonet/data/stage4/%s_stage4_hourly.nc" % (tomorrow.year, ) ) p01m = nc.variables['p01m'] totals += np.sum(p01m[:ets_tidx, :, :], axis=0) else: totals = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0) nc.close() if np.ma.max(totals) > 0: pass else: print('No StageIV data found, aborting...') sys.exit() # set a small non-zero number to keep things non-zero totals = np.where(totals > 0.001, totals, 0.001) xaxis = np.arange(MYWEST, MYEAST, 0.01) yaxis = np.arange(MYSOUTH, MYNORTH, 0.01) xi, yi = np.meshgrid(xaxis, yaxis) nn = NearestNDInterpolator((lons.flatten(), lats.flatten()), totals.flatten()) STAGE4[:] = nn(xi, yi) write_grid(STAGE4, 'stage4') printt("load_stage4() finished...")
def init_year(ts): """ Create a new NetCDF file for a year of our specification! """ fp = iemre.get_daily_mrms_ncname(ts.year) nc = ncopen(fp, 'a', timeout=300) nc.createDimension('nv', 2) lat = nc.variables['lat'] lat.bounds = "lat_bnds" lat[:] = np.arange(iemre.SOUTH + 0.005, iemre.NORTH, 0.01) lat_bnds = nc.createVariable('lat_bnds', np.float, ('lat', 'nv')) lat_bnds[:, 0] = np.arange(iemre.SOUTH, iemre.NORTH, 0.01) lat_bnds[:, 1] = np.arange(iemre.SOUTH + 0.01, iemre.NORTH + 0.01, 0.01) lon = nc.variables['lon'] lon.bounds = "lon_bnds" lon[:] = np.arange(iemre.WEST, iemre.EAST, 0.01) lon_bnds = nc.createVariable('lon_bnds', np.float, ('lon', 'nv')) lon_bnds[:, 0] = np.arange(iemre.WEST, iemre.EAST, 0.01) lon_bnds[:, 1] = np.arange(iemre.WEST + 0.01, iemre.EAST + 0.01, 0.01) nc.close()
def main(): """Go Main Go""" sts = datetime.datetime(2018, 4, 20, 0) sts = sts.replace(tzinfo=pytz.utc) ets = datetime.datetime(2018, 5, 11, 0) ets = ets.replace(tzinfo=pytz.utc) nc = ncopen(iemre.get_hourly_ncname(sts.year)) lons = nc.variables['lon'][:] lats = nc.variables['lat'][:] running = np.zeros((len(nc.dimensions['lat']), len(nc.dimensions['lon']))) maxval = np.zeros((len(nc.dimensions['lat']), len(nc.dimensions['lon']))) interval = datetime.timedelta(hours=1) now = sts i, j = iemre.find_ij(-93.61, 41.99) while now < ets: offset = iemre.hourly_offset(now) p01m = np.sum(nc.variables['p01m'][offset - 24:offset], axis=0) # 0.05in is 1.27 mm this = np.where(p01m > THRESHOLD, 1, 0) running = np.where(this == 1, 0, running + 1) maxval = np.where(running > maxval, running, maxval) print("%s %s %s" % (now, running[j, i], maxval[j, i])) now += interval # maxval = numpy.where(domain == 1, maxval, 1.e20) m = plot.MapPlot(sector='midwest', title=('Max Period ' 'between 24 Hour 0.25+ inch Total Precipitation'), subtitle=('Period of 20 Apr - 11 May 2018, ' 'based on NCEP Stage IV data')) extra = lons[-1] + (lons[-1] - lons[-2]) lons[-1] = extra # lons = np.concatenate([lons, [extra, ]]) extra = lats[-1] + (lats[-1] - lats[-2]) lats[-1] = extra # lats = np.concatenate([lats, [extra, ]]) lons, lats = np.meshgrid(lons, lats) # m.pcolormesh(x, y, maxval / 24.0, numpy.arange(0,25,1), units='days') maxval = np.where(maxval > 800, 73., maxval) cmap = plt.get_cmap('terrain') m.contourf(lons, lats, maxval / 24.0, np.arange(1, 11.1, 1), cmap=cmap, units='days', clip_on=False) m.postprocess(filename='test.png')
def main(): """Go Main Go""" nc = ncopen("/mesonet/data/iemre/1979_narr.nc") data = np.sum(nc.variables['apcp'][:, :, :], axis=0) m = MapPlot(sector='conus', axisbg='tan', title=(""), subtitle='', titlefontsize=16) t = distance(data, 'MM').value('IN') m.pcolormesh(nc.variables['lon'][:], nc.variables['lat'][:], t, np.arange(0, 60, 2), units='F') m.postprocess(filename='test.png')
def main(): """Go Main Go""" for year in range(1893, 2018): nc = ncopen(get_daily_ncname(year), 'a', timeout=600) gridsize = nc.dimensions['lat'].size * nc.dimensions['lon'].size for vname in ['p01d', 'high_tmpk', 'low_tmpk']: for i in range(nc.variables[vname].shape[0]): calday = nc.variables[vname][i, :, :] # Get a count of missing values missing = np.sum(calday.mask) / float(gridsize) if missing > 0.5: print("%s_12z->%s %s %.2f" % (vname, vname, i, missing)) nc.variables[vname][i, :, :] = ( nc.variables[vname + '_12z'][i, :, :] ) nc.close()
def main(argv): """Run Main""" hour = int(argv[1]) job = argv[2] sector = argv[3] netprod = argv[4] # Load EET netpng = Image.open("%s_%s_%s.gif" % (sector, netprod, job)) sz = (netpng.size[1], netpng.size[0]) net = (np.fromstring(netpng.tobytes(), np.uint8)).reshape(sz) if sector == "US": # Load up our tmpc surface data nc = ncopen('data/ructemps.nc') tmpc = nc.variables['tmpc'][hour, :, :] tmpc = np.flipud(tmpc) nc.close() # mask out the net based on temperature net = np.where(tmpc > 3.0, net, 15) else: # No Filtering... for now net[:, :] = 15. # Load N0Q n0qpng = Image.open("%s_N0Q_%s.gif" % (sector, job)) n0q = (np.fromstring(n0qpng.tobytes(), np.uint8)).reshape(sz) # Clean n0q if netprod == 'EET': n0q = np.where(net < 10, 0, n0q) else: n0q = np.where(net < 2, 0, n0q) # Make the image, please! png = Image.fromarray(n0q) # png.putpalette( make_colorramp() ) png.putpalette(n0qpng.getpalette()) meta = PngImagePlugin.PngInfo() meta.add_text('gentime', datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")) png.save('%s_N0Q_CLEAN_%s.png' % (sector, job), pnginfo=meta)
def main(): """Go Main Go""" form = cgi.FieldStorage() ts1 = datetime.datetime.strptime(form.getfirst("date1"), "%Y-%m-%d") ts2 = datetime.datetime.strptime(form.getfirst("date2"), "%Y-%m-%d") if ts1 > ts2: send_error("date1 larger than date2") if ts1.year != ts2.year: send_error("multi-year query not supported yet...") # Make sure we aren't in the future tsend = datetime.date.today() if ts2.date() > tsend: ts2 = datetime.datetime.now() - datetime.timedelta(days=1) lat = float(form.getfirst("lat")) lon = float(form.getfirst("lon")) if lon < iemre.WEST or lon > iemre.EAST: send_error("lon value outside of bounds: %s to %s" % (iemre.WEST, iemre.EAST)) if lat < iemre.SOUTH or lat > iemre.NORTH: send_error("lat value outside of bounds: %s to %s" % (iemre.SOUTH, iemre.NORTH)) # fmt = form["format"][0] i, j = iemre.find_ij(lon, lat) offset1 = iemre.daily_offset(ts1) offset2 = iemre.daily_offset(ts2) + 1 # Get our netCDF vars with ncopen(iemre.get_daily_ncname(ts1.year)) as nc: hightemp = datatypes.temperature( nc.variables["high_tmpk"][offset1:offset2, j, i], "K").value("F") high12temp = datatypes.temperature( nc.variables["high_tmpk_12z"][offset1:offset2, j, i], "K").value("F") lowtemp = datatypes.temperature( nc.variables["low_tmpk"][offset1:offset2, j, i], "K").value("F") low12temp = datatypes.temperature( nc.variables["low_tmpk_12z"][offset1:offset2, j, i], "K").value("F") precip = nc.variables["p01d"][offset1:offset2, j, i] / 25.4 precip12 = nc.variables["p01d_12z"][offset1:offset2, j, i] / 25.4 # Get our climatology vars c2000 = ts1.replace(year=2000) coffset1 = iemre.daily_offset(c2000) c2000 = ts2.replace(year=2000) coffset2 = iemre.daily_offset(c2000) + 1 with ncopen(iemre.get_dailyc_ncname()) as cnc: chigh = datatypes.temperature( cnc.variables["high_tmpk"][coffset1:coffset2, j, i], "K").value("F") clow = datatypes.temperature( cnc.variables["low_tmpk"][coffset1:coffset2, j, i], "K").value("F") cprecip = cnc.variables["p01d"][coffset1:coffset2, j, i] / 25.4 if ts1.year > 1980: i2, j2 = prismutil.find_ij(lon, lat) with ncopen("/mesonet/data/prism/%s_daily.nc" % (ts1.year, )) as nc: prism_precip = nc.variables["ppt"][offset1:offset2, j2, i2] / 25.4 else: prism_precip = [None] * (offset2 - offset1) if ts1.year > 2010: j2 = int((lat - iemre.SOUTH) * 100.0) i2 = int((lon - iemre.WEST) * 100.0) with ncopen(iemre.get_daily_mrms_ncname(ts1.year)) as nc: mrms_precip = nc.variables["p01d"][offset1:offset2, j2, i2] / 25.4 else: mrms_precip = [None] * (offset2 - offset1) res = {"data": []} for i in range(0, offset2 - offset1): now = ts1 + datetime.timedelta(days=i) res["data"].append({ "date": now.strftime("%Y-%m-%d"), "mrms_precip_in": clean(mrms_precip[i]), "prism_precip_in": clean(prism_precip[i]), "daily_high_f": clean(hightemp[i]), "12z_high_f": clean(high12temp[i]), "climate_daily_high_f": clean(chigh[i]), "daily_low_f": clean(lowtemp[i]), "12z_low_f": clean(low12temp[i]), "climate_daily_low_f": clean(clow[i]), "daily_precip_in": clean(precip[i]), "12z_precip_in": clean(precip12[i]), "climate_daily_precip_in": clean(cprecip[i]), }) ssw("Content-type: application/json\n\n") ssw(json.dumps(res))
def grid_hour(ts): """ I proctor the gridding of data on an hourly basis @param ts Timestamp of the analysis, we'll consider a 20 minute window """ pprint("grid_hour called...") nc = ncopen(iemre.get_hourly_ncname(ts.year), "a", timeout=300) domain = nc.variables["hasdata"][:, :] nc.close() ts0 = ts - datetime.timedelta(minutes=10) ts1 = ts + datetime.timedelta(minutes=10) utcnow = datetime.datetime.utcnow() utcnow = utcnow.replace(tzinfo=pytz.utc) - datetime.timedelta(hours=36) # If we are near realtime, look in IEMAccess instead of ASOS database mybuf = 2.0 params = ( iemre.WEST - mybuf, iemre.SOUTH - mybuf, iemre.WEST - mybuf, iemre.NORTH + mybuf, iemre.EAST + mybuf, iemre.NORTH + mybuf, iemre.EAST + mybuf, iemre.SOUTH - mybuf, iemre.WEST - mybuf, iemre.SOUTH - mybuf, ts0, ts1, ) if utcnow < ts: dbconn = get_dbconn("iem", user="******") sql = """SELECT t.id as station, ST_x(geom) as lon, ST_y(geom) as lat, max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf, max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt, max(getskyc(skyc1)) as max_skyc1, max(getskyc(skyc2)) as max_skyc2, max(getskyc(skyc3)) as max_skyc3, max(case when phour > 0 and phour < 1000 then phour else 0 end) as phour, max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf, max(case when sknt >= 0 then sknt else 0 end) as sknt, max(case when sknt >= 0 then drct else 0 end) as drct from current_log s JOIN stations t on (s.iemid = t.iemid) WHERE ST_Contains( ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s %s, %s %s, %s %s, %s %s))'), geom) and valid >= %s and valid < %s GROUP by station, lon, lat """ else: dbconn = get_dbconn("asos", user="******") sql = """SELECT station, ST_x(geom) as lon, st_y(geom) as lat, max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf, max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt, max(getskyc(skyc1)) as max_skyc1, max(getskyc(skyc2)) as max_skyc2, max(getskyc(skyc3)) as max_skyc3, max(case when p01i > 0 and p01i < 1000 then p01i else 0 end) as phour, max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf, max(case when sknt >= 0 then sknt else 0 end) as sknt, max(case when sknt >= 0 then drct else 0 end) as drct from alldata a JOIN stations t on (a.station = t.id) WHERE ST_Contains( ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s %s, %s %s, %s %s, %s %s))'), geom) and (t.network ~* 'ASOS' or t.network = 'AWOS') and valid >= %s and valid < %s GROUP by station, lon, lat""" df = read_sql(sql, dbconn, params=params, index_col="station") pprint("got database results") if df.empty: print(("%s has no entries, FAIL") % (ts.strftime("%Y-%m-%d %H:%M"), )) return ures, vres = grid_wind(df, domain) pprint("grid_wind is done") if ures is None: print("iemre.hourly_analysis failure for uwnd at %s" % (ts, )) else: write_grid(ts, "uwnd", ures) write_grid(ts, "vwnd", vres) tmpf = generic_gridder(df, "max_tmpf", domain) pprint("grid tmpf is done") if tmpf is None: print("iemre.hourly_analysis failure for tmpk at %s" % (ts, )) else: dwpf = generic_gridder(df, "max_dwpf", domain) pprint("grid dwpf is done") # require that dwpk <= tmpk mask = ~np.isnan(dwpf) mask[mask] &= dwpf[mask] > tmpf[mask] dwpf = np.where(mask, tmpf, dwpf) write_grid(ts, "tmpk", masked_array(tmpf, data_units="degF").to("degK")) write_grid(ts, "dwpk", masked_array(dwpf, data_units="degF").to("degK")) res = grid_skyc(df, domain) pprint("grid skyc is done") if res is None: print("iemre.hourly_analysis failure for skyc at %s" % (ts, )) else: write_grid(ts, "skyc", res)
def do(date): """ Process for a given date 6z file has 6z to 9z data """ pgconn = get_dbconn("coop") ccursor = pgconn.cursor() ccursor2 = pgconn.cursor() sts = date.replace(hour=6) # 6z ets = sts + datetime.timedelta(days=1) now = sts interval = datetime.timedelta(hours=3) mode = NC_MODE lats, lons = None, None while now < ets: # See if we have Grib data first fn = now.strftime( "/mesonet/ARCHIVE/data/%Y/%m/%d/model/NARR/rad_%Y%m%d%H00.grib") if os.path.isfile(fn): mode = GRIB_MODE grb = pygrib.open(fn)[1] if lats is None: lats, lons = grb.latlons() total = grb["values"] * 10800.0 else: total += grb["values"] * 10800.0 now += interval continue fn = now.strftime( "/mesonet/ARCHIVE/data/%Y/%m/%d/model/NARR/rad_%Y%m%d%H00.nc") if not os.path.isfile(fn): LOG.info("MISSING NARR: %s", fn) sys.exit() with ncopen(fn, timeout=300) as nc: rad = nc.variables["Downward_shortwave_radiation_flux"][0, :, :] if now == sts: xc = nc.variables["x"][:] * 1000.0 # convert to meters yc = nc.variables["y"][:] * 1000.0 # convert to meters total = rad * 10800.0 # 3 hr rad to total rad else: total += rad * 10800.0 now += interval ccursor.execute( """ SELECT station, ST_x(geom), ST_y(geom), temp24_hour from alldata a JOIN stations t on (a.station = t.id) where day = %s """, (date.strftime("%Y-%m-%d"), ), ) for row in ccursor: if mode == NC_MODE: (x, y) = pyproj.transform(P4326, LCC, row[1], row[2]) (gridxs, gridys, distances) = get_gp(xc, yc, x, y) else: # Note the hacky reversing of grid coords (gridys, gridxs, distances) = get_gp2(lons, lats, row[1], row[2]) z0 = total[gridys[0], gridxs[0]] z1 = total[gridys[1], gridxs[1]] z2 = total[gridys[2], gridxs[2]] z3 = total[gridys[3], gridxs[3]] val = (z0 / distances[0] + z1 / distances[1] + z2 / distances[2] + z3 / distances[3]) / (1.0 / distances[0] + 1.0 / distances[1] + 1.0 / distances[2] + 1.0 / distances[3]) rad_mj = float(val) / 1000000.0 if rad_mj < 0: LOG.info("WHOA! Negative RAD: %.2f, station: %s", rad_mj, row[0]) continue if np.isnan(rad_mj): LOG.info("NARR SRAD is NaN, station: %s", row[0]) rad_mj = None # if our station is 12z, then this day's data goes into 'tomorrow' # if our station is not, then this day is today date2 = date.strftime("%Y-%m-%d") if row[3] in range(4, 13): date2 = (date + datetime.timedelta(days=1)).strftime("%Y-%m-%d") ccursor2.execute( f"UPDATE alldata_{row[0][:2]} SET narr_srad = %s WHERE " "day = %s and station = %s", (rad_mj, date2, row[0]), ) ccursor2.close() pgconn.commit() pgconn.close()
def plotter(fdict): """ Go """ ctx = util.get_autoplot_context(fdict, get_description()) date = ctx['date'] sector = ctx['sector'] days = ctx['trailing'] threshold = ctx['threshold'] window_sts = date - datetime.timedelta(days=days) if window_sts.year != date.year: raise ValueError('Sorry, do not support multi-year plots yet!') if len(sector) != 2: raise ValueError("Sorry, this does not support multi-state plots yet.") idx0 = iemre.daily_offset(window_sts) idx1 = iemre.daily_offset(date) ncfn = iemre.get_daily_mrms_ncname(date.year) ncvar = 'p01d' if not os.path.isfile(ncfn): raise ValueError("No data for that year, sorry.") # Get the state weight df = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom from states where state_abbr = %s """, util.get_dbconn('postgis'), params=(sector, ), index_col=None, geom_col='the_geom') czs = CachingZonalStats(iemre.MRMS_AFFINE) with util.ncopen(ncfn) as nc: czs.gen_stats( np.zeros((nc.variables['lat'].size, nc.variables['lon'].size)), df['the_geom']) hasdata = None jslice = None islice = None for nav in czs.gridnav: hasdata = np.ones((nav.ysz, nav.xsz)) hasdata[nav.mask] = 0. # careful here as y is flipped in this context jslice = slice(nc.variables['lat'].size - (nav.y0 + nav.ysz), nc.variables['lat'].size - nav.y0) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata = np.flipud(hasdata) today = distance(nc.variables[ncvar][idx1, jslice, islice], 'MM').value('IN') if (idx1 - idx0) < 32: p01d = distance( np.sum(nc.variables[ncvar][idx0:idx1, jslice, islice], 0), 'MM').value('IN') else: # Too much data can overwhelm this app, need to chunk it for i in range(idx0, idx1, 10): i2 = min([i + 10, idx1]) if idx0 == i: p01d = distance( np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0), 'MM').value('IN') else: p01d += distance( np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0), 'MM').value('IN') # Get climatology with util.ncopen(iemre.get_dailyc_mrms_ncname()) as nc: if (idx1 - idx0) < 32: c_p01d = distance( np.sum(nc.variables[ncvar][idx0:idx1, jslice, islice], 0), 'MM').value('IN') else: # Too much data can overwhelm this app, need to chunk it for i in range(idx0, idx1, 10): i2 = min([i + 10, idx1]) if idx0 == i: c_p01d = distance( np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0), 'MM').value('IN') else: c_p01d += distance( np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0), 'MM').value('IN') # we actually don't care about weights at this fine of scale cells = np.sum(np.where(hasdata > 0, 1, 0)) departure = p01d - c_p01d # Update departure and today to values unconsidered below when out of state departure = np.where(hasdata > 0, departure, -9999) today = np.where(hasdata > 0, today, 0) ranges = [[-99, -3], [-3, -2], [-2, -1], [-1, 0], [0, 1], [1, 2], [2, 3], [3, 99]] x = [] x2 = [] labels = [] for (minv, maxv) in ranges: labels.append("%.0f to %.0f" % (minv, maxv)) # How many departure cells in this range hits = np.logical_and(departure < maxv, departure > minv) hits2 = np.logical_and(hits, today > threshold) x.append(np.sum(np.where(hits, 1, 0)) / float(cells) * 100.) x2.append(np.sum(np.where(hits2, 1, 0)) / float(cells) * 100.) (fig, ax) = plt.subplots(1, 1) ax.set_title(("%s NOAA MRMS %s %.2f inch Precip Coverage") % (state_names[sector], date.strftime("%-d %b %Y"), threshold)) ax.bar(np.arange(8) - 0.2, x, align='center', width=0.4, label='Trailing %s Day Departure' % (days, )) ax.bar(np.arange(8) + 0.2, x2, align='center', width=0.4, label='%s Coverage (%.1f%% Tot)' % (date.strftime("%-d %b %Y"), sum(x2))) for i, (_x1, _x2) in enumerate(zip(x, x2)): ax.text(i - 0.2, _x1 + 1, "%.1f" % (_x1, ), ha='center') ax.text(i + 0.2, _x2 + 1, "%.1f" % (_x2, ), ha='center') ax.set_xticks(np.arange(8)) ax.set_xticklabels(labels) ax.set_xlabel("Trailing %s Day Precip Departure [in]" % (days, )) ax.set_position([0.1, 0.2, 0.8, 0.7]) ax.legend(loc=(0., -0.2), ncol=2) ax.set_ylabel("Areal Coverage of %s [%%]" % (state_names[sector], )) ax.grid(True) ax.set_xlim(-0.5, 7.5) ax.set_ylim(0, max([max(x2), max(x)]) + 5) return fig
def workflow(valid): """Our workflow""" if valid.month == 1 and valid.day == 1: print("prism_adjust_stage4, sorry Jan 1 processing is a TODO!") return # read prism tidx = daily_offset(valid) nc = ncopen("/mesonet/data/prism/%s_daily.nc" % (valid.year, ), "r") ppt = nc.variables["ppt"][tidx, :, :] # missing as zero ppt = np.where(ppt.mask, 0, ppt) lons = nc.variables["lon"][:] lats = nc.variables["lat"][:] nc.close() (lons, lats) = np.meshgrid(lons, lats) (i, j) = prismutil.find_ij(DEBUGLON, DEBUGLAT) LOG.debug("prism debug point ppt: %.3f", ppt[j, i]) # Interpolate this onto the stage4 grid nc = ncopen( ("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), "a", timeout=300, ) p01m = nc.variables["p01m"] p01m_status = nc.variables["p01m_status"] s4lons = nc.variables["lon"][:] s4lats = nc.variables["lat"][:] i, j = find_ij(s4lons, s4lats, DEBUGLON, DEBUGLAT) # Values are in the hourly arrears, so start at -23 and thru current hour sts_tidx = hourly_offset(valid - datetime.timedelta(hours=23)) ets_tidx = hourly_offset(valid + datetime.timedelta(hours=1)) s4total = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0) LOG.debug( "stage4 s4total: %.3f lon: %.2f (%.2f) lat: %.2f (%.2f)", s4total[i, j], s4lons[i, j], DEBUGLON, s4lats[i, j], DEBUGLAT, ) # make sure the s4total does not have zeros s4total = np.where(s4total < 0.001, 0.001, s4total) nn = NearestNDInterpolator((lons.flatten(), lats.flatten()), ppt.flat) prism_on_s4grid = nn(s4lons, s4lats) LOG.debug( "shape of prism_on_s4grid: %s s4lons: %s ll: %.2f s4lats: %s ll: %.2f", np.shape(prism_on_s4grid), np.shape(s4lons), s4lons[0, 0], np.shape(s4lats), s4lats[0, 0], ) multiplier = prism_on_s4grid / s4total LOG.debug( "prism avg: %.3f stageIV avg: %.3f prismons4grid avg: %.3f mul: %.3f", np.mean(ppt), np.mean(s4total), np.mean(prism_on_s4grid), np.mean(multiplier), ) LOG.debug( "Boone IA0807 prism: %.3f stageIV: %.4f prismons4grid: %.3f mul: %.3f", ppt[431, 746], s4total[i, j], prism_on_s4grid[i, j], multiplier[i, j], ) # Do the work now, we should not have to worry about the scale factor for tidx in range(sts_tidx, ets_tidx): oldval = p01m[tidx, :, :] # we threshold the s4total to at least 0.001, so we divide by 24 here # and denote that if the multiplier is zero, then we net zero newval = np.where(oldval < 0.001, 0.00004, oldval) * multiplier nc.variables["p01m"][tidx, :, :] = newval LOG.debug( "adjust tidx: %s oldval: %.3f newval: %.3f", tidx, oldval[i, j], newval[i, j], ) # make sure have data if np.ma.max(newval) > 0: p01m_status[tidx] = 2 else: print(("prism_adjust_stage4 NOOP for time %s[idx:%s]") % ( (datetime.datetime(valid.year, 1, 1, 0) + datetime.timedelta(hours=tidx)).strftime("%Y-%m-%dT%H"), tidx, )) nc.close()
def create_netcdf(valid): """Create and return the netcdf file""" ncfn = "/mesonet/data/iemre/cfs_%s.nc" % (valid.strftime("%Y%m%d%H"), ) nc = ncopen(ncfn, "w") nc.title = "IEM Regridded CFS Member 1 Forecast %s" % (valid.year, ) nc.platform = "Grided Forecast" nc.description = "IEM Regridded CFS on 0.125 degree grid" nc.institution = "Iowa State University, Ames, IA, USA" nc.source = "Iowa Environmental Mesonet" nc.project_id = "IEM" nc.realization = 1 nc.Conventions = "CF-1.0" nc.contact = "Daryl Herzmann, [email protected], 515-294-5978" nc.history = ("%s Generated") % ( datetime.datetime.now().strftime("%d %B %Y"), ) nc.comment = "No comment at this time" # Setup Dimensions nc.createDimension("lat", iemre.NY) nc.createDimension("lon", iemre.NX) days = iemre.daily_offset(valid.replace(month=12, day=31)) + 1 nc.createDimension("time", int(days)) # Setup Coordinate Variables lat = nc.createVariable("lat", np.float, ("lat", )) lat.units = "degrees_north" lat.long_name = "Latitude" lat.standard_name = "latitude" lat.axis = "Y" lat[:] = iemre.YAXIS lon = nc.createVariable("lon", np.float, ("lon", )) lon.units = "degrees_east" lon.long_name = "Longitude" lon.standard_name = "longitude" lon.axis = "X" lon[:] = iemre.XAXIS tm = nc.createVariable("time", np.float, ("time", )) tm.units = "Days since %s-01-01 00:00:0.0" % (valid.year, ) tm.long_name = "Time" tm.standard_name = "time" tm.axis = "T" tm.calendar = "gregorian" tm[:] = np.arange(0, int(days)) high = nc.createVariable("high_tmpk", np.uint16, ("time", "lat", "lon"), fill_value=65535) high.units = "K" high.scale_factor = 0.01 high.long_name = "2m Air Temperature Daily High" high.standard_name = "2m Air Temperature" high.coordinates = "lon lat" low = nc.createVariable("low_tmpk", np.uint16, ("time", "lat", "lon"), fill_value=65535) low.units = "K" low.scale_factor = 0.01 low.long_name = "2m Air Temperature Daily Low" low.standard_name = "2m Air Temperature" low.coordinates = "lon lat" p01d = nc.createVariable("p01d", np.uint16, ("time", "lat", "lon"), fill_value=65535) p01d.units = "mm" p01d.scale_factor = 0.01 p01d.long_name = "Precipitation" p01d.standard_name = "Precipitation" p01d.coordinates = "lon lat" p01d.description = "Precipitation accumulation for the day" rsds = nc.createVariable("srad", np.uint16, ("time", "lat", "lon"), fill_value=65535) rsds.units = "W m-2" rsds.scale_factor = 0.01 rsds.long_name = "surface_downwelling_shortwave_flux_in_air" rsds.standard_name = "surface_downwelling_shortwave_flux_in_air" rsds.coordinates = "lon lat" rsds.description = "Global Shortwave Irradiance" nc.close() nc = ncopen(ncfn, "a") return nc
def create_netcdf(ts): """Make the file.""" fn = ts.strftime("/mesonet/data/ndfd/%Y%m%d%H_ndfd.nc") nc = ncopen(fn, "w") nc.title = "NWS NDFD Forecast From %s" % (ts.strftime("%Y-%m-%dT%H:%MZ"), ) nc.contact = "Daryl Herzmann, [email protected], 515-294-5978" nc.history = "%s Generated" % (utc().strftime("%Y-%m-%dT%H:%MZ"), ) grbs = pygrib.open(("/mesonet/ARCHIVE/data/2019/05/06/model/" "ndfd/00/ndfd.t00z.awp2p5f001.grib2")) grb = grbs[1] shape = grb.values.shape lats, lons = grb.latlons() # Setup dimensions nc.createDimension("lat", shape[0]) nc.createDimension("lon", shape[1]) nc.createDimension("day", 7) # Setup Coordinate Variables lat = nc.createVariable("lat", np.float, ("lat", "lon")) lat.units = "degrees_north" lat.long_name = "Latitude" lat.standard_name = "latitude" lat.axis = "Y" lat[:] = lats lon = nc.createVariable("lon", np.float, ("lat", "lon")) lon.units = "degrees_east" lon.long_name = "Longitude" lon.standard_name = "longitude" lon.axis = "X" lon[:] = lons day = nc.createVariable("day", np.float, ("day", )) day.units = "days since %s-01-01" % (ts.year, ) doy = int(ts.strftime("%j")) - 1 day[:] = range(doy, doy + 7) high = nc.createVariable("high_tmpk", np.uint16, ("day", "lat", "lon"), fill_value=65535) high.units = "K" high.scale_factor = 0.01 high.long_name = "2m Air Temperature Daily High" high.standard_name = "2m Air Temperature" high.coordinates = "lon lat" low = nc.createVariable("low_tmpk", np.uint16, ("day", "lat", "lon"), fill_value=65535) low.units = "K" low.scale_factor = 0.01 low.long_name = "2m Air Temperature Daily Low" low.standard_name = "2m Air Temperature" low.coordinates = "lon lat" p01d = nc.createVariable("p01d", np.uint16, ("day", "lat", "lon"), fill_value=65535) p01d.units = "mm" p01d.scale_factor = 0.01 p01d.long_name = "Precipitation" p01d.standard_name = "Precipitation" p01d.coordinates = "lon lat" p01d.description = "Precipitation accumulation for the day" nc.close() return ncopen(fn, "a")
def main(): """Go Main Go""" # Run for a period of 121 days ets = datetime.datetime.now() - datetime.timedelta(days=1) sts = ets - datetime.timedelta(days=121) # Get the normal accumm cnc = ncopen(iemre.get_dailyc_ncname(), "r") lons = cnc.variables["lon"][:] lats = cnc.variables["lat"][:] index0 = iemre.daily_offset(sts) index1 = iemre.daily_offset(ets) if index1 < index0: # Uh oh, we are spanning a year clprecip = np.sum(cnc.variables["p01d"][:index1, :, :], 0) clprecip = clprecip + np.sum(cnc.variables["p01d"][index0:, :, :], 0) else: clprecip = np.sum(cnc.variables["p01d"][index0:index1, :, :], 0) cnc.close() # Get the observed precip if sts.year != ets.year: # spanner, darn onc = ncopen(iemre.get_daily_ncname(sts.year)) obprecip = np.sum(onc.variables["p01d"][index0:, :, :], 0) onc.close() onc = ncopen(iemre.get_daily_ncname(ets.year)) obprecip = obprecip + np.sum(onc.variables["p01d"][:index1, :, :], 0) onc.close() else: ncfn = iemre.get_daily_ncname(sts.year) onc = ncopen(ncfn, "r") obprecip = np.sum(onc.variables["p01d"][index0:index1, :, :], 0) onc.close() lons, lats = np.meshgrid(lons, lats) # Plot departure from normal mp = MapPlot( sector="midwest", title=("Precipitation Departure %s - %s") % (sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")), subtitle="based on IEM Estimates", ) mp.pcolormesh(lons, lats, (obprecip - clprecip) / 25.4, np.arange(-10, 10, 1)) mp.postprocess(pqstr="plot c 000000000000 summary/4mon_diff.png bogus png") mp.close() # Plot normals mp = MapPlot( sector="midwest", title=("Normal Precipitation:: %s - %s") % (sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")), subtitle="based on IEM Estimates", ) mp.pcolormesh(lons, lats, (clprecip) / 25.4, np.arange(0, 30, 2)) mp.postprocess( pqstr="plot c 000000000000 summary/4mon_normals.png bogus png") mp.close() # Plot Obs mp = MapPlot( sector="midwest", title=("Estimated Precipitation:: %s - %s") % (sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")), subtitle="based on IEM Estimates", ) mp.pcolormesh(lons, lats, (obprecip) / 25.4, np.arange(0, 30, 2)) mp.postprocess( pqstr="plot c 000000000000 summary/4mon_stage4obs.png bogus png") mp.close()
def plotter(fdict): """ Go """ ctx = util.get_autoplot_context(fdict, get_description()) date = ctx["date"] sector = ctx["sector"] threshold = ctx["threshold"] threshold_mm = distance(threshold, "IN").value("MM") window_sts = date - datetime.timedelta(days=90) if window_sts.year != date.year: raise NoDataFound("Sorry, do not support multi-year plots yet!") # idx0 = iemre.daily_offset(window_sts) idx1 = iemre.daily_offset(date) ncfn = iemre.get_daily_mrms_ncname(date.year) if not os.path.isfile(ncfn): raise NoDataFound("No data found.") ncvar = "p01d" # Get the state weight df = gpd.GeoDataFrame.from_postgis( """ SELECT the_geom from states where state_abbr = %s """, util.get_dbconn("postgis"), params=(sector, ), index_col=None, geom_col="the_geom", ) czs = CachingZonalStats(iemre.MRMS_AFFINE) with util.ncopen(ncfn) as nc: czs.gen_stats( np.zeros((nc.variables["lat"].size, nc.variables["lon"].size)), df["the_geom"], ) jslice = None islice = None for nav in czs.gridnav: # careful here as y is flipped in this context jslice = slice( nc.variables["lat"].size - (nav.y0 + nav.ysz), nc.variables["lat"].size - nav.y0, ) islice = slice(nav.x0, nav.x0 + nav.xsz) grid = np.zeros( (jslice.stop - jslice.start, islice.stop - islice.start)) total = np.zeros( (jslice.stop - jslice.start, islice.stop - islice.start)) for i, idx in enumerate(range(idx1, idx1 - 90, -1)): total += nc.variables[ncvar][idx, jslice, islice] grid = np.where(np.logical_and(grid == 0, total > threshold_mm), i, grid) lon = nc.variables["lon"][islice] lat = nc.variables["lat"][jslice] mp = MapPlot( sector="state", state=sector, titlefontsize=14, subtitlefontsize=12, title=("NOAA MRMS Q3: Number of Recent Days " 'till Accumulating %s" of Precip') % (threshold, ), subtitle=("valid %s: based on per calendar day " "estimated preciptation, MultiSensorPass2 and " "RadarOnly products") % (date.strftime("%-d %b %Y"), ), ) x, y = np.meshgrid(lon, lat) cmap = get_cmap(ctx["cmap"]) cmap.set_over("k") cmap.set_under("white") mp.pcolormesh(x, y, grid, np.arange(0, 81, 10), cmap=cmap, units="days") mp.drawcounties() mp.drawcities() return mp.fig
def main(): """Go Main Go""" basedir = "/mnt/nrel/akrherz/livneh" outdir = "swatfiles_livneh" if os.path.isdir(outdir): print("ABORT: as %s exists" % (outdir, )) return os.mkdir(outdir) for dirname in ['precipitation', 'temperature']: os.mkdir("%s/%s" % (outdir, dirname)) pgconn = get_dbconn('idep') huc12df = gpd.GeoDataFrame.from_postgis(""" SELECT huc12, ST_Transform(simple_geom, %s) as geo from wbd_huc12 WHERE swat_use ORDER by huc12 """, pgconn, params=(PROJSTR,), index_col='huc12', geom_col='geo') hucs = huc12df.index.values years = range(1989, 2011) nc = ncopen("%s/livneh_NAmerExt_15Oct2014.198109.nc" % (basedir, )) # compute the affine ncaffine = Affine(nc.variables['lon'][1] - nc.variables['lon'][0], 0., nc.variables['lon'][0], 0., nc.variables['lat'][0] - nc.variables['lat'][1], nc.variables['lat'][-1]) czs = CachingZonalStats(ncaffine) nc.close() fps = [] for year in years: for month in range(1, 13): nc = ncopen(("%s/livneh_NAmerExt_15Oct2014.%s%02i.nc" ) % (basedir, year, month)) basedate, timesz = get_basedate(nc) for i in tqdm(range(timesz), desc="%s%02i" % (year, month)): tasmax = np.flipud(nc.variables['Tmax'][i, :, :]) tasmin = np.flipud(nc.variables['Tmin'][i, :, :]) pr = np.flipud(nc.variables['Prec'][i, :, :]) mytasmax = czs.gen_stats(tasmax, huc12df['geo']) mytasmin = czs.gen_stats(tasmin, huc12df['geo']) mypr = czs.gen_stats(pr, huc12df['geo']) for j, huc12 in enumerate(hucs): if i == 0 and year == years[0] and month == 1: fps.append([open(('%s/precipitation/P%s.txt' ) % (outdir, huc12), 'w'), open(('%s/temperature/T%s.txt' ) % (outdir, huc12), 'w')]) fps[j][0].write( "%s\n" % (basedate.strftime("%Y%m%d"), )) fps[j][1].write( "%s\n" % (basedate.strftime("%Y%m%d"), )) fps[j][0].write(("%.1f\n" ) % (mypr[j] if mypr[j] is not None else 0, )) fps[j][1].write(("%.2f,%.2f\n" ) % (mytasmax[j], mytasmin[j])) for fp in fps: fp[0].close() fp[1].close()
def main(): """Go main""" # Wow, why did I do it this way... better not change it as I bet the # other end is ignoring the headers... fmt = ("LAT,LONG,STN,DATE,TIME,T,TD,WCI,RH,THI,DIR,SPD,GST,ALT,SLP,VIS," "SKY,CEIL,CLD,SKYSUM,PR6,PR24,WX,MINT6,MAXT6,MINT24,MAXT24,AUTO," "PR1,PTMP1,PTMP2,PTMP3,PTMP4,SUBS1,SUBS2,SUBS3,SUBS4,STATIONNAME,") format_tokens = fmt.split(",") utc = datetime.datetime.utcnow() fn = "/mesonet/data/madis/mesonet1/%s.nc" % (utc.strftime("%Y%m%d_%H00"), ) if not os.path.isfile(fn): time.sleep(60) fn = "/mesonet/data/madis/mesonet1/%s.nc" % ( utc.strftime("%Y%m%d_%H00"), ) if not os.path.isfile(fn): if utc.minute > 30: print("madis2csv %s does not exist" % (fn, )) sys.exit() nc = ncopen(fn, "r", timeout=300) if nc is None: print(("madis2csv Numerous attempts to open MADIS netcdf %s failed!") % (fn, )) sys.exit(0) stations = chartostring(nc.variables["stationId"][:]) stationname = chartostring(nc.variables["stationName"][:]) tmpf = temperature(nc.variables["temperature"][:], "K").value("F") dwpf = temperature(nc.variables["dewpoint"][:], "K").value("F") drct = nc.variables["windDir"][:] smps = nc.variables["windSpeed"][:] * 1.94384449 alti = nc.variables["altimeter"][:] * 29.9196 / 1013.2 # in hPa vsby = nc.variables["visibility"][:] * 0.000621371192 # in hPa providers = chartostring(nc.variables["dataProvider"][:]) lat = nc.variables["latitude"][:] lon = nc.variables["longitude"][:] # ele = nc.variables["elevation"][:] p01m = nc.variables["precipAccum"][:] * 25.4 ptmp1 = temperature(nc.variables["roadTemperature1"][:], "K").value("F") ptmp2 = temperature(nc.variables["roadTemperature2"][:], "K").value("F") ptmp3 = temperature(nc.variables["roadTemperature3"][:], "K").value("F") ptmp4 = temperature(nc.variables["roadTemperature4"][:], "K").value("F") subs1 = temperature(nc.variables["roadSubsurfaceTemp1"][:], "K").value("F") subs2 = temperature(nc.variables["roadSubsurfaceTemp2"][:], "K").value("F") subs3 = temperature(nc.variables["roadSubsurfaceTemp3"][:], "K").value("F") subs4 = temperature(nc.variables["roadSubsurfaceTemp4"][:], "K").value("F") times = nc.variables["observationTime"][:] nc.close() db = {} for recnum in range(len(stations) - 1, -1, -1): station = stations[recnum] if station in db: continue ot = times[recnum] if numpy.ma.is_masked(ot) or ot > 2141347600: continue ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ot) ts = ts.replace(tzinfo=pytz.UTC) db[station] = { "STN": station, "PROVIDER": providers[recnum], "LAT": lat[recnum], "LONG": lon[recnum], "STATIONNAME": stationname[recnum].replace(",", " "), "DATE": ts.day, "TIME": ts.strftime("%H%M"), "T": sanity_check(tmpf[recnum], -100, 150, "%.0f", ""), "DIR": sanity_check(drct[recnum], -1, 361, "%.0f", ""), "TD": sanity_check(dwpf[recnum], -100, 150, "%.0f", ""), "SPD": sanity_check(smps[recnum], -1, 150, "%.0f", ""), "ALT": sanity_check(alti[recnum], 2000, 4000, "%.0f", ""), "VIS": sanity_check(vsby[recnum], -1, 40, "%.0f", ""), "PR1": sanity_check(p01m[recnum], -0.01, 10, "%.0f", ""), "SUBS1": sanity_check(subs1[recnum], -100, 150, "%.0f", ""), "SUBS2": sanity_check(subs2[recnum], -100, 150, "%.0f", ""), "SUBS3": sanity_check(subs3[recnum], -100, 150, "%.0f", ""), "SUBS4": sanity_check(subs4[recnum], -100, 150, "%.0f", ""), "PTMP1": sanity_check(ptmp1[recnum], -100, 150, "%.0f", ""), "PTMP2": sanity_check(ptmp2[recnum], -100, 150, "%.0f", ""), "PTMP3": sanity_check(ptmp3[recnum], -100, 150, "%.0f", ""), "PTMP4": sanity_check(ptmp4[recnum], -100, 150, "%.0f", ""), } out = open("/tmp/madis.csv", "w") out.write("%s\n" % (fmt, )) for stid in db: for key in format_tokens: if key in db[stid]: out.write("%s" % (db[stid][key], )) out.write(",") out.write("\n") out.close() pqstr = "data c %s fn/madis.csv bogus csv" % (utc.strftime("%Y%m%d%H%M"), ) cmd = "pqinsert -i -p '%s' /tmp/madis.csv" % (pqstr, ) subprocess.call(cmd, shell=True) os.remove("/tmp/madis.csv") out = open("/tmp/madis.csv", "w") out.write("%s\n" % (fmt, )) for stid in db: if db[stid]["PROVIDER"] in ["IADOT", "NEDOR"]: for key in format_tokens: if key in db[stid]: out.write("%s" % (db[stid][key], )) out.write(",") out.write("\n") out.close() pqstr = "data c %s fn/madis_iamn.csv bogus csv" % ( utc.strftime("%Y%m%d%H%M"), ) cmd = "pqinsert -i -p '%s' /tmp/madis.csv" % (pqstr, ) subprocess.call(cmd, shell=True) os.remove("/tmp/madis.csv")
def plotter(fdict): """ Go """ ctx = get_autoplot_context(fdict, get_description()) year = ctx['year'] thres = ctx['thres'] metric = distance(thres, 'IN').value('MM') state = ctx['state'][:2] sts = datetime.datetime(year, 10, 1) ets = datetime.datetime(year + 1, 5, 1) rows = [] pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr = %s """, pgconn, params=(state, ), index_col='state_abbr', geom_col='the_geom') sidx = iemre.daily_offset(sts) ncfn = iemre.get_daily_ncname(sts.year) if not os.path.isfile(ncfn): raise NoDataFound("Data for year %s not found" % (sts.year, )) with ncopen(ncfn) as nc: czs = CachingZonalStats(iemre.AFFINE) hasdata = np.zeros( (nc.dimensions['lat'].size, nc.dimensions['lon'].size)) czs.gen_stats(hasdata, states['the_geom']) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice, islice]) st = np.flipud(hasdata) stpts = np.sum(np.where(hasdata > 0, 1, 0)) snowd = nc.variables['snowd_12z'][sidx:, :, :] for i in range(snowd.shape[0]): rows.append({ 'valid': sts + datetime.timedelta(days=i), 'coverage': f(st, snowd[i], metric, stpts), }) eidx = iemre.daily_offset(ets) with ncopen(iemre.get_daily_ncname(ets.year)) as nc: snowd = nc.variables['snowd_12z'][:eidx, :, :] for i in range(snowd.shape[0]): rows.append({ 'valid': datetime.date(ets.year, 1, 1) + datetime.timedelta(days=i), 'coverage': f(st, snowd[i], metric, stpts), }) df = pd.DataFrame(rows) df = df[np.isfinite(df['coverage'])] (fig, ax) = plt.subplots(1, 1, sharex=True, figsize=(8, 6)) ax.bar(df['valid'].values, df['coverage'].values, fc='tan', ec='tan', align='center') ax.set_title(("IEM Estimated Areal Snow Coverage Percent of %s\n" " percentage of state reporting at least %.2fin snow" " cover") % (reference.state_names[state], thres)) ax.set_ylabel("Areal Coverage [%]") ax.xaxis.set_major_locator(mdates.DayLocator([1, 15])) ax.xaxis.set_major_formatter(mdates.DateFormatter("%-d %b\n%Y")) ax.set_yticks(range(0, 101, 25)) ax.grid(True) return fig, df
def append_cfs(res, lon, lat): """Append on needed CFS data.""" gridx, gridy = find_ij(lon, lat) lastyear = max(res["data"].keys()) thisyear = datetime.date.today().year lastdate = datetime.date(thisyear, 8, 31) if lastyear != thisyear: # We don't have any data yet for this year, so we add some res["data"][thisyear] = {"dates": [], "high": [], "low": [], "rh": []} else: # shrug if res["data"][lastyear]["dates"]: lastdate = datetime.datetime.strptime( res["data"][thisyear]["dates"][-1], "%Y-%m-%d" ).date() # go find the most recent CFS 0z file valid = datetime.date.today() attempt = 0 while True: testfn = valid.strftime("/mesonet/data/iemre/cfs_%Y%m%d00.nc") if os.path.isfile(testfn): break valid -= datetime.timedelta(hours=24) attempt += 1 if attempt > 9: return None try: nc = ncopen(testfn, timeout=NCOPEN_TIMEOUT) except Exception as exp: LOG.error(exp) return None if nc is None: LOG.debug("Failing %s as nc is None", testfn) return None high = ( masked_array(nc.variables["high_tmpk"][:, gridy, gridx], units.degK) .to(units.degF) .m ) low = ( masked_array(nc.variables["low_tmpk"][:, gridy, gridx], units.degK) .to(units.degF) .m ) # RH hack # found ~20% bias with this value, so arb addition for now rh = ( relative_humidity_from_dewpoint( masked_array(high, units.degF), masked_array(low, units.degF) ).m * 100.0 + 20.0 ) rh = np.where(rh > 95, 95, rh) entry = res["data"][thisyear] # lastdate is either August 31 or a date after, so our first forecast # date is i+1 tidx = daily_offset(lastdate + datetime.timedelta(days=1)) for i in range(tidx, 365): lts = datetime.date(thisyear, 1, 1) + datetime.timedelta(days=i) if lts.month in [9, 10, 11]: entry["dates"].append(lts.strftime("%Y-%m-%d")) entry["high"].append(_i(high[i])) entry["low"].append(_i(low[i])) entry["rh"].append(_i(rh[i])) return res
def init_year(ts): """ Create a new NetCDF file for a year of our specification! """ fp = iemre.get_dailyc_ncname() nc = ncopen(fp, "w") nc.title = "IEM Daily Reanalysis Climatology %s" % (ts.year, ) nc.platform = "Grided Climatology" nc.description = "IEM daily analysis on a 0.125 degree grid" nc.institution = "Iowa State University, Ames, IA, USA" nc.source = "Iowa Environmental Mesonet" nc.project_id = "IEM" nc.realization = 1 nc.Conventions = "CF-1.0" # *cough* nc.contact = "Daryl Herzmann, [email protected], 515-294-5978" nc.history = "%s Generated" % ( datetime.datetime.now().strftime("%d %B %Y"), ) nc.comment = "No Comment at this time" # Setup Dimensions nc.createDimension("lat", iemre.NY) nc.createDimension("lon", iemre.NX) ts2 = datetime.datetime(ts.year + 1, 1, 1) days = (ts2 - ts).days nc.createDimension("time", int(days)) # Setup Coordinate Variables lat = nc.createVariable("lat", np.float, ("lat", )) lat.units = "degrees_north" lat.long_name = "Latitude" lat.standard_name = "latitude" lat.axis = "Y" lat[:] = iemre.YAXIS lon = nc.createVariable("lon", np.float, ("lon", )) lon.units = "degrees_east" lon.long_name = "Longitude" lon.standard_name = "longitude" lon.axis = "X" lon[:] = iemre.XAXIS tm = nc.createVariable("time", np.float, ("time", )) tm.units = "Days since %s-01-01 00:00:0.0" % (ts.year, ) tm.long_name = "Time" tm.standard_name = "time" tm.axis = "T" tm.calendar = "gregorian" tm[:] = np.arange(0, int(days)) # Tracked variables hasdata = nc.createVariable("hasdata", np.int8, ("lat", "lon")) hasdata.units = "1" hasdata.long_name = "Analysis Available for Grid Cell" hasdata.coordinates = "lon lat" hasdata[:] = 0 high = nc.createVariable("high_tmpk", np.float, ("time", "lat", "lon"), fill_value=1.0e20) high.units = "K" high.long_name = "2m Air Temperature Daily High" high.standard_name = "2m Air Temperature" high.coordinates = "lon lat" low = nc.createVariable("low_tmpk", np.float, ("time", "lat", "lon"), fill_value=1.0e20) low.units = "K" low.long_name = "2m Air Temperature Daily Low" low.standard_name = "2m Air Temperature" low.coordinates = "lon lat" p01d = nc.createVariable("p01d", np.float, ("time", "lat", "lon"), fill_value=1.0e20) p01d.units = "mm" p01d.long_name = "Precipitation" p01d.standard_name = "Precipitation" p01d.coordinates = "lon lat" p01d.description = "Precipitation accumulation for the day" nc.close()
def plotter(fdict): """ Go """ ctx = util.get_autoplot_context(fdict, get_description()) ptype = ctx["ptype"] sdate = ctx["sdate"] edate = ctx["edate"] src = ctx["src"] opt = ctx["opt"] usdm = ctx["usdm"] if sdate.year != edate.year: raise NoDataFound("Sorry, do not support multi-year plots yet!") days = (edate - sdate).days sector = ctx["sector"] if sdate == edate: title = sdate.strftime("%-d %B %Y") else: title = "%s to %s (inclusive)" % ( sdate.strftime("%-d %b"), edate.strftime("%-d %b %Y"), ) x0 = 0 x1 = -1 y0 = 0 y1 = -1 state = None if len(sector) == 2: state = sector sector = "state" if src == "mrms": ncfn = iemre.get_daily_mrms_ncname(sdate.year) clncfn = iemre.get_dailyc_mrms_ncname() ncvar = "p01d" source = "MRMS Q3" subtitle = "NOAA MRMS Project, GaugeCorr and RadarOnly" elif src == "iemre": ncfn = iemre.get_daily_ncname(sdate.year) clncfn = iemre.get_dailyc_ncname() ncvar = "p01d_12z" source = "IEM Reanalysis" subtitle = "IEM Reanalysis is derived from various NOAA datasets" else: ncfn = "/mesonet/data/prism/%s_daily.nc" % (sdate.year,) clncfn = "/mesonet/data/prism/prism_dailyc.nc" ncvar = "ppt" source = "OSU PRISM" subtitle = ( "PRISM Climate Group, Oregon State Univ., " "http://prism.oregonstate.edu, created 4 Feb 2004." ) mp = MapPlot( sector=sector, state=state, axisbg="white", nocaption=True, title="%s:: %s Precip %s" % (source, title, PDICT3[opt]), subtitle="Data from %s" % (subtitle,), titlefontsize=14, ) idx0 = iemre.daily_offset(sdate) idx1 = iemre.daily_offset(edate) + 1 if not os.path.isfile(ncfn): raise NoDataFound("No data for that year, sorry.") with util.ncopen(ncfn) as nc: if state is not None: x0, y0, x1, y1 = util.grid_bounds( nc.variables["lon"][:], nc.variables["lat"][:], state_bounds[state], ) elif sector in SECTORS: bnds = SECTORS[sector] x0, y0, x1, y1 = util.grid_bounds( nc.variables["lon"][:], nc.variables["lat"][:], [bnds[0], bnds[2], bnds[1], bnds[3]], ) lats = nc.variables["lat"][y0:y1] lons = nc.variables["lon"][x0:x1] if sdate == edate: p01d = mm2inch(nc.variables[ncvar][idx0, y0:y1, x0:x1]) elif (idx1 - idx0) < 32: p01d = mm2inch( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0) ) else: # Too much data can overwhelm this app, need to chunk it for i in range(idx0, idx1, 10): i2 = min([i + 10, idx1]) if idx0 == i: p01d = mm2inch( np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0) ) else: p01d += mm2inch( np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0) ) if np.ma.is_masked(np.max(p01d)): raise NoDataFound("Data Unavailable") plot_units = "inches" cmap = plt.get_cmap(ctx["cmap"]) cmap.set_bad("white") if opt == "dep": # Do departure work now with util.ncopen(clncfn) as nc: climo = mm2inch( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0) ) p01d = p01d - climo [maxv] = np.percentile(np.abs(p01d), [99]) clevs = np.around(np.linspace(0 - maxv, maxv, 11), decimals=2) elif opt == "per": with util.ncopen(clncfn) as nc: climo = mm2inch( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0) ) p01d = p01d / climo * 100.0 cmap.set_under("white") cmap.set_over("black") clevs = [1, 10, 25, 50, 75, 100, 125, 150, 200, 300, 500] plot_units = "percent" else: p01d = np.where(p01d < 0.001, np.nan, p01d) cmap.set_under("white") clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10] if days > 6: clevs = [0.01, 0.3, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 10, 15, 20] if days > 29: clevs = [0.01, 0.5, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35] if days > 90: clevs = [0.01, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35, 40] x2d, y2d = np.meshgrid(lons, lats) if ptype == "c": mp.contourf( x2d, y2d, p01d, clevs, cmap=cmap, units=plot_units, iline=False ) else: res = mp.pcolormesh(x2d, y2d, p01d, clevs, cmap=cmap, units=plot_units) res.set_rasterized(True) if sector != "midwest": mp.drawcounties() mp.drawcities() if usdm == "yes": mp.draw_usdm(edate, filled=False, hatched=True) return mp.fig
def main(): """GO Main Go""" pgconn = get_dbconn('iem') icursor = pgconn.cursor() utcnow = datetime.datetime.utcnow() for i in range(10): now = utcnow - datetime.timedelta(hours=i) fn = "/mesonet/data/madis/mesonet1/%s.nc" % ( now.strftime("%Y%m%d_%H00"), ) if os.path.isfile(fn): break if not os.path.isfile(fn): print('extract_madis.py found no files? last: %s' % (fn, )) return with ncopen(fn) as nc: providers = chartostring(nc.variables["dataProvider"][:]) stations = chartostring(nc.variables['stationId'][:]) nc_tmpk = nc.variables["temperature"][:] nc_dwpk = nc.variables["dewpoint"][:] nc_alti = nc.variables["altimeter"][:] tmpkqcd = nc.variables["temperatureQCD"][:] dwpkqcd = nc.variables["dewpointQCD"][:] altiqcd = nc.variables["altimeterQCD"][:] times = nc.variables["observationTime"][:] for p in range(providers.shape[0]): provider = providers[p] if provider not in ['IEM', 'IADOT']: continue ticks = int(times[p]) ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ticks) ts = ts.replace(tzinfo=pytz.UTC) (tmpf, tmpf_qc_av, tmpf_qc_sc) = (None, None, None) (dwpf, dwpf_qc_av, dwpf_qc_sc) = (None, None, None) (alti, alti_qc_av, alti_qc_sc) = (None, None, None) if not np.ma.is_masked(nc_tmpk[p]): tmpf = check(temperature(nc_tmpk[p], 'K').value('F')) tmpf_qc_av = figure(nc_tmpk[p], tmpkqcd[p, 0]) tmpf_qc_sc = figure(nc_tmpk[p], tmpkqcd[p, 6]) if not np.ma.is_masked(nc_dwpk[p]): dwpf = check(temperature(nc_dwpk[p], 'K').value('F')) dwpf_qc_av = figure(nc_dwpk[p], dwpkqcd[p, 0]) dwpf_qc_sc = figure(nc_dwpk[p], dwpkqcd[p, 6]) if not np.ma.is_masked(nc_alti[p]): alti = check((nc_alti[p] / 100.0) * 0.0295298) alti_qc_av = figure_alti(altiqcd[p, 0] * 0.0295298) alti_qc_sc = figure_alti(altiqcd[p, 6] * 0.0295298) sql = """ UPDATE current_qc SET tmpf = %s, tmpf_qc_av = %s, tmpf_qc_sc = %s, dwpf = %s, dwpf_qc_av = %s, dwpf_qc_sc = %s, alti = %s, alti_qc_av = %s, alti_qc_sc = %s, valid = %s WHERE station = %s """ args = (tmpf, tmpf_qc_av, tmpf_qc_sc, dwpf, dwpf_qc_av, dwpf_qc_sc, alti, alti_qc_av, alti_qc_sc, ts, stations[p]) icursor.execute(sql, args) icursor.close() pgconn.commit() pgconn.close()
def plotter(fdict): """ Go """ import matplotlib matplotlib.use('agg') import matplotlib.pyplot as plt import matplotlib.dates as mdates ctx = get_autoplot_context(fdict, get_description()) year = ctx['year'] threshold = ctx['threshold'] period = ctx['period'] state = ctx['state'] pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr = %s """, pgconn, params=(state, ), index_col='state_abbr', geom_col='the_geom') nc = ncopen(iemre.get_daily_ncname(year)) precip = nc.variables['p01d'] czs = CachingZonalStats(iemre.AFFINE) hasdata = np.zeros((nc.dimensions['lat'].size, nc.dimensions['lon'].size)) czs.gen_stats(hasdata, states['the_geom']) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice, islice]) hasdata = np.flipud(hasdata) datapts = np.sum(np.where(hasdata > 0, 1, 0)) now = datetime.date(year, 1, 1) now += datetime.timedelta(days=(period-1)) ets = datetime.date(year, 12, 31) today = datetime.date.today() if ets > today: ets = today days = [] coverage = [] while now <= ets: idx = iemre.daily_offset(now) sevenday = np.sum(precip[(idx-period):idx, :, :], 0) pday = np.where(hasdata > 0, sevenday[:, :], -1) tots = np.sum(np.where(pday >= (threshold * 25.4), 1, 0)) days.append(now) coverage.append(tots / float(datapts) * 100.0) now += datetime.timedelta(days=1) df = pd.DataFrame(dict(day=pd.Series(days), coverage=pd.Series(coverage))) (fig, ax) = plt.subplots(1, 1) ax.bar(days, coverage, fc='g', ec='g') ax.set_title(("IEM Estimated Areal Coverage Percent of %s\n" " receiving %.2f inches of rain over trailing %s day period" ) % (reference.state_names[state], threshold, period)) ax.set_ylabel("Areal Coverage [%]") ax.xaxis.set_major_formatter(mdates.DateFormatter('%b\n%Y')) ax.set_yticks(range(0, 101, 25)) ax.grid(True) return fig, df
def main(): """Do Something Fun!""" form = cgi.FieldStorage() ts = datetime.datetime.strptime(form.getfirst("date", "2019-03-01"), "%Y-%m-%d") lat = float(form.getfirst("lat", 41.99)) lon = float(form.getfirst("lon", -95.1)) fmt = form.getfirst("format", "json") if fmt != 'json': ssw("Content-type: text/plain\n\n") ssw("ERROR: Service only emits json at this time") return i, j = iemre.find_ij(lon, lat) offset = iemre.daily_offset(ts) res = { 'data': [], } fn = iemre.get_daily_ncname(ts.year) ssw('Content-type: application/json\n\n') if not os.path.isfile(fn): ssw(json.dumps(res)) sys.exit() if i is None or j is None: ssw(json.dumps({'error': 'Coordinates outside of domain'})) return if ts.year > 1980: ncfn = "/mesonet/data/prism/%s_daily.nc" % (ts.year, ) if not os.path.isfile(ncfn): prism_precip = None else: i2, j2 = prismutil.find_ij(lon, lat) with ncopen(ncfn) as nc: prism_precip = nc.variables['ppt'][offset, j2, i2] / 25.4 else: prism_precip = None if ts.year > 2010: ncfn = iemre.get_daily_mrms_ncname(ts.year) if not os.path.isfile(ncfn): mrms_precip = None else: j2 = int((lat - iemre.SOUTH) * 100.0) i2 = int((lon - iemre.WEST) * 100.0) with ncopen(ncfn) as nc: mrms_precip = nc.variables['p01d'][offset, j2, i2] / 25.4 else: mrms_precip = None c2000 = ts.replace(year=2000) coffset = iemre.daily_offset(c2000) with ncopen(fn) as nc: with ncopen(iemre.get_dailyc_ncname()) as cnc: res['data'].append({ 'prism_precip_in': myrounder(prism_precip, 2), 'mrms_precip_in': myrounder(mrms_precip, 2), 'daily_high_f': myrounder( datatypes.temperature( nc.variables['high_tmpk'][offset, j, i], 'K').value('F'), 1), '12z_high_f': myrounder( datatypes.temperature( nc.variables['high_tmpk_12z'][offset, j, i], 'K').value('F'), 1), 'climate_daily_high_f': myrounder( datatypes.temperature( cnc.variables['high_tmpk'][coffset, j, i], 'K').value("F"), 1), 'daily_low_f': myrounder( datatypes.temperature( nc.variables['low_tmpk'][offset, j, i], 'K').value("F"), 1), '12z_low_f': myrounder( datatypes.temperature( nc.variables['low_tmpk_12z'][offset, j, i], 'K').value('F'), 1), 'avg_dewpoint_f': myrounder( datatypes.temperature( nc.variables['avg_dwpk'][offset, j, i], 'K').value('F'), 1), 'climate_daily_low_f': myrounder( datatypes.temperature( cnc.variables['low_tmpk'][coffset, j, i], 'K').value("F"), 1), 'daily_precip_in': myrounder(nc.variables['p01d'][offset, j, i] / 25.4, 2), '12z_precip_in': myrounder(nc.variables['p01d_12z'][offset, j, i] / 25.4, 2), 'climate_daily_precip_in': myrounder(cnc.variables['p01d'][coffset, j, i] / 25.4, 2), 'srad_mj': myrounder( nc.variables['rsds'][offset, j, i] * 86400. / 1000000., 2), 'avg_windspeed_mps': myrounder(nc.variables['wind_speed'][offset, j, i], 2), }) ssw(json.dumps(res))
def plotter(fdict): """ Go """ ctx = util.get_autoplot_context(fdict, get_description()) date = ctx['date'] sector = ctx['sector'] threshold = ctx['threshold'] threshold_mm = distance(threshold, 'IN').value('MM') window_sts = date - datetime.timedelta(days=90) if window_sts.year != date.year: raise ValueError('Sorry, do not support multi-year plots yet!') # idx0 = iemre.daily_offset(window_sts) idx1 = iemre.daily_offset(date) ncfn = iemre.get_daily_mrms_ncname(date.year) ncvar = 'p01d' nc = util.ncopen(ncfn) if nc is None: raise ValueError("No data for that year, sorry.") # Get the state weight df = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom from states where state_abbr = %s """, util.get_dbconn('postgis'), params=(sector, ), index_col=None, geom_col='the_geom') czs = CachingZonalStats(iemre.MRMS_AFFINE) czs.gen_stats( np.zeros((nc.variables['lat'].size, nc.variables['lon'].size)), df['the_geom']) jslice = None islice = None for nav in czs.gridnav: # careful here as y is flipped in this context jslice = slice(nc.variables['lat'].size - (nav.y0 + nav.ysz), nc.variables['lat'].size - nav.y0) islice = slice(nav.x0, nav.x0 + nav.xsz) grid = np.zeros((jslice.stop - jslice.start, islice.stop - islice.start)) total = np.zeros((jslice.stop - jslice.start, islice.stop - islice.start)) for i, idx in enumerate(range(idx1, idx1 - 90, -1)): total += nc.variables[ncvar][idx, jslice, islice] grid = np.where(np.logical_and(grid == 0, total > threshold_mm), i, grid) lon = nc.variables['lon'][islice] lat = nc.variables['lat'][jslice] nc.close() mp = MapPlot(sector='state', state=sector, titlefontsize=14, subtitlefontsize=12, title=("NOAA MRMS Q3: Number of Recent Days " "till Accumulating %s\" of Precip") % (threshold, ), subtitle=("valid %s: based on per calendar day " "estimated preciptation, GaugeCorr and " "RadarOnly products") % (date.strftime("%-d %b %Y"), )) x, y = np.meshgrid(lon, lat) cmap = plt.get_cmap(ctx['cmap']) cmap.set_over('k') cmap.set_under('white') mp.pcolormesh(x, y, grid, np.arange(0, 81, 10), cmap=cmap, units='days') mp.drawcounties() mp.drawcities() return mp.fig
def application(environ, start_response): """Go main go""" os.chdir("/tmp") form = parse_formvars(environ) ts0 = datetime.datetime.strptime(form.get("date0"), "%Y-%m-%d") ts1 = datetime.datetime.strptime(form.get("date1"), "%Y-%m-%d") base = int(form.get("base", 50)) ceil = int(form.get("ceil", 86)) # Make sure we aren't in the future tsend = datetime.date.today() if ts1.date() >= tsend: ts1 = tsend - datetime.timedelta(days=1) ts1 = datetime.datetime(ts1.year, ts1.month, ts1.day) fmt = form.get("format") offset0 = iemre.daily_offset(ts0) offset1 = iemre.daily_offset(ts1) with ncopen(iemre.get_daily_ncname(ts0.year)) as nc: # 2-D precipitation, inches precip = np.sum(nc.variables["p01d"][offset0:offset1, :, :] / 25.4, axis=0) # GDD H = datatypes.temperature(nc.variables["high_tmpk"][offset0:offset1], "K").value("F") H = np.where(H < base, base, H) H = np.where(H > ceil, ceil, H) L = datatypes.temperature(nc.variables["low_tmpk"][offset0:offset1], "K").value("F") L = np.where(L < base, base, L) gdd = np.sum((H + L) / 2.0 - base, axis=0) if fmt == "json": # For example: 19013 ugc = "IAC" + form.get("county")[2:] # Go figure out where this is! postgis = get_dbconn("postgis") pcursor = postgis.cursor() pcursor.execute( """ SELECT ST_x(ST_Centroid(geom)), ST_y(ST_Centroid(geom)) from ugcs WHERE ugc = %s and end_ts is null """, (ugc, ), ) row = pcursor.fetchone() lat = row[1] lon = row[0] (i, j) = iemre.find_ij(lon, lat) myGDD = gdd[j, i] myPrecip = precip[j, i] res = {"data": []} res["data"].append({ "gdd": "%.0f" % (myGDD, ), "precip": "%.1f" % (myPrecip, ), "latitude": "%.4f" % (lat, ), "longitude": "%.4f" % (lon, ), }) headers = [("Content-type", "application/json")] start_response("200 OK", headers) return [json.dumps(res).encode("ascii")] # Time to create the shapefiles basefn = "iemre_%s_%s" % (ts0.strftime("%Y%m%d"), ts1.strftime("%Y%m")) w = shapefile.Writer(basefn) w.field("GDD", "F", 10, 2) w.field("PREC_IN", "F", 10, 2) for x in iemre.XAXIS: for y in iemre.YAXIS: w.poly([[ (x, y), (x, y + iemre.DY), (x + iemre.DX, y + iemre.DY), (x + iemre.DX, y), (x, y), ]]) for i in range(len(iemre.XAXIS)): for j in range(len(iemre.YAXIS)): w.record(gdd[j, i], precip[j, i]) w.close() # Create zip file, send it back to the clients shutil.copyfile("/opt/iem/data/gis/meta/4326.prj", "%s.prj" % (basefn, )) z = zipfile.ZipFile("%s.zip" % (basefn, ), "w", zipfile.ZIP_DEFLATED) for suffix in ["shp", "shx", "dbf", "prj"]: z.write("%s.%s" % (basefn, suffix)) z.close() headers = [ ("Content-type", "application/octet-stream"), ("Content-Disposition", "attachment; filename=%s.zip" % (basefn, )), ] start_response("200 OK", headers) content = open(basefn + ".zip", "rb").read() for suffix in ["zip", "shp", "shx", "dbf", "prj"]: os.unlink("%s.%s" % (basefn, suffix)) return [content]
def main(): """Go Main Go""" pgconn = get_dbconn("coop") ccursor = pgconn.cursor() fn = sys.argv[1] station = sys.argv[2] with ncopen(fn) as nc: byear = nc.variables["byear"][:] maxt = nc.variables["maxt"][:] mint = nc.variables["mint"][:] pcpn = nc.variables["pcpn"][:] snow = nc.variables["snow"][:] snwg = nc.variables["snwg"][:] current = read_sql( """ SELECT day, high, low, precip, snow, snowd from alldata_ia WHERE station = %s ORDER by day ASC """, pgconn, params=(station, ), index_col="day", ) added = 0 for yr in range(byear, 2016): for mo in range(12): for dy in range(31): try: date = datetime.date(yr, mo + 1, dy + 1) except ValueError: continue high = maxt[yr - byear, mo, dy] if (np.ma.is_masked(high) or np.isnan(high) or high < -100 or high > 150): high = None else: high = int(high) low = mint[yr - byear, mo, dy] if (np.ma.is_masked(low) or np.isnan(low) or low < -100 or low > 150): low = None else: low = int(low) precip = convert(pcpn[yr - byear, mo, dy], 2) snowfall = convert(snow[yr - byear, mo, dy], 1) snowd = convert(snwg[yr - byear, mo, dy], 1) if all( [a is None for a in [high, low, precip, snowfall, snowd]]): continue if date not in current.index.values: sday = "%02i%02i" % (date.month, date.day) added += 1 ccursor.execute( """ INSERT into alldata_ia(station, day, high, low, precip, snow, sday, year, month, snowd) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """, ( station, date, high, low, precip, snowfall, sday, int(date.year), int(date.month), snowd, ), ) print("added %s" % (added, )) ccursor.close() pgconn.commit()
def init_year(ts): """ Create a new NetCDF file for a year of our specification! """ # Load up the example grib file to base our file on grbs = pygrib.open(TEMPLATE_FN) grb = grbs[1] # grid shape is y, x lats, lons = grb.latlons() fp = "%s/%s_stage4_hourly.nc" % (BASEDIR, ts.year) nc = ncopen(fp, "w") nc.title = "IEM Packaged NOAA Stage IV for %s" % (ts.year, ) nc.platform = "Grided Estimates" nc.description = "NOAA Stage IV on HRAP Grid" nc.institution = "Iowa State University, Ames, IA, USA" nc.source = "Iowa Environmental Mesonet" nc.project_id = "IEM" nc.realization = 1 nc.Conventions = "CF-1.0" nc.contact = "Daryl Herzmann, [email protected], 515-294-5978" nc.history = ("%s Generated") % ( datetime.datetime.now().strftime("%d %B %Y"), ) nc.comment = "No Comment at this time" # Setup Dimensions nc.createDimension("x", lats.shape[1]) nc.createDimension("y", lats.shape[0]) nc.createDimension("bnds", 2) ts2 = datetime.datetime(ts.year + 1, 1, 1) days = (ts2 - ts).days print("Year %s has %s days" % (ts.year, days)) nc.createDimension("time", int(days) * 24) # Setup Coordinate Variables lat = nc.createVariable("lat", np.float, ("y", "x")) lat.units = "degrees_north" lat.long_name = "Latitude" lat.standard_name = "latitude" lat.axis = "Y" lat[:] = lats lon = nc.createVariable("lon", np.float, ("y", "x")) lon.units = "degrees_east" lon.long_name = "Longitude" lon.standard_name = "longitude" lon.axis = "X" lon[:] = lons tm = nc.createVariable("time", np.float, ("time", )) tm.units = "Hours since %s-01-01 00:00:0.0" % (ts.year, ) tm.long_name = "Time" tm.standard_name = "time" tm.axis = "T" tm.calendar = "gregorian" tm[:] = np.arange(0, int(days) * 24) tm.bounds = "time_bnds" tmb = nc.createVariable("time_bnds", "d", ("time", "bnds")) tmb[:, 0] = np.arange(0, int(days) * 24) - 1 tmb[:, 1] = np.arange(0, int(days) * 24) p01m = nc.createVariable("p01m", np.ushort, ("time", "y", "x"), fill_value=65535) p01m.scale_factor = 0.01 p01m.add_offset = 0.0 p01m.units = "mm" p01m.long_name = "Precipitation" p01m.standard_name = "Precipitation" p01m.coordinates = "lon lat" p01m.description = "Precipitation accumulation for the previous hour" # Track variable status to prevent double writes of data, prior to bias # correction making some changes status = nc.createVariable("p01m_status", np.int8, ("time", ), fill_value=-1) status.units = "1" status.long_name = "p01m Variable Status" status.description = "-1 unset, 1 grib data copied, 2 QC Run" nc.close()
def process(df, ncfn, south, west): """Do some extraction.""" fn1 = "/mesonet/share/pickup/yieldfx/baseline/%s" % (ncfn, ) fn2 = "/mesonet/share/pickup/yieldfx/2019/%s" % (ncfn, ) if not os.path.isfile(fn1) or not os.path.isfile(fn2): print("Missing %s or %s" % (fn1, fn2)) return with ncopen(fn1) as nc: with ncopen(fn2) as nc2019: for _, row in df.iterrows(): if row["State"] not in [ "nd", "sd", "ne", "ks", "mo", "ia", "mn", "wi", "il", "in", "ky", "oh", "mi", ]: continue i = int((row["long"] - west) / 0.1250) j = int((row["lat"] - south) / 0.1250) nc_prcp = nc.variables["prcp"][:, j, i] nc_tmax = nc.variables["tmax"][:, j, i] nc_tmin = nc.variables["tmin"][:, j, i] nc_srad = nc.variables["srad"][:, j, i] nc2019_prcp = nc2019.variables["prcp"][:, j, i] nc2019_tmax = nc2019.variables["tmax"][:, j, i] nc2019_tmin = nc2019.variables["tmin"][:, j, i] nc2019_srad = nc2019.variables["srad"][:, j, i] fp = open( ("/mesonet/share/pickup/yieldfx/county/" "%s_%s_%.4f_%.4f.csv") % ( row["State"].replace(" ", "_"), row["County"].replace(" ", "_"), 0 - row["long"], row["lat"], ), "w", ) fp.write(("year,yday,prcp (mm/day),srad (W/m^2),tmax (deg c)," "tmin (deg c)\n")) base = datetime.date(1980, 1, 1) for tstep, days in enumerate(nc.variables["time"]): ts = base + datetime.timedelta(days=int(days)) fp.write("%s,%s,%.3f,%.3f,%.3f,%.3f\n" % ( ts.year, int(ts.strftime("%j")), nc_prcp[tstep], nc_srad[tstep] * 1e6 / 86400.0, nc_tmax[tstep], nc_tmin[tstep], )) base = datetime.date(2019, 1, 1) for tstep, days in enumerate(nc2019.variables["time"]): ts = base + datetime.timedelta(days=int(days)) fp.write("%s,%s,%.3f,%.3f,%.3f,%.3f\n" % ( ts.year, int(ts.strftime("%j")), nc2019_prcp[tstep], nc2019_srad[tstep] * 1e6 / 86400.0, nc2019_tmax[tstep], nc2019_tmin[tstep], )) fp.close()
def init_year(ts): """ Create a new NetCDF file for a year of our specification! """ fn = iemre.get_daily_ncname(ts.year) nc = ncopen(fn, 'w') nc.title = "IEM Daily Reanalysis %s" % (ts.year, ) nc.platform = "Grided Observations" nc.description = "IEM daily analysis on a 0.125 degree grid" nc.institution = "Iowa State University, Ames, IA, USA" nc.source = "Iowa Environmental Mesonet" nc.project_id = "IEM" nc.realization = 1 nc.Conventions = 'CF-1.0' nc.contact = "Daryl Herzmann, [email protected], 515-294-5978" nc.history = ("%s Generated") % ( datetime.datetime.now().strftime("%d %B %Y"), ) nc.comment = "No Comment at this time" # Setup Dimensions nc.createDimension('lat', iemre.NY) nc.createDimension('lon', iemre.NX) days = ((ts.replace(year=ts.year + 1)) - ts).days nc.createDimension('time', int(days)) # Setup Coordinate Variables lat = nc.createVariable('lat', np.float, ('lat', )) lat.units = "degrees_north" lat.long_name = "Latitude" lat.standard_name = "latitude" lat.axis = "Y" lat[:] = iemre.YAXIS lon = nc.createVariable('lon', np.float, ('lon', )) lon.units = "degrees_east" lon.long_name = "Longitude" lon.standard_name = "longitude" lon.axis = "X" lon[:] = iemre.XAXIS tm = nc.createVariable('time', np.float, ('time', )) tm.units = "Days since %s-01-01 00:00:0.0" % (ts.year, ) tm.long_name = "Time" tm.standard_name = "time" tm.axis = "T" tm.calendar = "gregorian" tm[:] = np.arange(0, int(days)) # Tracked variables hasdata = nc.createVariable('hasdata', np.int8, ('lat', 'lon')) hasdata.units = '1' hasdata.long_name = 'Analysis Available for Grid Cell' hasdata.coordinates = "lon lat" hasdata[:] = 0 high = nc.createVariable('high_tmpk', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) high.units = "K" high.scale_factor = 0.01 high.long_name = "2m Air Temperature Daily High" high.standard_name = "2m Air Temperature" high.coordinates = "lon lat" low = nc.createVariable('low_tmpk', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) low.units = "K" low.scale_factor = 0.01 low.long_name = "2m Air Temperature Daily Low" low.standard_name = "2m Air Temperature" low.coordinates = "lon lat" high12 = nc.createVariable('high_tmpk_12z', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) high12.units = "K" high12.scale_factor = 0.01 high12.long_name = "2m Air Temperature 24 Hour Max at 12 UTC" high12.standard_name = "2m Air Temperature" high12.coordinates = "lon lat" low12 = nc.createVariable('low_tmpk_12z', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) low12.units = "K" low12.scale_factor = 0.01 low12.long_name = "2m Air Temperature 12 Hour Min at 12 UTC" low12.standard_name = "2m Air Temperature" low12.coordinates = "lon lat" p01d = nc.createVariable('p01d', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) p01d.units = 'mm' p01d.scale_factor = 0.01 p01d.long_name = 'Precipitation' p01d.standard_name = 'Precipitation' p01d.coordinates = "lon lat" p01d.description = "Precipitation accumulation for the day" p01d12 = nc.createVariable('p01d_12z', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) p01d12.units = 'mm' p01d12.scale_factor = 0.01 p01d12.long_name = 'Precipitation' p01d12.standard_name = 'Precipitation' p01d12.coordinates = "lon lat" p01d12.description = "24 Hour Precipitation Ending 12 UTC" # 0 -> 65535 so 0 to 6553.5 rsds = nc.createVariable('rsds', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) rsds.units = "W m-2" rsds.scale_factor = 0.1 rsds.long_name = 'surface_downwelling_shortwave_flux_in_air' rsds.standard_name = 'surface_downwelling_shortwave_flux_in_air' rsds.coordinates = "lon lat" rsds.description = "Global Shortwave Irradiance" snow = nc.createVariable('snow_12z', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) snow.units = 'mm' snow.scale_factor = 0.01 snow.long_name = 'Snowfall' snow.standard_name = 'Snowfall' snow.coordinates = "lon lat" snow.description = "Snowfall accumulation for the day" # 0 to 6553.5 snowd = nc.createVariable('snowd_12z', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) snowd.units = 'mm' snowd.scale_factor = 0.1 snowd.long_name = 'Snow Depth' snowd.standard_name = 'Snow Depth' snowd.coordinates = "lon lat" snowd.description = "Snow depth at time of observation" v1 = nc.createVariable('avg_dwpk', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) v1.units = 'K' v1.scale_factor = 0.01 v1.long_name = '2m Average Dew Point Temperature' v1.standard_name = 'Dewpoint' v1.coordinates = "lon lat" v1.description = "Dew Point average computed by averaging mixing ratios" v2 = nc.createVariable('wind_speed', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) v2.units = 'm s-1' v2.scale_factor = 0.001 v2.long_name = 'Wind Speed' v2.standard_name = 'Wind Speed' v2.coordinates = "lon lat" v2.description = "Daily averaged wind speed magnitude" nc.close()
def do_day(valid): """ Process a day please """ idx = iemre.daily_offset(valid) with ncopen(iemre.get_daily_ncname(valid.year), 'r', timeout=300) as nc: high = temperature(nc.variables['high_tmpk_12z'][idx, :, :], 'K').value('F') low = temperature(nc.variables['low_tmpk_12z'][idx, :, :], 'K').value('F') precip = distance(nc.variables['p01d_12z'][idx, :, :], 'MM').value("IN") snow = distance(nc.variables['snow_12z'][idx, :, :], 'MM').value("IN") snowd = distance(nc.variables['snowd_12z'][idx, :, :], 'MM').value("IN") # build out the state mappers pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr not in ('AK', 'HI', 'DC') """, pgconn, index_col='state_abbr', geom_col='the_geom') czs = CachingZonalStats(iemre.AFFINE) sthigh = czs.gen_stats(np.flipud(high), states['the_geom']) stlow = czs.gen_stats(np.flipud(low), states['the_geom']) stprecip = czs.gen_stats(np.flipud(precip), states['the_geom']) stsnow = czs.gen_stats(np.flipud(snow), states['the_geom']) stsnowd = czs.gen_stats(np.flipud(snowd), states['the_geom']) statedata = {} for i, state in enumerate(states.index.values): statedata[state] = dict(high=sthigh[i], low=stlow[i], precip=stprecip[i], snow=stsnow[i], snowd=stsnowd[i]) update_database(state + "0000", valid, statedata[state]) # build out climate division mappers climdiv = gpd.GeoDataFrame.from_postgis(""" SELECT geom, iemid from climdiv where st_abbrv not in ('AK', 'HI', 'DC') """, pgconn, index_col='iemid', geom_col='geom') czs = CachingZonalStats(iemre.AFFINE) sthigh = czs.gen_stats(np.flipud(high), climdiv['geom']) stlow = czs.gen_stats(np.flipud(low), climdiv['geom']) stprecip = czs.gen_stats(np.flipud(precip), climdiv['geom']) stsnow = czs.gen_stats(np.flipud(snow), climdiv['geom']) stsnowd = czs.gen_stats(np.flipud(snowd), climdiv['geom']) for i, iemid in enumerate(climdiv.index.values): row = dict(high=sthigh[i], low=stlow[i], precip=stprecip[i], snow=stsnow[i], snowd=stsnowd[i]) # we must have temperature data if row['high'] is np.ma.masked or row['low'] is np.ma.masked: print( ("compute_0000 %s has missing temperature data, using state") % (iemid, )) row = statedata[iemid[:2]] update_database(iemid, valid, row)
def init_year(ts): """ Create a new NetCDF file for a year of our specification! """ fn = iemre.get_hourly_ncname(ts.year) nc = ncopen(fn, 'w') nc.title = "IEM Hourly Reanalysis %s" % (ts.year, ) nc.platform = "Grided Observations" nc.description = "IEM hourly analysis on a 0.125 degree grid" nc.institution = "Iowa State University, Ames, IA, USA" nc.source = "Iowa Environmental Mesonet" nc.project_id = "IEM" nc.realization = 1 nc.Conventions = 'CF-1.0' nc.contact = "Daryl Herzmann, [email protected], 515-294-5978" nc.history = ("%s Generated") % ( datetime.datetime.now().strftime("%d %B %Y"), ) nc.comment = "No Comment at this time" # Setup Dimensions nc.createDimension('lat', iemre.NY) nc.createDimension('lon', iemre.NX) ts2 = datetime.datetime(ts.year + 1, 1, 1) days = (ts2 - ts).days print('Year %s has %s days' % (ts.year, days)) nc.createDimension('time', int(days) * 24) # Setup Coordinate Variables lat = nc.createVariable('lat', np.float, ('lat', )) lat.units = "degrees_north" lat.long_name = "Latitude" lat.standard_name = "latitude" lat.axis = "Y" lat[:] = iemre.YAXIS lon = nc.createVariable('lon', np.float, ('lon', )) lon.units = "degrees_east" lon.long_name = "Longitude" lon.standard_name = "longitude" lon.axis = "X" lon[:] = iemre.XAXIS tm = nc.createVariable('time', np.float, ('time', )) tm.units = "Hours since %s-01-01 00:00:0.0" % (ts.year, ) tm.long_name = "Time" tm.standard_name = "time" tm.axis = "T" tm.calendar = "gregorian" tm[:] = np.arange(0, int(days) * 24) # Tracked variables hasdata = nc.createVariable('hasdata', np.int8, ('lat', 'lon')) hasdata.units = '1' hasdata.long_name = 'Analysis Available for Grid Cell' hasdata.coordinates = "lon lat" hasdata[:] = 0 # can storage -128->127 actual values are 0 to 100 skyc = nc.createVariable('skyc', np.int8, ('time', 'lat', 'lon'), fill_value=-128) skyc.long_name = "ASOS Sky Coverage" skyc.stanard_name = "ASOS Sky Coverage" skyc.units = "%" skyc.valid_range = [0, 100] skyc.coordinates = "lon lat" # 0->65535 tmpk = nc.createVariable('tmpk', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) tmpk.units = "K" tmpk.scale_factor = 0.01 tmpk.long_name = "2m Air Temperature" tmpk.standard_name = "2m Air Temperature" tmpk.coordinates = "lon lat" # 0->65535 0 to 655.35 dwpk = nc.createVariable('dwpk', np.uint16, ('time', 'lat', 'lon'), fill_value=65335) dwpk.units = "K" dwpk.scale_factor = 0.01 dwpk.long_name = "2m Air Dew Point Temperature" dwpk.standard_name = "2m Air Dew Point Temperature" dwpk.coordinates = "lon lat" # 0->65535 0 to 65.535 uwnd = nc.createVariable('uwnd', np.uint16, ('time', 'lat', 'lon'), fill_value=65335) uwnd.units = "meters per second" uwnd.scale_factor = 0.001 uwnd.long_name = "U component of the wind" uwnd.standard_name = "U component of the wind" uwnd.coordinates = "lon lat" # 0->65535 0 to 65.535 vwnd = nc.createVariable('vwnd', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) vwnd.units = "meters per second" vwnd.scale_factor = 0.001 vwnd.long_name = "V component of the wind" vwnd.standard_name = "V component of the wind" vwnd.coordinates = "lon lat" # 0->65535 0 to 655.35 p01m = nc.createVariable('p01m', np.uint16, ('time', 'lat', 'lon'), fill_value=65535) p01m.units = 'mm' p01m.scale_factor = 0.01 p01m.long_name = 'Precipitation' p01m.standard_name = 'Precipitation' p01m.coordinates = "lon lat" p01m.description = "Precipitation accumulation for the hour valid time" nc.close()
def init_year(ts): """ Create a new NetCDF file for a year of our specification! """ fn = iemre.get_dailyc_mrms_ncname() nc = ncopen(fn, "w") nc.title = "IEM Daily Reanalysis Climatology %s" % (ts.year, ) nc.platform = "Grided Climatology" nc.description = "IEM daily analysis on a 0.01 degree grid" nc.institution = "Iowa State University, Ames, IA, USA" nc.source = "Iowa Environmental Mesonet" nc.project_id = "IEM" nc.realization = 1 nc.Conventions = "CF-1.0" # *cough* nc.contact = "Daryl Herzmann, [email protected], 515-294-5978" nc.history = "%s Generated" % ( datetime.datetime.now().strftime("%d %B %Y"), ) nc.comment = "No Comment at this time" # Setup Dimensions nc.createDimension("lat", (iemre.NORTH - iemre.SOUTH) * 100.0) nc.createDimension("lon", (iemre.EAST - iemre.WEST) * 100.0) ts2 = datetime.datetime(ts.year + 1, 1, 1) days = (ts2 - ts).days nc.createDimension("time", int(days)) nc.createDimension("nv", 2) # Setup Coordinate Variables lat = nc.createVariable("lat", np.float, ("lat", )) lat.units = "degrees_north" lat.long_name = "Latitude" lat.standard_name = "latitude" lat.bounds = "lat_bnds" lat.axis = "Y" # Grid centers lat[:] = np.arange(iemre.SOUTH + 0.005, iemre.NORTH, 0.01) lat_bnds = nc.createVariable("lat_bnds", np.float, ("lat", "nv")) lat_bnds[:, 0] = np.arange(iemre.SOUTH, iemre.NORTH, 0.01) lat_bnds[:, 1] = np.arange(iemre.SOUTH + 0.01, iemre.NORTH + 0.01, 0.01) lon = nc.createVariable("lon", np.float, ("lon", )) lon.units = "degrees_east" lon.long_name = "Longitude" lon.standard_name = "longitude" lon.bounds = "lon_bnds" lon.axis = "X" lon[:] = np.arange(iemre.WEST, iemre.EAST, 0.01) lon_bnds = nc.createVariable("lon_bnds", np.float, ("lon", "nv")) lon_bnds[:, 0] = np.arange(iemre.WEST, iemre.EAST, 0.01) lon_bnds[:, 1] = np.arange(iemre.WEST + 0.01, iemre.EAST + 0.01, 0.01) tm = nc.createVariable("time", np.float, ("time", )) tm.units = "Days since %s-01-01 00:00:0.0" % (ts.year, ) tm.long_name = "Time" tm.standard_name = "time" tm.axis = "T" tm.calendar = "gregorian" tm[:] = np.arange(0, int(days)) p01d = nc.createVariable("p01d", np.uint16, ("time", "lat", "lon"), fill_value=65535) p01d.units = "mm" p01d.scale_factor = 0.01 p01d.long_name = "Precipitation" p01d.standard_name = "Precipitation" p01d.coordinates = "lon lat" p01d.description = "Precipitation accumulation for the day" nc.close()
def plotter(fdict): """ Go """ ctx = util.get_autoplot_context(fdict, get_description()) ptype = ctx['ptype'] sdate = ctx['sdate'] edate = ctx['edate'] src = ctx['src'] opt = ctx['opt'] usdm = ctx['usdm'] if sdate.year != edate.year: raise ValueError('Sorry, do not support multi-year plots yet!') days = (edate - sdate).days sector = ctx['sector'] if sdate == edate: title = sdate.strftime("%-d %B %Y") else: title = "%s to %s (inclusive)" % (sdate.strftime("%-d %b"), edate.strftime("%-d %b %Y")) x0 = 0 x1 = -1 y0 = 0 y1 = -1 state = None if len(sector) == 2: state = sector sector = 'state' if src == 'mrms': ncfn = iemre.get_daily_mrms_ncname(sdate.year) clncfn = iemre.get_dailyc_mrms_ncname() ncvar = 'p01d' source = 'MRMS Q3' subtitle = 'NOAA MRMS Project, GaugeCorr and RadarOnly' elif src == 'iemre': ncfn = iemre.get_daily_ncname(sdate.year) clncfn = iemre.get_dailyc_ncname() ncvar = 'p01d_12z' source = 'IEM Reanalysis' subtitle = 'IEM Reanalysis is derived from various NOAA datasets' else: ncfn = "/mesonet/data/prism/%s_daily.nc" % (sdate.year, ) clncfn = "/mesonet/data/prism/prism_dailyc.nc" ncvar = 'ppt' source = 'OSU PRISM' subtitle = ('PRISM Climate Group, Oregon State Univ., ' 'http://prism.oregonstate.edu, created 4 Feb 2004.') mp = MapPlot(sector=sector, state=state, axisbg='white', nocaption=True, title='%s:: %s Precip %s' % (source, title, PDICT3[opt]), subtitle='Data from %s' % (subtitle, ), titlefontsize=14) idx0 = iemre.daily_offset(sdate) idx1 = iemre.daily_offset(edate) + 1 if not os.path.isfile(ncfn): raise ValueError("No data for that year, sorry.") with util.ncopen(ncfn) as nc: if state is not None: x0, y0, x1, y1 = util.grid_bounds(nc.variables['lon'][:], nc.variables['lat'][:], state_bounds[state]) elif sector in SECTORS: bnds = SECTORS[sector] x0, y0, x1, y1 = util.grid_bounds( nc.variables['lon'][:], nc.variables['lat'][:], [bnds[0], bnds[2], bnds[1], bnds[3]]) lats = nc.variables['lat'][y0:y1] lons = nc.variables['lon'][x0:x1] if (idx1 - idx0) < 32: p01d = distance( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0), 'MM').value('IN') else: # Too much data can overwhelm this app, need to chunk it for i in range(idx0, idx1, 10): i2 = min([i + 10, idx1]) if idx0 == i: p01d = distance( np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0), 'MM').value('IN') else: p01d += distance( np.sum(nc.variables[ncvar][i:i2, y0:y1, x0:x1], 0), 'MM').value('IN') if np.ma.is_masked(np.max(p01d)): raise ValueError("Data Unavailable") units = 'inches' cmap = plt.get_cmap(ctx['cmap']) cmap.set_bad('white') if opt == 'dep': # Do departure work now with util.ncopen(clncfn) as nc: climo = distance( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0), 'MM').value('IN') p01d = p01d - climo [maxv] = np.percentile(np.abs(p01d), [ 99, ]) clevs = np.around(np.linspace(0 - maxv, maxv, 11), decimals=2) elif opt == 'per': with util.ncopen(clncfn) as nc: climo = distance( np.sum(nc.variables[ncvar][idx0:idx1, y0:y1, x0:x1], 0), 'MM').value('IN') p01d = p01d / climo * 100. cmap.set_under('white') cmap.set_over('black') clevs = [1, 10, 25, 50, 75, 100, 125, 150, 200, 300, 500] units = 'percent' else: p01d = np.where(p01d < 0.001, np.nan, p01d) cmap.set_under('white') clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10] if days > 6: clevs = [0.01, 0.3, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 10, 15, 20] if days > 29: clevs = [0.01, 0.5, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35] if days > 90: clevs = [0.01, 1, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 35, 40] x2d, y2d = np.meshgrid(lons, lats) if ptype == 'c': mp.contourf(x2d, y2d, p01d, clevs, cmap=cmap, units=units, iline=False) else: res = mp.pcolormesh(x2d, y2d, p01d, clevs, cmap=cmap, units=units) res.set_rasterized(True) if sector != 'midwest': mp.drawcounties() mp.drawcities() if usdm == 'yes': mp.draw_usdm(edate, filled=False, hatched=True) return mp.fig
def main(argv): """Go Main Go""" network = argv[1] wxcfn = argv[2] utc = datetime.datetime.utcnow() utc = utc.replace(tzinfo=pytz.UTC) out = open(wxcfn, 'w') out.write("""Weather Central 001d0300 Surface Data TimeStamp=%s 12 5 Station 25 Station Name 8 Lat 10 Lon 2 Hour 2 Minute 5 Air Temperature F 5 Dew Point F 5 Wind Direction deg 5 Wind Speed mph 5 Heat Index F 5 Wind Chill F """ % (utc.strftime("%Y.%m.%d.%H%M"), )) fn = None for i in range(4): now = utc - datetime.timedelta(hours=i) testfn = now.strftime("/mesonet/data/madis/mesonet1/%Y%m%d_%H00.nc") if os.path.isfile(testfn): fn = testfn break if fn is None: sys.exit() indices = {} BOGUS = datetime.datetime(2000, 1, 1) BOGUS = BOGUS.replace(tzinfo=pytz.UTC) nc = ncopen(fn) providers = chartostring(nc.variables["dataProvider"][:]) stations = chartostring(nc.variables["stationId"][:]) names = chartostring(nc.variables["stationName"][:]) for i, provider in enumerate(providers): if provider != network: continue sid = stations[i] # We have an ob! ticks = int(nc.variables["observationTime"][i]) ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ticks) ts = ts.replace(tzinfo=pytz.UTC) if ts > indices.get(sid, {'ts': BOGUS})['ts']: indices[sid] = {'ts': ts, 'idx': i} for sid in indices: idx = indices[sid]['idx'] name = names[idx] latitude = nc.variables['latitude'][idx] longitude = nc.variables['longitude'][idx] tmpf = s(nc.variables['temperature'][idx]) dwpf = s(nc.variables['dewpoint'][idx]) qcd = nc.variables['temperatureQCD'][idx][0] if qcd < -10 or qcd > 10: tmpf = "M" dwpf = "M" heat = "M" if tmpf != "M" and dwpf != "M": t = temperature(nc.variables['temperature'][idx], 'K') d = temperature(nc.variables['dewpoint'][idx], 'K') # relh = meteorology.relh(t, d).value("%") heat = "%5.1f" % (meteorology.heatindex(t, d).value("F"), ) drct = s2(nc.variables['windDir'][idx]) smps = s2(nc.variables['windSpeed'][idx]) sped = "M" if smps != "M": sped = "%5.1f" % (nc.variables['windSpeed'][idx] * 2.23694, ) wcht = "M" if tmpf != "M" and sped != "M": t = temperature(nc.variables['temperature'][idx], 'K') sped = speed(nc.variables['windSpeed'][idx], 'MPS') wcht = "%5.1f" % (meteorology.windchill(t, sped).value("F"), ) ts = indices[sid]['ts'] out.write(("%5.5s %25.25s %8.4f %10.4f " "%02i %02i %5s %5s %5s %5s %5s %5s\n") % (sid, name, latitude, longitude, ts.hour, ts.minute, tmpf, dwpf, drct, sped, heat, wcht)) nc.close() out.close() pqstr = "data c 000000000000 wxc/wxc_%s.txt bogus txt" % ( network.lower(), ) subprocess.call("/home/ldm/bin/pqinsert -p '%s' %s" % (pqstr, wxcfn), shell=True) os.remove(wxcfn)
def init_year(ts): """ Create a new NetCDF file for a year of our specification! """ nc = ncopen("/mesonet/data/ndfd/ndfd_dailyc.nc", "w") nc.title = "NDFD" nc.contact = "Daryl Herzmann, [email protected], 515-294-5978" nc.history = "%s Generated" % ( datetime.datetime.now().strftime("%d %B %Y"), ) grbs = pygrib.open("/mesonet/ARCHIVE/data/2019/05/06/model/ndfd/" "00/ndfd.t00z.awp2p5f001.grib2") grb = grbs[1] shape = grb.values.shape lats, lons = grb.latlons() # Setup Dimensions nc.createDimension("lat", shape[0]) nc.createDimension("lon", shape[1]) ts2 = datetime.datetime(ts.year + 1, 1, 1) days = (ts2 - ts).days nc.createDimension("time", int(days)) # Setup Coordinate Variables lat = nc.createVariable("lat", np.float, ("lat", "lon")) lat.units = "degrees_north" lat.long_name = "Latitude" lat.standard_name = "latitude" lat.bounds = "lat_bnds" lat.axis = "Y" lat[:] = lats lon = nc.createVariable("lon", np.float, ("lat", "lon")) lon.units = "degrees_east" lon.long_name = "Longitude" lon.standard_name = "longitude" lon.bounds = "lon_bnds" lon.axis = "X" lon[:] = lons tm = nc.createVariable("time", np.float, ("time", )) tm.units = "Days since %s-01-01 00:00:0.0" % (ts.year, ) tm.long_name = "Time" tm.standard_name = "time" tm.axis = "T" tm.calendar = "gregorian" tm[:] = np.arange(0, int(days)) p01d = nc.createVariable("p01d", np.uint16, ("time", "lat", "lon"), fill_value=65535) p01d.units = "mm" p01d.scale_factor = 0.01 p01d.long_name = "Precipitation" p01d.standard_name = "Precipitation" p01d.coordinates = "lon lat" p01d.description = "Precipitation accumulation for the day" high = nc.createVariable("high_tmpk", np.uint16, ("time", "lat", "lon"), fill_value=65535) high.units = "K" high.scale_factor = 0.01 high.long_name = "2m Air Temperature Daily High" high.standard_name = "2m Air Temperature" high.coordinates = "lon lat" low = nc.createVariable("low_tmpk", np.uint16, ("time", "lat", "lon"), fill_value=65535) low.units = "K" low.scale_factor = 0.01 low.long_name = "2m Air Temperature Daily Low" low.standard_name = "2m Air Temperature" low.coordinates = "lon lat" gdd = nc.createVariable("gdd50", np.uint16, ("time", "lat", "lon"), fill_value=65535) gdd.units = "F" gdd.scale_factor = 0.01 gdd.long_name = "Growing Degree Days" gdd.standard_name = "Growing Degree Days" gdd.coordinates = "lon lat" nc.close()
def main(): """Go Main Go""" pgconn = get_dbconn("iem") icursor = pgconn.cursor() now = datetime.datetime.utcnow() - datetime.timedelta(hours=3) fn = "/mesonet/data/madis/metar/%s.nc" % (now.strftime("%Y%m%d_%H00"),) table = "current_qc" if not os.path.isfile(fn): sys.exit() nc = ncopen(fn) ids = chartostring(nc.variables["stationName"][:]) nc_tmpk = nc.variables["temperature"] nc_dwpk = nc.variables["dewpoint"] nc_alti = nc.variables["altimeter"] tmpkqcd = nc.variables["temperatureQCD"] dwpkqcd = nc.variables["dewpointQCD"] altiqcd = nc.variables["altimeterQCD"] for j in range(ids.shape[0]): sid = ids[j] if len(sid) < 4: continue if sid[0] == "K": ts = datetime.datetime(1970, 1, 1) + datetime.timedelta( seconds=int(nc.variables["timeObs"][j]) ) ts = ts.replace(tzinfo=pytz.utc) (tmpf, tmpf_qc_av, tmpf_qc_sc) = ("Null", "Null", "Null") (dwpf, dwpf_qc_av, dwpf_qc_sc) = ("Null", "Null", "Null") (alti, alti_qc_av, alti_qc_sc) = ("Null", "Null", "Null") if ( not np.ma.is_masked(nc_tmpk[j]) and not np.ma.is_masked(tmpkqcd[j, 0]) and not np.ma.is_masked(tmpkqcd[j, 6]) ): tmpf = check(temperature(nc_tmpk[j], "K").value("F")) tmpf_qc_av = figure(nc_tmpk[j], tmpkqcd[j, 0]) tmpf_qc_sc = figure(nc_tmpk[j], tmpkqcd[j, 6]) if ( not np.ma.is_masked(nc_dwpk[j]) and not np.ma.is_masked(dwpkqcd[j, 0]) and not np.ma.is_masked(dwpkqcd[j, 6]) ): dwpf = check(temperature(nc_dwpk[j], "K").value("F")) dwpf_qc_av = figure(nc_dwpk[j], dwpkqcd[j, 0]) dwpf_qc_sc = figure(nc_dwpk[j], dwpkqcd[j, 6]) if not np.ma.is_masked(nc_alti[j]): alti = check(nc_alti[j] / 100.0 * 0.0295298) alti_qc_av = figure_alti(altiqcd[j, 0] * 0.0295298) alti_qc_sc = figure_alti(altiqcd[j, 6] * 0.0295298) sql = """ UPDATE %s SET tmpf = %s, tmpf_qc_av = %s, tmpf_qc_sc = %s, dwpf = %s, dwpf_qc_av = %s, dwpf_qc_sc = %s, alti = %s, alti_qc_av = %s, alti_qc_sc = %s, valid = '%s' WHERE station = '%s' """ % ( table, tmpf, tmpf_qc_av, tmpf_qc_sc, dwpf, dwpf_qc_av, dwpf_qc_sc, alti, alti_qc_av, alti_qc_sc, ts.strftime("%Y-%m-%d %H:%M+00"), sid[1:], ) sql = sql.replace("--", "Null").replace("nan", "Null") try: icursor.execute(sql) except Exception as exp: print(exp) print(sql) nc.close() icursor.close() pgconn.commit() pgconn.close()